Skip to main content

durable_workflow/
lib.rs

1#![doc = include_str!("../README.md")]
2
3use std::{
4    any::{type_name, Any, TypeId},
5    collections::{BTreeMap, HashMap},
6    future::Future,
7    io::{self, Read},
8    pin::Pin,
9    sync::{
10        atomic::{AtomicBool, Ordering},
11        Arc, Mutex, OnceLock,
12    },
13    task::{Context as TaskContext, Poll},
14    time::{Duration, Instant, SystemTime, UNIX_EPOCH},
15};
16
17use apache_avro::{from_avro_datum, to_avro_datum, types::Value as AvroDatum, Schema};
18use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
19use chrono::DateTime;
20use futures_util::{future::OptionFuture, task::noop_waker_ref};
21use serde::{
22    de::DeserializeOwned,
23    ser::{SerializeMap, SerializeSeq},
24    Deserialize, Deserializer, Serialize, Serializer,
25};
26pub use serde_json::{json, Value};
27use sha2::{Digest, Sha256};
28use thiserror::Error;
29pub use uuid::Uuid;
30
31pub const WORKER_PROTOCOL_VERSION: &str = "1.19";
32/// First additive worker protocol that defines portable worker-affinity features.
33pub const PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION: &str = "1.18";
34pub const CONTROL_PLANE_VERSION: &str = "2";
35pub const DEFAULT_CODEC: &str = "avro";
36pub const SDK_VERSION: &str = concat!("durable-workflow-rust/", env!("CARGO_PKG_VERSION"));
37/// Worker-registration capability for authored condition-wait occurrence identity.
38pub const CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY: &str =
39    "condition_wait_occurrence_identity";
40/// Worker-registration capability for portable memo upserts.
41pub const MEMO_UPSERTS_CAPABILITY: &str = "memo_upserts";
42/// Worker-registration capability for server-routed read-only queries.
43pub const QUERY_TASKS_CAPABILITY: &str = "query_tasks";
44/// Worker-registration capability for canonical typed search attributes.
45pub const TYPED_SEARCH_ATTRIBUTES_CAPABILITY: &str = "typed_search_attributes";
46/// Worker-registration capability for synchronous workflow updates.
47pub const WORKFLOW_UPDATES_CAPABILITY: &str = "workflow_updates";
48/// Worker-registration capability for durable named input streams.
49pub const MESSAGE_STREAMS_CAPABILITY: &str = "message_streams";
50/// Worker-registration capability for persisted first-completion selection.
51pub const DURABLE_SELECTION_CAPABILITY: &str = "durable_selection";
52pub const MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.15";
53pub const MESSAGE_STREAM_SIGNAL: &str = "__durable_workflow_message_stream";
54pub const MESSAGE_STREAM_SCHEMA: &str = "durable-workflow.v2.message-stream.message";
55pub const MESSAGE_STREAM_CURSOR_SCHEMA: &str = "durable-workflow.v2.message-stream.cursor";
56pub const MESSAGE_STREAM_MAX_BATCH: usize = 100;
57/// First additive worker protocol that defines query-task transport.
58pub const QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
59/// First additive worker protocol that defines typed search-attribute upserts.
60pub const SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
61/// First additive worker protocol that defines portable memo upserts.
62pub const MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.14";
63/// First additive worker protocol that preserves declared search-attribute types.
64pub const TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.16";
65/// First additive worker protocol that defines external durable condition waits.
66pub const CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.9";
67/// First additive worker protocol that preserves authored condition-wait occurrences.
68pub const CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.17";
69/// First additive worker protocol that defines durable selection groups.
70pub const DURABLE_SELECTION_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.19";
71
72pub fn worker_protocol_supports_message_streams(version: &str) -> bool {
73    let Some((major, minor)) = version.split_once('.') else {
74        return false;
75    };
76    major == "1" && minor.parse::<u64>().is_ok_and(|minor| minor >= 15)
77}
78
79fn validate_user_signal_name(signal_name: &str) -> Result<()> {
80    if signal_name == MESSAGE_STREAM_SIGNAL {
81        return Err(Error::Codec(format!(
82            "signal name {MESSAGE_STREAM_SIGNAL:?} is reserved by the workflow runtime"
83        )));
84    }
85    Ok(())
86}
87
88const MAX_LONG_POLL_TIMEOUT_SECONDS: u64 = 60;
89const WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE: &str =
90    "Workflow task waiting for scheduled history.";
91const WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE: &str = "WorkflowTaskWaitingForHistory";
92const MISSING_TASK_PAYLOAD_CODEC: &str = "\0missing-task-payload-codec";
93const NULL_TASK_PAYLOAD_CODEC: &str = "\0null-task-payload-codec";
94const NON_STRING_TASK_PAYLOAD_CODEC: &str = "\0non-string-task-payload-codec";
95const MAX_MEMO_ENTRIES: usize = 100;
96const MAX_MEMO_VALUE_SIZE_BYTES: usize = 10_240;
97const MAX_MEMO_TOTAL_SIZE_BYTES: usize = 65_536;
98
99const QUERY_TASK_FINAL_REJECTION_REASONS: &[&str] = &[
100    "lease_expired",
101    "query_task_not_found",
102    "query_task_not_leased",
103    "query_task_timed_out",
104];
105
106/// Truthful service-worker manifest for features this SDK currently refuses.
107pub fn portable_worker_affinity_capability_manifest() -> Value {
108    json!({
109        "local_activities": {
110            "supported": false,
111            "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
112            "reason": "rust_worker_does_not_execute_record_local_activity",
113        },
114        "worker_sessions": {
115            "supported": false,
116            "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
117            "reason": "rust_worker_has_no_typed_session_lifecycle",
118        },
119        "sticky_execution": {
120            "supported": false,
121            "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
122            "reason": "rust_worker_uses_complete_durable_history_replay",
123        },
124    })
125}
126
127/// Canonical Avro Value schema packaged with the crate and parsed by the runtime.
128pub const AVRO_VALUE_SCHEMA_JSON: &str =
129    include_str!("../schema/durable_workflow.protocol.Value.v1.avsc");
130pub const AVRO_VALUE_SCHEMA_FINGERPRINT_HEX: &str = "e2a33dff55802237";
131pub const AVRO_VALUE_SCHEMA_FINGERPRINT: [u8; 8] = [0xe2, 0xa3, 0x3d, 0xff, 0x55, 0x80, 0x22, 0x37];
132const AVRO_SINGLE_OBJECT_MAGIC: [u8; 2] = [0xc3, 0x01];
133
134static AVRO_VALUE_SCHEMA: OnceLock<std::result::Result<Schema, String>> = OnceLock::new();
135static AVRO_VALUE_ORDERED_MAP_ENCODING_SCHEMA: OnceLock<std::result::Result<Schema, String>> =
136    OnceLock::new();
137
138#[derive(Clone, Copy)]
139enum RequestProtocol {
140    ControlPlane,
141    Worker(&'static str),
142}
143
144pub type Result<T> = std::result::Result<T, Error>;
145
146#[derive(Debug, Error)]
147pub enum Error {
148    #[error("transport error: {0}")]
149    Transport(#[from] reqwest::Error),
150    #[error(
151        "invalid Durable Workflow base URL: omit the SDK-owned /api suffix and pass the Server or Cloud runtime base URL; the SDK appends /api automatically"
152    )]
153    InvalidBaseUrl,
154    #[error("json error: {0}")]
155    Json(#[from] serde_json::Error),
156    #[error("http {status}: {body}")]
157    Http {
158        status: reqwest::StatusCode,
159        body: String,
160    },
161    #[error("codec error: {0}")]
162    Codec(String),
163    #[error(transparent)]
164    QueryFailed(QueryFailure),
165    #[error(transparent)]
166    Protocol(ProtocolFailure),
167    #[error(transparent)]
168    NonDeterministicReplay(ReplayFailure),
169    #[error(transparent)]
170    ChildWorkflowFailed(ChildWorkflowFailure),
171    #[error(transparent)]
172    ActivityFailed(ActivityFailure),
173    #[error(transparent)]
174    ParallelFailed(ParallelFailure),
175    #[error(transparent)]
176    SagaCompensationFailed(SagaCompensationFailure),
177    #[error(transparent)]
178    InvalidParallelGroup(ParallelGroupError),
179    #[error(transparent)]
180    DurableOperationCancelled(DurableOperationCancelled),
181    #[error(transparent)]
182    WorkflowCancellationRequested(WorkflowCancellationRequested),
183    #[error(transparent)]
184    WorkflowCommandRejected(WorkflowCommandRejection),
185    #[error(transparent)]
186    WorkflowFailed(WorkflowTerminalOutcome),
187    #[error(transparent)]
188    WorkflowCancelled(WorkflowTerminalOutcome),
189    #[error(transparent)]
190    WorkflowTerminated(WorkflowTerminalOutcome),
191    #[error(transparent)]
192    WorkflowTimedOut(WorkflowTerminalOutcome),
193    #[error(transparent)]
194    ActivityTaskRejected(ActivityTaskRejection),
195    #[error("workflow handler {0:?} is not registered")]
196    WorkflowNotRegistered(String),
197    #[error("activity handler {0:?} is not registered")]
198    ActivityNotRegistered(String),
199    #[error(
200        "{handler_kind} handler {handler_name:?} {value_kind} type {rust_type} is incompatible with the fixed Avro Value codec: {message}"
201    )]
202    HandlerType {
203        handler_kind: HandlerKind,
204        handler_name: String,
205        value_kind: HandlerValueKind,
206        rust_type: &'static str,
207        message: String,
208    },
209    #[error("workflow future yielded without emitting a durable command")]
210    WorkflowYieldedWithoutCommand,
211    #[error(
212        "workflow_stream_command_identity_missing: workflow stream authoring requires a non-empty server-provided workflow_command_id"
213    )]
214    MissingWorkflowCommandIdentity,
215    #[error("workflow state lock is poisoned")]
216    WorkflowStatePoisoned,
217    #[error("timer duration is too large for the worker protocol")]
218    TimerDurationOverflow,
219    #[error(transparent)]
220    InvalidConditionWaitOptions(#[from] ConditionWaitOptionsError),
221    #[error(transparent)]
222    InvalidSearchAttributeUpdate(#[from] SearchAttributeUpdateError),
223    #[error("operation timed out")]
224    Timeout,
225    #[error(
226        "missing {role}-plane credentials: configure ClientBuilder::{role}_token or ClientBuilder::token; a {opposite_role}-plane token cannot authorize this request"
227    )]
228    MissingRoleCredentials {
229        role: &'static str,
230        opposite_role: &'static str,
231    },
232    #[error("worker loop error: {0}")]
233    WorkerLoop(String),
234    #[error(
235        "workflow command contract for {workflow_type:?} declares update validators, but this Rust SDK cannot execute synchronous pre-accept update validation"
236    )]
237    UnsupportedUpdateValidators { workflow_type: String },
238    #[error("{primary}; worker deregistration also failed: {deregistration}")]
239    WorkerShutdown {
240        primary: Box<Error>,
241        deregistration: Box<Error>,
242    },
243    #[error("invalid child workflow options: {0}")]
244    InvalidChildWorkflowOptions(String),
245    #[error("invalid workflow memo update: {0}")]
246    InvalidMemoUpdate(String),
247    #[error(
248        "workflow_memo_updates_unavailable: the connected runtime did not advertise workflow memo update support"
249    )]
250    WorkflowMemoUpdatesUnavailable,
251    #[error(transparent)]
252    InvalidActivityOptions(ActivityOptionsError),
253    #[error(transparent)]
254    InvalidContinueAsNewOptions(#[from] ContinueAsNewOptionsError),
255    #[doc(hidden)]
256    #[error("workflow requested continue as new")]
257    ContinueAsNew(ContinueAsNewRequest),
258}
259
260/// Validation failure for a durable condition-wait definition.
261#[derive(Clone, Debug, Error, PartialEq, Eq)]
262pub enum ConditionWaitOptionsError {
263    #[error("condition_key must be non-empty")]
264    EmptyKey,
265    #[error("condition_definition_fingerprint must be non-empty")]
266    EmptyPredicateIdentity,
267    #[error("condition timeout is too large for the worker protocol")]
268    TimeoutOverflow,
269}
270
271/// Stable identity and optional durable timeout for a condition wait.
272///
273/// `predicate_identity` is recorded as the worker protocol's
274/// `condition_definition_fingerprint` and must change whenever predicate
275/// behavior changes. Prefer the [`wait_condition!`] macro when the predicate
276/// is written inline; it derives this identity from the predicate tokens.
277#[derive(Clone, Debug, PartialEq, Eq)]
278pub struct ConditionWaitOptions {
279    condition_key: String,
280    predicate_identity: String,
281    timeout: Option<Duration>,
282}
283
284impl ConditionWaitOptions {
285    pub fn new(condition_key: impl Into<String>, predicate_identity: impl Into<String>) -> Self {
286        Self {
287            condition_key: condition_key.into(),
288            predicate_identity: predicate_identity.into(),
289            timeout: None,
290        }
291    }
292
293    pub fn timeout(mut self, timeout: Duration) -> Self {
294        self.timeout = Some(timeout);
295        self
296    }
297
298    fn validate(
299        &self,
300    ) -> std::result::Result<ValidatedConditionWaitOptions, ConditionWaitOptionsError> {
301        let condition_key = self.condition_key.trim();
302        if condition_key.is_empty() {
303            return Err(ConditionWaitOptionsError::EmptyKey);
304        }
305        let predicate_identity = self.predicate_identity.trim();
306        if predicate_identity.is_empty() {
307            return Err(ConditionWaitOptionsError::EmptyPredicateIdentity);
308        }
309        let timeout_seconds = self
310            .timeout
311            .map(|timeout| {
312                timeout
313                    .as_secs()
314                    .checked_add(u64::from(timeout.subsec_nanos() > 0))
315                    .ok_or(ConditionWaitOptionsError::TimeoutOverflow)
316            })
317            .transpose()?;
318
319        Ok(ValidatedConditionWaitOptions {
320            condition_key: condition_key.to_string(),
321            predicate_identity: predicate_identity.to_string(),
322            timeout_seconds,
323        })
324    }
325}
326
327#[derive(Clone, Debug, PartialEq, Eq)]
328struct ValidatedConditionWaitOptions {
329    condition_key: String,
330    predicate_identity: String,
331    timeout_seconds: Option<u64>,
332}
333
334const CONDITION_WAIT_OCCURRENCE_PREFIX: &str = "rust:condition-wait:";
335
336/// Unambiguous terminal result of a durable condition wait.
337#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
338#[serde(rename_all = "snake_case")]
339pub enum ConditionWaitResult {
340    Satisfied,
341    TimedOut,
342}
343
344impl ConditionWaitResult {
345    pub fn is_satisfied(self) -> bool {
346        self == Self::Satisfied
347    }
348
349    pub fn is_timed_out(self) -> bool {
350        self == Self::TimedOut
351    }
352}
353
354/// Build the stable condition definition identity used by [`wait_condition!`].
355#[doc(hidden)]
356pub fn __condition_definition_fingerprint(source: &str) -> String {
357    let mut digest = Sha256::new();
358    digest.update(b"durable-workflow-rust.wait-condition.v1\0");
359    digest.update(source.as_bytes());
360    format!("sha256:{:x}", digest.finalize())
361}
362
363/// Create a durable condition wait whose predicate definition is fingerprinted
364/// from its inline Rust tokens.
365///
366/// The returned [`ConditionWaitCall`] must be awaited. The timeout form is
367/// `wait_condition!(ctx, "approval", timeout: duration, || predicate)`.
368#[macro_export]
369macro_rules! wait_condition {
370    ($ctx:expr, $key:expr, timeout: $timeout:expr, $predicate:expr $(,)?) => {{
371        $ctx.wait_condition(
372            $crate::ConditionWaitOptions::new(
373                $key,
374                $crate::__condition_definition_fingerprint(concat!(
375                    module_path!(),
376                    "\0",
377                    stringify!($predicate)
378                )),
379            )
380            .timeout($timeout),
381            $predicate,
382        )
383    }};
384    ($ctx:expr, $key:expr, $predicate:expr $(,)?) => {{
385        $ctx.wait_condition(
386            $crate::ConditionWaitOptions::new(
387                $key,
388                $crate::__condition_definition_fingerprint(concat!(
389                    module_path!(),
390                    "\0",
391                    stringify!($predicate)
392                )),
393            ),
394            $predicate,
395        )
396    }};
397}
398
399const MAX_SEARCH_ATTRIBUTES_PER_UPDATE: usize = 100;
400const MAX_SEARCH_ATTRIBUTE_KEY_LENGTH: usize = 64;
401const MAX_SEARCH_ATTRIBUTE_STRING_LENGTH: usize = 255;
402const MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH: usize = 255;
403const MAX_SEARCH_ATTRIBUTE_UPDATE_BYTES: usize = 65_536;
404
405/// Validation failure for a typed workflow search-attribute update.
406#[derive(Clone, Debug, Error, PartialEq, Eq)]
407pub enum SearchAttributeUpdateError {
408    #[error("search-attribute update requires at least one attribute")]
409    Empty,
410    #[error("search attribute key {0:?} must be 1-64 URL-safe ASCII characters")]
411    InvalidKey(String),
412    #[error("search-attribute update exceeds the limit of 100 attributes")]
413    TooManyAttributes,
414    #[error("search attribute {key:?} {kind} value exceeds {limit} bytes")]
415    ValueTooLong {
416        key: String,
417        kind: &'static str,
418        limit: usize,
419    },
420    #[error(
421        "search attribute {0:?} must not contain an empty string value; use delete() to remove it"
422    )]
423    EmptyString(String),
424    #[error("search attribute {0:?} has a non-finite float value")]
425    NonFiniteFloat(String),
426    #[error("search attribute {0:?} must use an RFC 3339 datetime with an explicit timezone")]
427    InvalidDateTime(String),
428    #[error("search-attribute update exceeds the 65536-byte protocol limit")]
429    PayloadTooLarge,
430}
431
432/// One public typed search-attribute value.
433#[derive(Clone, Debug, PartialEq)]
434pub enum SearchAttributeValue {
435    String(String),
436    Keyword(String),
437    KeywordList(Vec<String>),
438    Int(i64),
439    Float(f64),
440    Bool(bool),
441    DateTime(String),
442    Delete,
443}
444
445impl SearchAttributeValue {
446    fn type_name(&self) -> Option<&'static str> {
447        match self {
448            Self::String(_) => Some("string"),
449            Self::Keyword(_) => Some("keyword"),
450            Self::KeywordList(_) => Some("keyword_list"),
451            Self::Int(_) => Some("int"),
452            Self::Float(_) => Some("float"),
453            Self::Bool(_) => Some("bool"),
454            Self::DateTime(_) => Some("datetime"),
455            Self::Delete => None,
456        }
457    }
458
459    fn normalized(self, key: &str) -> std::result::Result<Self, SearchAttributeUpdateError> {
460        let normalize_string = |value: String, kind: &'static str, limit: usize| {
461            let value = value.trim().to_string();
462            if value.is_empty() {
463                return Err(SearchAttributeUpdateError::EmptyString(key.to_string()));
464            }
465            if value.len() > limit {
466                return Err(SearchAttributeUpdateError::ValueTooLong {
467                    key: key.to_string(),
468                    kind,
469                    limit,
470                });
471            }
472            Ok(value)
473        };
474
475        match self {
476            Self::String(value) => Ok(Self::String(normalize_string(
477                value,
478                "string",
479                MAX_SEARCH_ATTRIBUTE_STRING_LENGTH,
480            )?)),
481            Self::Keyword(value) => Ok(Self::Keyword(normalize_string(
482                value,
483                "keyword",
484                MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH,
485            )?)),
486            Self::KeywordList(values) => {
487                let values = values
488                    .into_iter()
489                    .map(|value| {
490                        let value = value.trim().to_string();
491                        if value.len() > MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH {
492                            return Err(SearchAttributeUpdateError::ValueTooLong {
493                                key: key.to_string(),
494                                kind: "keyword-list entry",
495                                limit: MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH,
496                            });
497                        }
498                        Ok(value)
499                    })
500                    .collect::<std::result::Result<Vec<_>, _>>()?;
501                Ok(Self::KeywordList(values))
502            }
503            Self::Float(value) if !value.is_finite() => {
504                Err(SearchAttributeUpdateError::NonFiniteFloat(key.to_string()))
505            }
506            Self::DateTime(value) => {
507                let value =
508                    normalize_string(value, "datetime", MAX_SEARCH_ATTRIBUTE_STRING_LENGTH)?;
509                if DateTime::parse_from_rfc3339(&value).is_err() {
510                    return Err(SearchAttributeUpdateError::InvalidDateTime(key.to_string()));
511                }
512                Ok(Self::DateTime(value))
513            }
514            value => Ok(value),
515        }
516    }
517
518    fn into_json(self) -> Value {
519        match self {
520            Self::String(value) | Self::Keyword(value) | Self::DateTime(value) => {
521                Value::String(value)
522            }
523            Self::KeywordList(values) => {
524                Value::Array(values.into_iter().map(Value::String).collect())
525            }
526            Self::Int(value) => json!(value),
527            Self::Float(value) => json!(value),
528            Self::Bool(value) => json!(value),
529            Self::Delete => Value::Null,
530        }
531    }
532}
533
534/// Validated typed workflow-side search-attribute mutation.
535#[derive(Clone, Debug, Default, PartialEq)]
536pub struct SearchAttributeUpdate {
537    attributes: BTreeMap<String, SearchAttributeValue>,
538}
539
540impl SearchAttributeUpdate {
541    pub fn new() -> Self {
542        Self::default()
543    }
544
545    pub fn set(
546        mut self,
547        key: impl Into<String>,
548        value: SearchAttributeValue,
549    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
550        let key = key.into();
551        validate_search_attribute_key(&key)?;
552        if !self.attributes.contains_key(&key)
553            && self.attributes.len() >= MAX_SEARCH_ATTRIBUTES_PER_UPDATE
554        {
555            return Err(SearchAttributeUpdateError::TooManyAttributes);
556        }
557        self.attributes.insert(key.clone(), value.normalized(&key)?);
558        self.validate_size()?;
559        Ok(self)
560    }
561
562    pub fn string(
563        self,
564        key: impl Into<String>,
565        value: impl Into<String>,
566    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
567        self.set(key, SearchAttributeValue::String(value.into()))
568    }
569
570    pub fn keyword(
571        self,
572        key: impl Into<String>,
573        value: impl Into<String>,
574    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
575        self.set(key, SearchAttributeValue::Keyword(value.into()))
576    }
577
578    pub fn keyword_list<I, V>(
579        self,
580        key: impl Into<String>,
581        values: I,
582    ) -> std::result::Result<Self, SearchAttributeUpdateError>
583    where
584        I: IntoIterator<Item = V>,
585        V: Into<String>,
586    {
587        self.set(
588            key,
589            SearchAttributeValue::KeywordList(values.into_iter().map(Into::into).collect()),
590        )
591    }
592
593    pub fn int(
594        self,
595        key: impl Into<String>,
596        value: i64,
597    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
598        self.set(key, SearchAttributeValue::Int(value))
599    }
600
601    pub fn float(
602        self,
603        key: impl Into<String>,
604        value: f64,
605    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
606        self.set(key, SearchAttributeValue::Float(value))
607    }
608
609    pub fn bool(
610        self,
611        key: impl Into<String>,
612        value: bool,
613    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
614        self.set(key, SearchAttributeValue::Bool(value))
615    }
616
617    pub fn datetime(
618        self,
619        key: impl Into<String>,
620        value: impl Into<String>,
621    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
622        self.set(key, SearchAttributeValue::DateTime(value.into()))
623    }
624
625    pub fn delete(
626        self,
627        key: impl Into<String>,
628    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
629        self.set(key, SearchAttributeValue::Delete)
630    }
631
632    fn validate_size(&self) -> std::result::Result<(), SearchAttributeUpdateError> {
633        let (attributes, _) = self.clone().into_wire_parts();
634        if serde_json::to_vec(&attributes)
635            .map(|payload| payload.len() > MAX_SEARCH_ATTRIBUTE_UPDATE_BYTES)
636            .unwrap_or(true)
637        {
638            return Err(SearchAttributeUpdateError::PayloadTooLarge);
639        }
640        Ok(())
641    }
642
643    fn into_wire_parts(self) -> (Value, BTreeMap<String, String>) {
644        let mut attributes = serde_json::Map::new();
645        let mut attribute_types = BTreeMap::new();
646        for (key, value) in self.attributes {
647            if let Some(type_name) = value.type_name() {
648                attribute_types.insert(key.clone(), type_name.to_string());
649            }
650            attributes.insert(key, value.into_json());
651        }
652        (Value::Object(attributes), attribute_types)
653    }
654
655    fn validate(&self) -> std::result::Result<(), SearchAttributeUpdateError> {
656        if self.attributes.is_empty() {
657            return Err(SearchAttributeUpdateError::Empty);
658        }
659        self.validate_size()
660    }
661}
662
663fn validate_search_attribute_key(key: &str) -> std::result::Result<(), SearchAttributeUpdateError> {
664    let valid = !key.is_empty()
665        && key.len() <= MAX_SEARCH_ATTRIBUTE_KEY_LENGTH
666        && key
667            .bytes()
668            .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-' | b':'));
669    if valid {
670        Ok(())
671    } else {
672        Err(SearchAttributeUpdateError::InvalidKey(key.to_string()))
673    }
674}
675
676/// The registered handler family reported by [`Error::HandlerType`].
677#[derive(Clone, Copy, Debug, PartialEq, Eq)]
678pub enum HandlerKind {
679    Workflow,
680    Activity,
681}
682
683impl std::fmt::Display for HandlerKind {
684    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
685        formatter.write_str(match self {
686            Self::Workflow => "workflow",
687            Self::Activity => "activity",
688        })
689    }
690}
691
692/// Whether a typed handler failed to adapt its input or result.
693#[derive(Clone, Copy, Debug, PartialEq, Eq)]
694pub enum HandlerValueKind {
695    Input,
696    Result,
697}
698
699impl std::fmt::Display for HandlerValueKind {
700    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
701        formatter.write_str(match self {
702            Self::Input => "input",
703            Self::Result => "result",
704        })
705    }
706}
707
708/// The lifecycle command sent to a workflow execution.
709#[derive(Clone, Copy, Debug, PartialEq, Eq)]
710pub enum WorkflowCommandKind {
711    Cancel,
712    Terminate,
713}
714
715impl WorkflowCommandKind {
716    fn as_str(self) -> &'static str {
717        match self {
718            Self::Cancel => "cancel",
719            Self::Terminate => "terminate",
720        }
721    }
722}
723
724/// Optional structured fields for a cancellation or termination request.
725#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
726pub struct WorkflowCommandOptions {
727    #[serde(skip_serializing_if = "Option::is_none")]
728    pub reason: Option<String>,
729    #[serde(skip_serializing_if = "Option::is_none")]
730    pub request_id: Option<String>,
731}
732
733/// Server-enforced timeout policy for a workflow start.
734///
735/// These deadlines are distinct from [`WorkflowResultOptions::timeout`], which
736/// only bounds how long the caller waits. A server deadline produces a terminal
737/// [`Error::WorkflowTimedOut`] outcome whose reason is `execution_timeout` or
738/// `run_timeout`.
739#[derive(Clone, Debug, PartialEq, Eq)]
740pub struct WorkflowStartOptions {
741    pub execution_timeout_seconds: u64,
742    pub run_timeout_seconds: u64,
743}
744
745impl Default for WorkflowStartOptions {
746    fn default() -> Self {
747        Self {
748            execution_timeout_seconds: 3600,
749            run_timeout_seconds: 600,
750        }
751    }
752}
753
754impl WorkflowStartOptions {
755    pub fn new() -> Self {
756        Self::default()
757    }
758
759    pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
760        self.execution_timeout_seconds = seconds;
761        self
762    }
763
764    pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
765        self.run_timeout_seconds = seconds;
766        self
767    }
768
769    fn validate(&self) -> Result<()> {
770        if self.execution_timeout_seconds == 0 {
771            return Err(Error::Codec(
772                "execution_timeout_seconds must be at least 1".to_string(),
773            ));
774        }
775        if self.run_timeout_seconds == 0 {
776            return Err(Error::Codec(
777                "run_timeout_seconds must be at least 1".to_string(),
778            ));
779        }
780        if self.run_timeout_seconds > self.execution_timeout_seconds {
781            return Err(Error::Codec(
782                "run_timeout_seconds cannot exceed execution_timeout_seconds".to_string(),
783            ));
784        }
785
786        Ok(())
787    }
788}
789
790/// Optional routing overrides for a continue-as-new transition.
791///
792/// Omitted values retain the current workflow type and task queue. Server-owned
793/// instance metadata is not accepted here and is carried by the server.
794#[derive(Clone, Debug, Default, PartialEq, Eq)]
795pub struct ContinueAsNewOptions {
796    pub workflow_type: Option<String>,
797    pub task_queue: Option<String>,
798}
799
800impl ContinueAsNewOptions {
801    pub fn new() -> Self {
802        Self::default()
803    }
804
805    pub fn workflow_type(mut self, workflow_type: impl Into<String>) -> Self {
806        self.workflow_type = Some(workflow_type.into());
807        self
808    }
809
810    pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
811        self.task_queue = Some(task_queue.into());
812        self
813    }
814
815    fn validate(&self) -> std::result::Result<(), ContinueAsNewOptionsError> {
816        for (field, value) in [
817            ("workflow_type", self.workflow_type.as_deref()),
818            ("task_queue", self.task_queue.as_deref()),
819        ] {
820            if value.is_some_and(|value| value.trim().is_empty()) {
821                return Err(ContinueAsNewOptionsError {
822                    field,
823                    message: format!("{field} must not be empty"),
824                });
825            }
826        }
827        Ok(())
828    }
829}
830
831/// A stable validation error raised before a continue-as-new command is emitted.
832#[derive(Clone, Debug, Error, PartialEq, Eq)]
833#[error("invalid continue-as-new option {field}: {message}")]
834pub struct ContinueAsNewOptionsError {
835    pub field: &'static str,
836    pub message: String,
837}
838
839/// Public history-budget information attached to the current workflow task.
840#[derive(Clone, Debug, Default, PartialEq, Eq)]
841pub struct WorkflowHistoryBudget {
842    pub event_count: u64,
843    pub size_bytes: Option<u64>,
844    pub continue_as_new_recommended: bool,
845    pub pressure: Option<String>,
846}
847
848#[doc(hidden)]
849#[derive(Clone, Debug)]
850pub struct ContinueAsNewRequest {
851    arguments: AvroValue,
852    options: ContinueAsNewOptions,
853}
854
855impl WorkflowCommandOptions {
856    pub fn new() -> Self {
857        Self::default()
858    }
859
860    pub fn reason(mut self, reason: impl Into<String>) -> Self {
861        self.reason = Some(reason.into());
862        self
863    }
864
865    pub fn request_id(mut self, request_id: impl Into<String>) -> Self {
866        self.request_id = Some(request_id.into());
867        self
868    }
869}
870
871/// The accepted, machine-readable result of a lifecycle command.
872#[derive(Clone, Debug, PartialEq)]
873pub struct WorkflowCommandResult {
874    pub command: WorkflowCommandKind,
875    pub workflow_id: String,
876    pub run_id: Option<String>,
877    pub outcome: Option<String>,
878    pub reason: Option<String>,
879    pub command_status: Option<String>,
880    pub raw: Value,
881}
882
883/// A stable rejection returned by instance- or selected-run lifecycle commands.
884#[derive(Clone, Debug, Error)]
885#[error("workflow {command:?} rejected ({reason}, HTTP {status}): {message}")]
886pub struct WorkflowCommandRejection {
887    pub command: WorkflowCommandKind,
888    pub status: u16,
889    pub reason: String,
890    pub message: String,
891    pub workflow_id: String,
892    pub run_id: Option<String>,
893    pub target_scope: Option<String>,
894    pub body: Value,
895}
896
897/// Stable terminal categories returned by [`WorkflowHandle::result`].
898#[derive(Clone, Copy, Debug, PartialEq, Eq)]
899pub enum WorkflowTerminalKind {
900    Failed,
901    Cancelled,
902    Terminated,
903    TimedOut,
904}
905
906/// A typed terminal workflow outcome with durable identity and failure metadata.
907///
908/// Match the corresponding [`enum@Error`] variant and inspect these fields instead
909/// of parsing its display representation. Fields remain `None` when an older
910/// server did not publish that metadata.
911#[derive(Clone, Debug, Error)]
912#[error("workflow {workflow_id} run {run_id:?} ended as {kind:?} ({reason})")]
913pub struct WorkflowTerminalOutcome {
914    pub kind: WorkflowTerminalKind,
915    pub workflow_id: String,
916    pub run_id: Option<String>,
917    pub reason: String,
918    pub failure_category: Option<String>,
919    pub failure_id: Option<String>,
920    pub exception_type: Option<String>,
921    pub exception_class: Option<String>,
922    pub non_retryable: Option<bool>,
923    pub message: Option<String>,
924    pub exception: Option<Value>,
925    pub raw: Value,
926}
927
928/// A worker-side activity settlement or heartbeat rejected by durable state.
929#[derive(Clone, Debug, Error)]
930#[error("activity task {operation} rejected ({reason}, HTTP {status})")]
931pub struct ActivityTaskRejection {
932    pub operation: String,
933    pub status: u16,
934    pub reason: String,
935    pub task_id: String,
936    pub activity_attempt_id: String,
937    pub cancel_requested: bool,
938    pub can_continue: Option<bool>,
939    pub run_closed_reason: Option<String>,
940    pub body: Value,
941}
942
943/// Stable validation categories for [`ActivityOptions`].
944#[derive(Clone, Copy, Debug, PartialEq, Eq)]
945pub enum ActivityOptionsErrorKind {
946    EmptyTaskQueue,
947    EmptyRetryPolicy,
948    InvalidMaxAttempts,
949    BackoffWithoutRetryBudget,
950    TooManyBackoffIntervals,
951    InvalidBackoffCoefficient,
952    BackoffGenerationTooLarge,
953    BackoffOverflow,
954    EmptyNonRetryableErrorType,
955    TimeoutNotPositive,
956    TimeoutOverflow,
957    TimeoutOrder,
958}
959
960/// A machine-readable activity-options validation failure.
961#[derive(Clone, Debug, Error, PartialEq, Eq)]
962#[error("invalid activity options ({kind:?}, {field:?}): {message}")]
963pub struct ActivityOptionsError {
964    pub kind: ActivityOptionsErrorKind,
965    pub field: Option<&'static str>,
966    pub message: String,
967}
968
969impl ActivityOptionsError {
970    fn new(
971        kind: ActivityOptionsErrorKind,
972        field: Option<&'static str>,
973        message: impl Into<String>,
974    ) -> Self {
975        Self {
976            kind,
977            field,
978            message: message.into(),
979        }
980    }
981}
982
983/// Stable terminal categories returned when an awaited activity does not succeed.
984#[derive(Clone, Copy, Debug, PartialEq, Eq)]
985pub enum ActivityFailureKind {
986    Failed,
987    Cancelled,
988    TimedOut,
989}
990
991/// A stable, machine-readable terminal activity failure.
992///
993/// Match [`Error::ActivityFailed`] and inspect `kind`, `reason`,
994/// `failure_category`, or `timeout_kind`; display text is only diagnostic.
995#[derive(Clone, Debug, Error)]
996#[error("activity failed ({reason}): {message}")]
997pub struct ActivityFailure {
998    pub kind: ActivityFailureKind,
999    pub reason: String,
1000    pub message: String,
1001    pub activity_execution_id: Option<String>,
1002    pub activity_attempt_id: Option<String>,
1003    pub activity_type: Option<String>,
1004    pub activity_class: Option<String>,
1005    pub attempt_number: Option<u64>,
1006    pub failure_id: Option<String>,
1007    pub failure_category: Option<String>,
1008    pub timeout_kind: Option<String>,
1009    pub non_retryable: bool,
1010    pub exception_type: Option<String>,
1011    pub exception_class: Option<String>,
1012    pub code: Option<Value>,
1013    pub exception: Option<Value>,
1014}
1015
1016/// Stable terminal categories returned when an awaited child does not succeed.
1017#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1018pub enum ChildWorkflowFailureKind {
1019    Failed,
1020    Cancelled,
1021    Terminated,
1022}
1023
1024/// A stable, machine-readable child workflow failure delivered to its parent.
1025///
1026/// Match [`Error::ChildWorkflowFailed`] and inspect `reason` or `kind` instead
1027/// of parsing the display message. Child and parent identifiers retain the
1028/// relationship recorded in durable history across worker restarts.
1029#[derive(Clone, Debug, Error)]
1030#[error("child workflow failed ({reason}): {message}")]
1031pub struct ChildWorkflowFailure {
1032    pub kind: ChildWorkflowFailureKind,
1033    pub reason: String,
1034    pub message: String,
1035    pub parent_workflow_id: Option<String>,
1036    pub parent_workflow_run_id: Option<String>,
1037    pub child_workflow_id: Option<String>,
1038    pub child_workflow_run_id: Option<String>,
1039    pub child_workflow_type: Option<String>,
1040    pub failure_id: Option<String>,
1041    pub failure_category: Option<String>,
1042    pub exception_type: Option<String>,
1043    pub exception_class: Option<String>,
1044    pub non_retryable: bool,
1045    pub code: Option<Value>,
1046    pub exception: Option<Value>,
1047}
1048
1049/// The identity of one durable workflow execution.
1050#[derive(Clone, Debug, PartialEq, Eq)]
1051pub struct WorkflowIdentity {
1052    pub workflow_id: Option<String>,
1053    pub run_id: Option<String>,
1054}
1055
1056/// A successful child result together with its durable parent-child identity.
1057#[derive(Clone, Debug, PartialEq)]
1058pub struct ChildWorkflowResult {
1059    pub parent: WorkflowIdentity,
1060    pub child: WorkflowIdentity,
1061    pub child_workflow_type: Option<String>,
1062    pub result: Value,
1063}
1064
1065/// Lossless successful child result for fixed Avro Value workflows.
1066#[derive(Clone, Debug, PartialEq)]
1067pub struct ChildWorkflowAvroResult {
1068    pub parent: WorkflowIdentity,
1069    pub child: WorkflowIdentity,
1070    pub child_workflow_type: Option<String>,
1071    pub result: AvroValue,
1072}
1073
1074/// Stable user-facing identity for one member of a durable selection group.
1075#[derive(Clone, Debug, Deserialize, Hash, PartialEq, Eq, Serialize)]
1076#[serde(untagged)]
1077pub enum SelectionKey {
1078    Index(usize),
1079    Name(String),
1080}
1081
1082impl From<usize> for SelectionKey {
1083    fn from(value: usize) -> Self {
1084        Self::Index(value)
1085    }
1086}
1087
1088impl From<String> for SelectionKey {
1089    fn from(value: String) -> Self {
1090        Self::Name(value)
1091    }
1092}
1093
1094impl From<&str> for SelectionKey {
1095    fn from(value: &str) -> Self {
1096        Self::Name(value.to_string())
1097    }
1098}
1099
1100/// Typed result of explicitly awaiting a cancelled non-winning operation.
1101#[derive(Clone, Debug, Error, PartialEq, Eq)]
1102#[error("selected {operation_kind} operation {operation_identity} was explicitly cancelled")]
1103pub struct DurableOperationCancelled {
1104    pub selection_group_id: String,
1105    pub member_key: SelectionKey,
1106    pub member_index: usize,
1107    pub operation_kind: String,
1108    pub operation_identity: String,
1109}
1110
1111/// Stable identity for one enclosing deterministic parallel group.
1112///
1113/// The same fields are attached to every ordinary activity, timer, or child
1114/// workflow command in the group. Nested leaves carry an outer-to-inner path;
1115/// no Rust-specific wire command is introduced.
1116#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
1117pub struct ParallelGroupMetadata {
1118    pub parallel_group_id: String,
1119    pub parallel_group_kind: String,
1120    pub parallel_group_base_sequence: u64,
1121    pub parallel_group_size: usize,
1122    pub parallel_group_index: usize,
1123    #[serde(default, skip_serializing_if = "Option::is_none")]
1124    pub parallel_group_mode: Option<String>,
1125    #[serde(default, skip_serializing_if = "Option::is_none")]
1126    pub selection_member_key: Option<SelectionKey>,
1127    #[serde(default, skip_serializing_if = "Option::is_none")]
1128    pub selection_member_index: Option<usize>,
1129    #[serde(default, skip_serializing_if = "Option::is_none")]
1130    pub selection_member_base_sequence: Option<u64>,
1131    #[serde(default, skip_serializing_if = "Option::is_none")]
1132    pub selection_member_size: Option<usize>,
1133    #[serde(default, skip_serializing_if = "Option::is_none")]
1134    pub selection_member_kind: Option<String>,
1135}
1136
1137/// One input-ordered result returned by [`WorkflowContext::parallel`].
1138#[derive(Clone, Debug, PartialEq)]
1139pub enum ParallelResult {
1140    Activity(Value),
1141    ChildWorkflow(ChildWorkflowResult),
1142    Timer,
1143    Signal(Vec<Value>),
1144    Condition(ConditionWaitResult),
1145    Group(Vec<ParallelResult>),
1146}
1147
1148/// Lossless fixed-Avro counterpart to [`ParallelResult`].
1149#[derive(Clone, Debug, PartialEq)]
1150pub enum ParallelAvroResult {
1151    Activity(AvroValue),
1152    ChildWorkflow(ChildWorkflowAvroResult),
1153    Timer,
1154    Signal(Vec<AvroValue>),
1155    Condition(ConditionWaitResult),
1156    Group(Vec<ParallelAvroResult>),
1157}
1158
1159impl ParallelAvroResult {
1160    fn into_json_result(self) -> Result<ParallelResult> {
1161        match self {
1162            Self::Activity(value) => Ok(ParallelResult::Activity(value.into_json()?)),
1163            Self::ChildWorkflow(result) => Ok(ParallelResult::ChildWorkflow(ChildWorkflowResult {
1164                parent: result.parent,
1165                child: result.child,
1166                child_workflow_type: result.child_workflow_type,
1167                result: result.result.into_json()?,
1168            })),
1169            Self::Timer => Ok(ParallelResult::Timer),
1170            Self::Signal(values) => Ok(ParallelResult::Signal(
1171                values
1172                    .into_iter()
1173                    .map(AvroValue::into_json)
1174                    .collect::<Result<Vec<_>>>()?,
1175            )),
1176            Self::Condition(result) => Ok(ParallelResult::Condition(result)),
1177            Self::Group(results) => Ok(ParallelResult::Group(
1178                results
1179                    .into_iter()
1180                    .map(Self::into_json_result)
1181                    .collect::<Result<Vec<_>>>()?,
1182            )),
1183        }
1184    }
1185}
1186
1187/// One successful leaf retained when another parallel member failed.
1188#[derive(Clone, Debug, PartialEq)]
1189pub struct ParallelCompletion {
1190    pub member_path: Vec<usize>,
1191    pub result: ParallelResult,
1192}
1193
1194/// A deterministic join failed after some siblings had already completed.
1195///
1196/// `cause` retains the typed activity, child-workflow, cancellation, or codec
1197/// error. `completed` is declaration ordered and contains only durable
1198/// successes observed in the same replay. Late sibling completions can add
1199/// entries on a later replay without changing `member_path` or the selected
1200/// positional failure.
1201#[derive(Debug, Error)]
1202#[error("parallel group {group_id} member {member_path:?} failed: {cause}")]
1203pub struct ParallelFailure {
1204    pub group_id: String,
1205    pub member_path: Vec<usize>,
1206    pub group_path: Vec<ParallelGroupMetadata>,
1207    pub completed: Vec<ParallelCompletion>,
1208    #[source]
1209    pub cause: Box<Error>,
1210}
1211
1212/// Stable validation error returned before an invalid group emits commands.
1213#[derive(Clone, Debug, Error, PartialEq, Eq)]
1214#[error("invalid deterministic parallel group ({reason}): {message}")]
1215pub struct ParallelGroupError {
1216    pub reason: &'static str,
1217    pub member_path: Vec<usize>,
1218    pub message: String,
1219}
1220
1221/// Cooperative workflow cancellation observed at an author-controlled point.
1222#[derive(Clone, Debug, Error, PartialEq, Eq)]
1223#[error("workflow cancellation was requested")]
1224pub struct WorkflowCancellationRequested;
1225
1226/// A forward saga failure followed by a terminal compensation failure.
1227#[derive(Debug, Error)]
1228#[error(
1229    "saga forward execution failed; compensation activity {compensation_activity_type} (registration {compensation_registration_order}) also failed: {compensation_failure}"
1230)]
1231pub struct SagaCompensationFailure {
1232    pub initiating_failure: Box<Error>,
1233    pub compensation_failure: Box<Error>,
1234    pub compensation_activity_type: String,
1235    pub compensation_registration_order: usize,
1236}
1237
1238/// Server behavior when a parent closes while its child is still open.
1239#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
1240pub enum ParentClosePolicy {
1241    #[default]
1242    Abandon,
1243    RequestCancel,
1244    Terminate,
1245}
1246
1247impl ParentClosePolicy {
1248    fn as_str(self) -> &'static str {
1249        match self {
1250            Self::Abandon => "abandon",
1251            Self::RequestCancel => "request_cancel",
1252            Self::Terminate => "terminate",
1253        }
1254    }
1255}
1256
1257/// Durable retry policy for one child workflow invocation.
1258#[derive(Clone, Debug, Default, PartialEq, Eq)]
1259pub struct ChildWorkflowRetryPolicy {
1260    pub max_attempts: Option<u32>,
1261    pub backoff_seconds: Vec<u64>,
1262    pub non_retryable_error_types: Vec<String>,
1263}
1264
1265/// Options recorded with a child-workflow command.
1266///
1267/// The task queue is mandatory so routing is explicit and replay-stable.
1268#[derive(Clone, Debug, PartialEq, Eq)]
1269pub struct ChildWorkflowOptions {
1270    pub task_queue: String,
1271    pub parent_close_policy: ParentClosePolicy,
1272    pub retry_policy: Option<ChildWorkflowRetryPolicy>,
1273    pub execution_timeout_seconds: Option<u64>,
1274    pub run_timeout_seconds: Option<u64>,
1275}
1276
1277impl ChildWorkflowOptions {
1278    pub fn new(task_queue: impl Into<String>) -> Self {
1279        Self {
1280            task_queue: task_queue.into(),
1281            parent_close_policy: ParentClosePolicy::Abandon,
1282            retry_policy: None,
1283            execution_timeout_seconds: None,
1284            run_timeout_seconds: None,
1285        }
1286    }
1287
1288    pub fn parent_close_policy(mut self, policy: ParentClosePolicy) -> Self {
1289        self.parent_close_policy = policy;
1290        self
1291    }
1292
1293    pub fn retry_policy(mut self, policy: ChildWorkflowRetryPolicy) -> Self {
1294        self.retry_policy = Some(policy);
1295        self
1296    }
1297
1298    pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
1299        self.execution_timeout_seconds = Some(seconds);
1300        self
1301    }
1302
1303    pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
1304        self.run_timeout_seconds = Some(seconds);
1305        self
1306    }
1307}
1308
1309/// Backoff intervals for one durable activity retry policy.
1310#[derive(Clone, Debug, PartialEq, Eq)]
1311pub enum ActivityBackoff {
1312    /// Use these intervals between attempts. The server repeats the final
1313    /// interval if the retry budget contains more attempts than entries.
1314    Explicit(Vec<Duration>),
1315    /// Generate one interval for every retry using integer exponential growth.
1316    Exponential {
1317        initial_interval: Duration,
1318        coefficient: u32,
1319        maximum_interval: Option<Duration>,
1320    },
1321}
1322
1323/// Durable server-side retry policy for one activity execution.
1324#[derive(Clone, Debug, Default, PartialEq, Eq)]
1325pub struct ActivityRetryPolicy {
1326    pub max_attempts: Option<u32>,
1327    pub backoff: Option<ActivityBackoff>,
1328    pub non_retryable_error_types: Vec<String>,
1329}
1330
1331impl ActivityRetryPolicy {
1332    /// Start a policy with a finite attempt budget, including the first attempt.
1333    pub fn new(max_attempts: u32) -> Self {
1334        Self {
1335            max_attempts: Some(max_attempts),
1336            ..Self::default()
1337        }
1338    }
1339
1340    pub fn backoff_intervals(mut self, intervals: impl IntoIterator<Item = Duration>) -> Self {
1341        self.backoff = Some(ActivityBackoff::Explicit(intervals.into_iter().collect()));
1342        self
1343    }
1344
1345    pub fn exponential_backoff(
1346        mut self,
1347        initial_interval: Duration,
1348        coefficient: u32,
1349        maximum_interval: Option<Duration>,
1350    ) -> Self {
1351        self.backoff = Some(ActivityBackoff::Exponential {
1352            initial_interval,
1353            coefficient,
1354            maximum_interval,
1355        });
1356        self
1357    }
1358
1359    pub fn non_retryable_error_type(mut self, error_type: impl Into<String>) -> Self {
1360        self.non_retryable_error_types.push(error_type.into());
1361        self
1362    }
1363
1364    pub fn non_retryable_error_types(
1365        mut self,
1366        error_types: impl IntoIterator<Item = impl Into<String>>,
1367    ) -> Self {
1368        self.non_retryable_error_types
1369            .extend(error_types.into_iter().map(Into::into));
1370        self
1371    }
1372}
1373
1374/// Options recorded atomically on one deterministic `schedule_activity` command.
1375///
1376/// Durations are rounded up to whole seconds when encoded, so the server never
1377/// receives a shorter timeout or backoff than the caller requested.
1378#[derive(Clone, Debug, Default, PartialEq, Eq)]
1379pub struct ActivityOptions {
1380    pub task_queue: Option<String>,
1381    pub retry_policy: Option<ActivityRetryPolicy>,
1382    pub start_to_close_timeout: Option<Duration>,
1383    pub schedule_to_start_timeout: Option<Duration>,
1384    pub schedule_to_close_timeout: Option<Duration>,
1385    pub heartbeat_timeout: Option<Duration>,
1386}
1387
1388impl ActivityOptions {
1389    pub fn new() -> Self {
1390        Self::default()
1391    }
1392
1393    pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
1394        self.task_queue = Some(task_queue.into());
1395        self
1396    }
1397
1398    pub fn retry_policy(mut self, policy: ActivityRetryPolicy) -> Self {
1399        self.retry_policy = Some(policy);
1400        self
1401    }
1402
1403    pub fn start_to_close_timeout(mut self, timeout: Duration) -> Self {
1404        self.start_to_close_timeout = Some(timeout);
1405        self
1406    }
1407
1408    pub fn schedule_to_start_timeout(mut self, timeout: Duration) -> Self {
1409        self.schedule_to_start_timeout = Some(timeout);
1410        self
1411    }
1412
1413    pub fn schedule_to_close_timeout(mut self, timeout: Duration) -> Self {
1414        self.schedule_to_close_timeout = Some(timeout);
1415        self
1416    }
1417
1418    pub fn heartbeat_timeout(mut self, timeout: Duration) -> Self {
1419        self.heartbeat_timeout = Some(timeout);
1420        self
1421    }
1422
1423    fn validate(&self) -> std::result::Result<ValidatedActivityOptions, ActivityOptionsError> {
1424        if self
1425            .task_queue
1426            .as_deref()
1427            .is_some_and(|queue| queue.trim().is_empty())
1428        {
1429            return Err(ActivityOptionsError::new(
1430                ActivityOptionsErrorKind::EmptyTaskQueue,
1431                Some("task_queue"),
1432                "task_queue must not be empty",
1433            ));
1434        }
1435
1436        for (field, value) in [
1437            ("start_to_close_timeout", self.start_to_close_timeout),
1438            ("schedule_to_start_timeout", self.schedule_to_start_timeout),
1439            ("schedule_to_close_timeout", self.schedule_to_close_timeout),
1440            ("heartbeat_timeout", self.heartbeat_timeout),
1441        ] {
1442            if value.is_some_and(|value| value.is_zero()) {
1443                return Err(ActivityOptionsError::new(
1444                    ActivityOptionsErrorKind::TimeoutNotPositive,
1445                    Some(field),
1446                    format!("{field} must be positive"),
1447                ));
1448            }
1449        }
1450
1451        validate_timeout_order(
1452            "heartbeat_timeout",
1453            self.heartbeat_timeout,
1454            "start_to_close_timeout",
1455            self.start_to_close_timeout,
1456        )?;
1457        validate_timeout_order(
1458            "start_to_close_timeout",
1459            self.start_to_close_timeout,
1460            "schedule_to_close_timeout",
1461            self.schedule_to_close_timeout,
1462        )?;
1463        validate_timeout_order(
1464            "schedule_to_start_timeout",
1465            self.schedule_to_start_timeout,
1466            "schedule_to_close_timeout",
1467            self.schedule_to_close_timeout,
1468        )?;
1469
1470        Ok(ValidatedActivityOptions {
1471            task_queue: self.task_queue.clone(),
1472            retry_policy: self
1473                .retry_policy
1474                .as_ref()
1475                .map(validate_activity_retry_policy)
1476                .transpose()?,
1477            start_to_close_timeout: timeout_seconds(
1478                "start_to_close_timeout",
1479                self.start_to_close_timeout,
1480            )?,
1481            schedule_to_start_timeout: timeout_seconds(
1482                "schedule_to_start_timeout",
1483                self.schedule_to_start_timeout,
1484            )?,
1485            schedule_to_close_timeout: timeout_seconds(
1486                "schedule_to_close_timeout",
1487                self.schedule_to_close_timeout,
1488            )?,
1489            heartbeat_timeout: timeout_seconds("heartbeat_timeout", self.heartbeat_timeout)?,
1490        })
1491    }
1492}
1493
1494/// A deferred durable leaf or nested group for [`WorkflowContext::parallel`].
1495///
1496/// Constructors capture arguments but perform no I/O. The join validates the
1497/// complete tree, attaches the existing parallel-group metadata to every
1498/// ordinary command, schedules all leaves, and then suspends.
1499pub enum ParallelOperation {
1500    Activity {
1501        activity_type: String,
1502        options: ActivityOptions,
1503        arguments: Result<AvroValue>,
1504    },
1505    ChildWorkflow {
1506        workflow_type: String,
1507        options: ChildWorkflowOptions,
1508        arguments: Result<AvroValue>,
1509    },
1510    Timer(Duration),
1511    Signal(String),
1512    Condition {
1513        options: ConditionWaitOptions,
1514        predicate: Box<dyn Fn() -> Result<bool> + Send + 'static>,
1515    },
1516    Group(Vec<ParallelOperation>),
1517}
1518
1519impl ParallelOperation {
1520    pub fn activity<T: Serialize>(activity_type: impl Into<String>, args: T) -> Self {
1521        Self::activity_with_options(activity_type, ActivityOptions::new(), args)
1522    }
1523
1524    pub fn activity_with_options<T: Serialize>(
1525        activity_type: impl Into<String>,
1526        options: ActivityOptions,
1527        args: T,
1528    ) -> Self {
1529        Self::Activity {
1530            activity_type: activity_type.into(),
1531            options,
1532            arguments: AvroValue::from_serialize(&args),
1533        }
1534    }
1535
1536    pub fn child_workflow<T: Serialize>(
1537        workflow_type: impl Into<String>,
1538        options: ChildWorkflowOptions,
1539        args: T,
1540    ) -> Self {
1541        Self::ChildWorkflow {
1542            workflow_type: workflow_type.into(),
1543            options,
1544            arguments: AvroValue::from_serialize(&args),
1545        }
1546    }
1547
1548    pub fn timer(duration: Duration) -> Self {
1549        Self::Timer(duration)
1550    }
1551
1552    pub fn signal(signal_name: impl Into<String>) -> Self {
1553        Self::Signal(signal_name.into())
1554    }
1555
1556    pub fn condition<F>(options: ConditionWaitOptions, predicate: F) -> Self
1557    where
1558        F: Fn() -> Result<bool> + Send + 'static,
1559    {
1560        Self::Condition {
1561            options,
1562            predicate: Box::new(predicate),
1563        }
1564    }
1565
1566    pub fn group(operations: Vec<ParallelOperation>) -> Self {
1567        Self::Group(operations)
1568    }
1569}
1570
1571#[derive(Clone, Debug)]
1572struct ValidatedActivityOptions {
1573    task_queue: Option<String>,
1574    retry_policy: Option<Value>,
1575    start_to_close_timeout: Option<u64>,
1576    schedule_to_start_timeout: Option<u64>,
1577    schedule_to_close_timeout: Option<u64>,
1578    heartbeat_timeout: Option<u64>,
1579}
1580
1581fn validate_timeout_order(
1582    smaller_name: &'static str,
1583    smaller: Option<Duration>,
1584    larger_name: &'static str,
1585    larger: Option<Duration>,
1586) -> std::result::Result<(), ActivityOptionsError> {
1587    if matches!((smaller, larger), (Some(smaller), Some(larger)) if smaller > larger) {
1588        return Err(ActivityOptionsError::new(
1589            ActivityOptionsErrorKind::TimeoutOrder,
1590            Some(smaller_name),
1591            format!("{smaller_name} must be <= {larger_name}"),
1592        ));
1593    }
1594    Ok(())
1595}
1596
1597fn timeout_seconds(
1598    field: &'static str,
1599    value: Option<Duration>,
1600) -> std::result::Result<Option<u64>, ActivityOptionsError> {
1601    value
1602        .map(|value| {
1603            activity_protocol_seconds(value).ok_or_else(|| {
1604                ActivityOptionsError::new(
1605                    ActivityOptionsErrorKind::TimeoutOverflow,
1606                    Some(field),
1607                    format!("{field} is too large for the worker protocol"),
1608                )
1609            })
1610        })
1611        .transpose()
1612}
1613
1614fn duration_seconds_ceil(value: Duration) -> Option<u64> {
1615    value
1616        .as_secs()
1617        .checked_add(u64::from(value.subsec_nanos() > 0))
1618}
1619
1620fn activity_protocol_seconds(value: Duration) -> Option<u64> {
1621    duration_seconds_ceil(value).filter(|seconds| *seconds <= i64::MAX as u64)
1622}
1623
1624fn validate_activity_retry_policy(
1625    policy: &ActivityRetryPolicy,
1626) -> std::result::Result<Value, ActivityOptionsError> {
1627    if policy.max_attempts.is_none()
1628        && policy.backoff.is_none()
1629        && policy.non_retryable_error_types.is_empty()
1630    {
1631        return Err(ActivityOptionsError::new(
1632            ActivityOptionsErrorKind::EmptyRetryPolicy,
1633            Some("retry_policy"),
1634            "retry_policy must configure at least one field",
1635        ));
1636    }
1637    if policy.max_attempts == Some(0) {
1638        return Err(ActivityOptionsError::new(
1639            ActivityOptionsErrorKind::InvalidMaxAttempts,
1640            Some("retry_policy.max_attempts"),
1641            "max_attempts must be >= 1",
1642        ));
1643    }
1644    if policy
1645        .non_retryable_error_types
1646        .iter()
1647        .any(|error_type| error_type.trim().is_empty())
1648    {
1649        return Err(ActivityOptionsError::new(
1650            ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
1651            Some("retry_policy.non_retryable_error_types"),
1652            "non_retryable_error_types must not contain empty values",
1653        ));
1654    }
1655
1656    let backoff_seconds = match &policy.backoff {
1657        None => None,
1658        Some(backoff) => {
1659            let max_attempts = policy.max_attempts.ok_or_else(|| {
1660                ActivityOptionsError::new(
1661                    ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
1662                    Some("retry_policy.backoff"),
1663                    "backoff requires max_attempts",
1664                )
1665            })?;
1666            let retry_count = max_attempts.saturating_sub(1) as usize;
1667            let intervals = match backoff {
1668                ActivityBackoff::Explicit(intervals) => {
1669                    if intervals.len() > retry_count {
1670                        return Err(ActivityOptionsError::new(
1671                            ActivityOptionsErrorKind::TooManyBackoffIntervals,
1672                            Some("retry_policy.backoff"),
1673                            "backoff interval count must not exceed max_attempts - 1",
1674                        ));
1675                    }
1676                    intervals.clone()
1677                }
1678                ActivityBackoff::Exponential {
1679                    initial_interval,
1680                    coefficient,
1681                    maximum_interval,
1682                } => {
1683                    if *coefficient < 1 {
1684                        return Err(ActivityOptionsError::new(
1685                            ActivityOptionsErrorKind::InvalidBackoffCoefficient,
1686                            Some("retry_policy.backoff.coefficient"),
1687                            "backoff coefficient must be >= 1",
1688                        ));
1689                    }
1690                    if retry_count > 10_000 {
1691                        return Err(ActivityOptionsError::new(
1692                            ActivityOptionsErrorKind::BackoffGenerationTooLarge,
1693                            Some("retry_policy.max_attempts"),
1694                            "generated backoff supports at most 10000 retry intervals",
1695                        ));
1696                    }
1697                    let mut current = *initial_interval;
1698                    let mut intervals = Vec::with_capacity(retry_count);
1699                    for _ in 0..retry_count {
1700                        let interval = maximum_interval
1701                            .map(|maximum| current.min(maximum))
1702                            .unwrap_or(current);
1703                        intervals.push(interval);
1704                        if maximum_interval.is_some_and(|maximum| interval == maximum) {
1705                            break;
1706                        }
1707                        current = current.checked_mul(*coefficient).ok_or_else(|| {
1708                            ActivityOptionsError::new(
1709                                ActivityOptionsErrorKind::BackoffOverflow,
1710                                Some("retry_policy.backoff"),
1711                                "generated backoff interval overflowed",
1712                            )
1713                        })?;
1714                    }
1715                    intervals
1716                }
1717            };
1718            Some(
1719                intervals
1720                    .into_iter()
1721                    .map(|interval| {
1722                        activity_protocol_seconds(interval).ok_or_else(|| {
1723                            ActivityOptionsError::new(
1724                                ActivityOptionsErrorKind::BackoffOverflow,
1725                                Some("retry_policy.backoff"),
1726                                "backoff interval is too large for the worker protocol",
1727                            )
1728                        })
1729                    })
1730                    .collect::<std::result::Result<Vec<_>, _>>()?,
1731            )
1732        }
1733    };
1734
1735    let mut encoded = serde_json::Map::new();
1736    if let Some(max_attempts) = policy.max_attempts {
1737        encoded.insert("max_attempts".to_string(), json!(max_attempts));
1738    }
1739    if let Some(backoff_seconds) = backoff_seconds {
1740        encoded.insert("backoff_seconds".to_string(), json!(backoff_seconds));
1741    }
1742    if !policy.non_retryable_error_types.is_empty() {
1743        let mut canonical_error_types = Vec::new();
1744        for error_type in policy
1745            .non_retryable_error_types
1746            .iter()
1747            .map(|error_type| error_type.trim())
1748        {
1749            if !canonical_error_types.contains(&error_type) {
1750                canonical_error_types.push(error_type);
1751            }
1752        }
1753        encoded.insert(
1754            "non_retryable_error_types".to_string(),
1755            json!(canonical_error_types),
1756        );
1757    }
1758    Ok(Value::Object(encoded))
1759}
1760
1761/// A stable, machine-readable failure raised when workflow code no longer
1762/// reconstructs the durable command stream recorded in history.
1763#[derive(Clone, Debug, Error)]
1764#[error("non-deterministic workflow replay ({reason}) at sequence {sequence:?}: {message}")]
1765pub struct ReplayFailure {
1766    pub reason: String,
1767    pub sequence: Option<u64>,
1768    pub expected: Option<String>,
1769    pub actual: Option<String>,
1770    pub message: String,
1771}
1772
1773impl ReplayFailure {
1774    fn new(
1775        reason: impl Into<String>,
1776        sequence: Option<u64>,
1777        expected: Option<String>,
1778        actual: Option<String>,
1779        message: impl Into<String>,
1780    ) -> Self {
1781        Self {
1782            reason: reason.into(),
1783            sequence,
1784            expected,
1785            actual,
1786            message: message.into(),
1787        }
1788    }
1789}
1790
1791/// A stable, machine-readable workflow query or query-task settlement failure.
1792#[derive(Clone, Debug, Error)]
1793#[error("query failed ({reason}, HTTP {status}): {message}")]
1794pub struct QueryFailure {
1795    pub status: u16,
1796    pub reason: String,
1797    pub message: String,
1798    pub body: Value,
1799}
1800
1801/// A stable failure returned when a server rejects an SDK protocol version.
1802#[derive(Clone, Debug, Error)]
1803#[error("protocol rejected ({reason}, HTTP {status}): {message}")]
1804pub struct ProtocolFailure {
1805    pub status: u16,
1806    pub reason: String,
1807    pub message: String,
1808    pub supported_version: Option<String>,
1809    pub requested_version: Option<String>,
1810    pub body: Value,
1811}
1812
1813#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
1814pub struct PayloadEnvelope {
1815    pub codec: String,
1816    pub blob: String,
1817}
1818
1819impl PayloadEnvelope {
1820    pub fn avro<T: Serialize>(value: &T) -> Result<Self> {
1821        encode_payload(value, DEFAULT_CODEC)
1822    }
1823
1824    /// Encode an explicit typed value, including the bytes branch that JSON
1825    /// serialization cannot represent.
1826    pub fn avro_value(value: &AvroValue) -> Result<Self> {
1827        encode_avro_value(value)
1828    }
1829}
1830
1831/// Native adapter for the fixed language-neutral Avro Value schema.
1832#[derive(Clone, Debug)]
1833pub enum AvroValue {
1834    Null,
1835    Boolean(bool),
1836    Long(i64),
1837    Double(f64),
1838    Bytes(Vec<u8>),
1839    String(String),
1840    Array(Vec<AvroValue>),
1841    Map(BTreeMap<String, AvroValue>),
1842}
1843
1844impl PartialEq for AvroValue {
1845    fn eq(&self, other: &Self) -> bool {
1846        match (self, other) {
1847            (Self::Null, Self::Null) => true,
1848            (Self::Boolean(left), Self::Boolean(right)) => left == right,
1849            (Self::Long(left), Self::Long(right)) => left == right,
1850            (Self::Double(left), Self::Double(right)) => left.to_bits() == right.to_bits(),
1851            (Self::Bytes(left), Self::Bytes(right)) => left == right,
1852            (Self::String(left), Self::String(right)) => left == right,
1853            (Self::Array(left), Self::Array(right)) => left == right,
1854            (Self::Map(left), Self::Map(right)) => left == right,
1855            _ => false,
1856        }
1857    }
1858}
1859
1860impl AvroValue {
1861    fn from_serialize<T: Serialize>(value: &T) -> Result<Self> {
1862        Self::from_serde_value(
1863            serde_value::to_value(value).map_err(|error| {
1864                Error::Codec(format!("could not adapt value for Avro: {error}"))
1865            })?,
1866        )
1867    }
1868
1869    fn from_serde_value(value: serde_value::Value) -> Result<Self> {
1870        use serde_value::Value as SerdeValue;
1871
1872        match value {
1873            SerdeValue::Unit => Ok(Self::Null),
1874            SerdeValue::Bool(value) => Ok(Self::Boolean(value)),
1875            SerdeValue::I8(value) => Ok(Self::Long(i64::from(value))),
1876            SerdeValue::I16(value) => Ok(Self::Long(i64::from(value))),
1877            SerdeValue::I32(value) => Ok(Self::Long(i64::from(value))),
1878            SerdeValue::I64(value) => Ok(Self::Long(value)),
1879            SerdeValue::U8(value) => Ok(Self::Long(i64::from(value))),
1880            SerdeValue::U16(value) => Ok(Self::Long(i64::from(value))),
1881            SerdeValue::U32(value) => Ok(Self::Long(i64::from(value))),
1882            SerdeValue::U64(value) => i64::try_from(value).map(Self::Long).map_err(|_| {
1883                Error::Codec(
1884                    "integer_overflow: Avro Value long must be within signed 64-bit range"
1885                        .to_string(),
1886                )
1887            }),
1888            SerdeValue::F32(value) => Self::finite_double(f64::from(value)),
1889            SerdeValue::F64(value) => Self::finite_double(value),
1890            SerdeValue::Char(value) => Ok(Self::String(value.to_string())),
1891            SerdeValue::String(value) => Ok(Self::String(value)),
1892            SerdeValue::Bytes(value) => Ok(Self::Bytes(value)),
1893            SerdeValue::Option(None) => Ok(Self::Null),
1894            SerdeValue::Option(Some(value)) | SerdeValue::Newtype(value) => {
1895                Self::from_serde_value(*value)
1896            }
1897            SerdeValue::Seq(values) => values
1898                .into_iter()
1899                .map(Self::from_serde_value)
1900                .collect::<Result<Vec<_>>>()
1901                .map(Self::Array),
1902            SerdeValue::Map(values) => values
1903                .into_iter()
1904                .map(|(key, value)| {
1905                    let SerdeValue::String(key) = key else {
1906                        return Err(Error::Codec(
1907                            "invalid_map_key: Avro Value map keys must be strings".to_string(),
1908                        ));
1909                    };
1910
1911                    Ok((key, Self::from_serde_value(value)?))
1912                })
1913                .collect::<Result<BTreeMap<_, _>>>()
1914                .map(Self::Map),
1915        }
1916    }
1917
1918    fn finite_double(value: f64) -> Result<Self> {
1919        if !value.is_finite() {
1920            return Err(Error::Codec(
1921                "non_finite_float: Avro Value doubles must be finite".to_string(),
1922            ));
1923        }
1924
1925        Ok(Self::Double(value))
1926    }
1927
1928    fn into_json(self) -> Result<Value> {
1929        match self {
1930            Self::Null => Ok(Value::Null),
1931            Self::Boolean(value) => Ok(Value::Bool(value)),
1932            Self::Long(value) => Ok(Value::Number(value.into())),
1933            Self::Double(value) => serde_json::Number::from_f64(value)
1934                .map(Value::Number)
1935                .ok_or_else(|| {
1936                    Error::Codec(
1937                        "non_finite_float: decoded Avro Value double is not finite".to_string(),
1938                    )
1939                }),
1940            Self::Bytes(value) => Ok(json!({
1941                "$type": "bytes",
1942                "base64": BASE64.encode(value),
1943            })),
1944            Self::String(value) => Ok(Value::String(value)),
1945            Self::Array(values) => values
1946                .into_iter()
1947                .map(Self::into_json)
1948                .collect::<Result<Vec<_>>>()
1949                .map(Value::Array),
1950            Self::Map(values) => values
1951                .into_iter()
1952                .map(|(key, value)| Ok((key, value.into_json()?)))
1953                .collect::<Result<serde_json::Map<_, _>>>()
1954                .map(Value::Object),
1955        }
1956    }
1957
1958    fn into_serde_value(self) -> serde_value::Value {
1959        use serde_value::Value as SerdeValue;
1960
1961        match self {
1962            Self::Null => SerdeValue::Unit,
1963            Self::Boolean(value) => SerdeValue::Bool(value),
1964            Self::Long(value) => SerdeValue::I64(value),
1965            Self::Double(value) => SerdeValue::F64(value),
1966            Self::Bytes(value) => SerdeValue::Bytes(value),
1967            Self::String(value) => SerdeValue::String(value),
1968            Self::Array(values) => {
1969                SerdeValue::Seq(values.into_iter().map(Self::into_serde_value).collect())
1970            }
1971            Self::Map(values) => SerdeValue::Map(
1972                values
1973                    .into_iter()
1974                    .map(|(key, value)| (SerdeValue::String(key), value.into_serde_value()))
1975                    .collect(),
1976            ),
1977        }
1978    }
1979
1980    pub fn deserialize<T: DeserializeOwned>(self) -> Result<T> {
1981        self.into_serde_value().deserialize_into().map_err(|error| {
1982            Error::Codec(format!(
1983                "avro_value_type_mismatch: could not adapt decoded value: {error}"
1984            ))
1985        })
1986    }
1987}
1988
1989impl Serialize for AvroValue {
1990    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
1991    where
1992        S: Serializer,
1993    {
1994        match self {
1995            Self::Null => serializer.serialize_unit(),
1996            Self::Boolean(value) => serializer.serialize_bool(*value),
1997            Self::Long(value) => serializer.serialize_i64(*value),
1998            Self::Double(value) => serializer.serialize_f64(*value),
1999            Self::Bytes(value) => serializer.serialize_bytes(value),
2000            Self::String(value) => serializer.serialize_str(value),
2001            Self::Array(values) => {
2002                let mut sequence = serializer.serialize_seq(Some(values.len()))?;
2003                for value in values {
2004                    sequence.serialize_element(value)?;
2005                }
2006                sequence.end()
2007            }
2008            Self::Map(values) => {
2009                let mut map = serializer.serialize_map(Some(values.len()))?;
2010                for (key, value) in values {
2011                    map.serialize_entry(key, value)?;
2012                }
2013                map.end()
2014            }
2015        }
2016    }
2017}
2018
2019pub fn encode_avro_value(value: &AvroValue) -> Result<PayloadEnvelope> {
2020    let datum = avro_value_to_datum(value)?;
2021    let datum = to_avro_datum(avro_value_ordered_map_encoding_schema()?, datum)
2022        .map_err(|err| Error::Codec(format!("avro_value_encode_failed: {err}")))?;
2023    let mut bytes = Vec::with_capacity(datum.len() + 10);
2024    bytes.extend_from_slice(&AVRO_SINGLE_OBJECT_MAGIC);
2025    bytes.extend_from_slice(&AVRO_VALUE_SCHEMA_FINGERPRINT);
2026    bytes.extend_from_slice(&datum);
2027    Ok(PayloadEnvelope {
2028        codec: DEFAULT_CODEC.to_string(),
2029        blob: BASE64.encode(bytes),
2030    })
2031}
2032
2033pub fn decode_avro_value(envelope: &PayloadEnvelope) -> Result<AvroValue> {
2034    if envelope.codec != DEFAULT_CODEC {
2035        return Err(unsupported_payload_codec(&envelope.codec));
2036    }
2037    decode_avro_value_blob(&envelope.blob)
2038}
2039
2040pub fn encode_payload<T: Serialize>(value: &T, codec: &str) -> Result<PayloadEnvelope> {
2041    let blob = match codec {
2042        DEFAULT_CODEC => encode_avro_value(&AvroValue::from_serialize(value)?)?.blob,
2043        other => return Err(unsupported_payload_codec(other)),
2044    };
2045
2046    Ok(PayloadEnvelope {
2047        codec: codec.to_string(),
2048        blob,
2049    })
2050}
2051
2052pub fn decode_payload<T: DeserializeOwned>(envelope: &PayloadEnvelope) -> Result<T> {
2053    match envelope.codec.as_str() {
2054        DEFAULT_CODEC => decode_avro_value(envelope)?.deserialize(),
2055        other => Err(unsupported_payload_codec(other)),
2056    }
2057}
2058
2059fn handler_type_error<T>(
2060    handler_kind: HandlerKind,
2061    handler_name: &str,
2062    value_kind: HandlerValueKind,
2063    message: impl Into<String>,
2064) -> Error {
2065    Error::HandlerType {
2066        handler_kind,
2067        handler_name: handler_name.to_string(),
2068        value_kind,
2069        rust_type: type_name::<T>(),
2070        message: message.into(),
2071    }
2072}
2073
2074fn decode_handler_input<T: DeserializeOwned>(
2075    arguments: AvroValue,
2076    handler_kind: HandlerKind,
2077    handler_name: &str,
2078) -> Result<T> {
2079    let argument = match arguments {
2080        AvroValue::Array(mut arguments) if arguments.len() == 1 => {
2081            arguments.pop().expect("one typed handler argument")
2082        }
2083        AvroValue::Array(arguments) if arguments.is_empty() => AvroValue::Null,
2084        AvroValue::Array(arguments) => {
2085            return Err(handler_type_error::<T>(
2086                handler_kind,
2087                handler_name,
2088                HandlerValueKind::Input,
2089                format!(
2090                    "typed handlers accept one request value, but the task carried {} arguments",
2091                    arguments.len()
2092                ),
2093            ));
2094        }
2095        argument => argument,
2096    };
2097
2098    argument.deserialize().map_err(|error| {
2099        handler_type_error::<T>(
2100            handler_kind,
2101            handler_name,
2102            HandlerValueKind::Input,
2103            error.to_string(),
2104        )
2105    })
2106}
2107
2108fn encode_handler_result<T: Serialize>(
2109    result: &T,
2110    handler_kind: HandlerKind,
2111    handler_name: &str,
2112) -> Result<AvroValue> {
2113    AvroValue::from_serialize(result).map_err(|error| {
2114        handler_type_error::<T>(
2115            handler_kind,
2116            handler_name,
2117            HandlerValueKind::Result,
2118            error.to_string(),
2119        )
2120    })
2121}
2122
2123fn decode_handler_result<T: DeserializeOwned>(
2124    result: AvroValue,
2125    handler_kind: HandlerKind,
2126    handler_name: &str,
2127) -> Result<T> {
2128    result.deserialize().map_err(|error| {
2129        handler_type_error::<T>(
2130            handler_kind,
2131            handler_name,
2132            HandlerValueKind::Result,
2133            error.to_string(),
2134        )
2135    })
2136}
2137
2138#[cfg(test)]
2139fn encode_value_envelope(value: &Value, codec: &str) -> Result<Value> {
2140    Ok(serde_json::to_value(encode_payload(value, codec)?)?)
2141}
2142
2143fn decode_wire_value(value: &Value, fallback_codec: &str) -> Result<Value> {
2144    validate_payload_codec(fallback_codec)?;
2145
2146    if value.is_null() {
2147        return Ok(Value::Null);
2148    }
2149
2150    if let Some((codec, blob)) = payload_envelope_parts(value)? {
2151        return decode_blob(blob, codec);
2152    }
2153
2154    if let Some(blob) = value.as_str() {
2155        return decode_blob(blob, fallback_codec);
2156    }
2157
2158    Err(untagged_payload_value())
2159}
2160
2161fn encode_typed_envelope(value: &AvroValue, codec: &str) -> Result<Value> {
2162    let envelope = match codec {
2163        DEFAULT_CODEC => encode_avro_value(value)?,
2164        other => return Err(unsupported_payload_codec(other)),
2165    };
2166    Ok(serde_json::to_value(envelope)?)
2167}
2168
2169fn decode_wire_avro_value(value: &Value, fallback_codec: &str) -> Result<AvroValue> {
2170    validate_payload_codec(fallback_codec)?;
2171
2172    if value.is_null() {
2173        return Ok(AvroValue::Null);
2174    }
2175
2176    if let Some((codec, blob)) = payload_envelope_parts(value)? {
2177        validate_payload_codec(codec)?;
2178        return decode_avro_value_blob(blob);
2179    }
2180
2181    if let Some(blob) = value.as_str() {
2182        return match fallback_codec {
2183            DEFAULT_CODEC => decode_avro_value_blob(blob),
2184            other => Err(unsupported_payload_codec(other)),
2185        };
2186    }
2187
2188    Err(untagged_payload_value())
2189}
2190
2191fn normalize_avro_arguments(value: AvroValue) -> AvroValue {
2192    match value {
2193        AvroValue::Null => AvroValue::Array(Vec::new()),
2194        AvroValue::Array(_) => value,
2195        other => AvroValue::Array(vec![other]),
2196    }
2197}
2198
2199fn decode_blob(blob: &str, codec: &str) -> Result<Value> {
2200    match codec {
2201        DEFAULT_CODEC => decode_avro_value_blob(blob)?.into_json(),
2202        other => Err(unsupported_payload_codec(other)),
2203    }
2204}
2205
2206fn validate_payload_codec(codec: &str) -> Result<()> {
2207    match codec {
2208        DEFAULT_CODEC => Ok(()),
2209        MISSING_TASK_PAYLOAD_CODEC => {
2210            Err(invalid_task_payload_codec("task payload_codec is missing"))
2211        }
2212        NULL_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec("task payload_codec is null")),
2213        NON_STRING_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec(
2214            "task payload_codec must be a string",
2215        )),
2216        other => Err(unsupported_payload_codec(other)),
2217    }
2218}
2219
2220fn invalid_task_payload_codec(reason: &str) -> Error {
2221    Error::Codec(format!(
2222        "unsupported_payload_codec: {reason}; Durable Workflow 2.0 requires an explicit string payload_codec=\"avro\" before worker task execution"
2223    ))
2224}
2225
2226fn payload_envelope_parts(value: &Value) -> Result<Option<(&str, &str)>> {
2227    let Some(object) = value.as_object() else {
2228        return Ok(None);
2229    };
2230    if !object.contains_key("codec") && !object.contains_key("blob") {
2231        return Ok(None);
2232    }
2233
2234    let codec = object
2235        .get("codec")
2236        .and_then(Value::as_str)
2237        .ok_or_else(invalid_payload_envelope)?;
2238    validate_payload_codec(codec)?;
2239    let blob = object
2240        .get("blob")
2241        .and_then(Value::as_str)
2242        .ok_or_else(invalid_payload_envelope)?;
2243    Ok(Some((codec, blob)))
2244}
2245
2246fn invalid_payload_envelope() -> Error {
2247    Error::Codec(
2248        "invalid_payload_envelope: durable payloads must use an object with string codec=\"avro\" and blob fields"
2249            .to_string(),
2250    )
2251}
2252
2253fn validate_workflow_task_commands(commands: &[Value]) -> Result<()> {
2254    for command in commands {
2255        let Some(command) = command.as_object() else {
2256            continue;
2257        };
2258        let Some(command_type) = command.get("type").and_then(Value::as_str) else {
2259            continue;
2260        };
2261        let Some(payload_field) = workflow_command_payload_field(command_type) else {
2262            continue;
2263        };
2264
2265        if let Some(codec) = command.get("payload_codec") {
2266            let codec = codec.as_str().ok_or_else(invalid_payload_envelope)?;
2267            validate_payload_codec(codec)?;
2268        }
2269
2270        let payload = command
2271            .get(payload_field)
2272            .ok_or_else(invalid_payload_envelope)?;
2273        validate_outbound_payload_envelope(payload)?;
2274    }
2275    Ok(())
2276}
2277
2278fn workflow_completion_protocol_version(commands: &[Value]) -> &'static str {
2279    if commands.iter().any(|command| {
2280        command.get("type").and_then(Value::as_str) == Some("open_condition_wait")
2281            && command
2282                .get("condition_wait_occurrence_id")
2283                .and_then(Value::as_str)
2284                .is_some_and(|occurrence_id| !occurrence_id.is_empty())
2285    }) {
2286        CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
2287    } else if commands.iter().any(|command| {
2288        command.get("type").and_then(Value::as_str) == Some("upsert_search_attributes")
2289            && command.get("attribute_types").is_some()
2290    }) {
2291        TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION
2292    } else if commands
2293        .iter()
2294        .any(|command| command.get("type").and_then(Value::as_str) == Some("upsert_memo"))
2295    {
2296        MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION
2297    } else if commands
2298        .iter()
2299        .any(|command| command.get("type").and_then(Value::as_str) == Some("open_condition_wait"))
2300    {
2301        CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION
2302    } else if commands.iter().any(|command| {
2303        command.get("type").and_then(Value::as_str) == Some("upsert_search_attributes")
2304    }) {
2305        SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION
2306    } else {
2307        WORKER_PROTOCOL_VERSION
2308    }
2309}
2310
2311fn workflow_completion_protocol_version_with_message_streams(
2312    commands: &[Value],
2313    has_message_stream_metadata: bool,
2314) -> &'static str {
2315    let command_protocol = workflow_completion_protocol_version(commands);
2316    if has_message_stream_metadata && !worker_protocol_supports_message_streams(command_protocol) {
2317        MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION
2318    } else {
2319        command_protocol
2320    }
2321}
2322
2323fn workflow_command_payload_field(command_type: &str) -> Option<&'static str> {
2324    match command_type {
2325        "complete_workflow" | "complete_update" | "record_side_effect" => Some("result"),
2326        "schedule_activity" | "start_child_workflow" | "continue_as_new" => Some("arguments"),
2327        "start_service_operation" => Some("request_payload"),
2328        "upsert_memo" => Some("entries"),
2329        _ => None,
2330    }
2331}
2332
2333fn validate_outbound_payload_envelope(value: &Value) -> Result<()> {
2334    let Some((codec, blob)) = payload_envelope_parts(value)? else {
2335        return Err(untagged_payload_value());
2336    };
2337    validate_payload_codec(codec)?;
2338    decode_avro_value_blob(blob)?;
2339    Ok(())
2340}
2341
2342fn unsupported_payload_codec(codec: &str) -> Error {
2343    Error::Codec(format!(
2344        "unsupported_payload_codec: workflow payload codec {codec:?} is not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
2345    ))
2346}
2347
2348fn untagged_payload_value() -> Error {
2349    Error::Codec(
2350        "unsupported_payload_codec: untagged durable payload values are not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
2351            .to_string(),
2352    )
2353}
2354
2355fn decode_avro_value_blob(blob: &str) -> Result<AvroValue> {
2356    let bytes = BASE64.decode(blob).map_err(|err| {
2357        Error::Codec(format!(
2358            "invalid_payload_framing: expected strict base64 Avro single-object bytes: {err}"
2359        ))
2360    })?;
2361
2362    if serde_json::from_slice::<Value>(&bytes).is_ok() {
2363        return Err(unsupported_payload_codec("json"));
2364    }
2365
2366    if bytes.len() < 10 || bytes[..2] != AVRO_SINGLE_OBJECT_MAGIC {
2367        return Err(Error::Codec(
2368            "invalid_payload_framing: expected Avro single-object magic c301".to_string(),
2369        ));
2370    }
2371
2372    let fingerprint: [u8; 8] = bytes[2..10]
2373        .try_into()
2374        .map_err(|_| Error::Codec("invalid Avro fingerprint length".to_string()))?;
2375    if fingerprint != AVRO_VALUE_SCHEMA_FINGERPRINT {
2376        return Err(Error::Codec(format!(
2377            "unsupported_payload_schema: unknown CRC-64-AVRO fingerprint {}",
2378            fingerprint
2379                .iter()
2380                .map(|byte| format!("{byte:02x}"))
2381                .collect::<String>()
2382        )));
2383    }
2384
2385    let mut datum_reader = StrictAvroDatumReader::new(&bytes[10..]);
2386    // The current fingerprint selects the current immutable schema, so reader
2387    // resolution would only re-walk the same recursive union. Future retained
2388    // writer fingerprints supply a distinct reader schema in this branch.
2389    let datum = from_avro_datum(avro_value_schema()?, &mut datum_reader, None);
2390    if datum_reader.truncated {
2391        return Err(Error::Codec(
2392            "invalid_payload_framing: truncated Avro Value datum".to_string(),
2393        ));
2394    }
2395    let datum = datum.map_err(|err| {
2396        Error::Codec(format!(
2397            "invalid_payload_framing: malformed Avro Value datum: {err}"
2398        ))
2399    })?;
2400    if datum_reader.remaining() != 0 {
2401        return Err(Error::Codec(format!(
2402            "invalid_payload_framing: {} trailing bytes after Avro Value datum",
2403            datum_reader.remaining()
2404        )));
2405    }
2406    avro_value_from_datum(datum)
2407}
2408
2409struct StrictAvroDatumReader<'a> {
2410    bytes: &'a [u8],
2411    offset: usize,
2412    truncated: bool,
2413}
2414
2415impl<'a> StrictAvroDatumReader<'a> {
2416    fn new(bytes: &'a [u8]) -> Self {
2417        Self {
2418            bytes,
2419            offset: 0,
2420            truncated: false,
2421        }
2422    }
2423
2424    fn remaining(&self) -> usize {
2425        self.bytes.len() - self.offset
2426    }
2427}
2428
2429impl Read for StrictAvroDatumReader<'_> {
2430    fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
2431        let count = buffer.len().min(self.remaining());
2432        buffer[..count].copy_from_slice(&self.bytes[self.offset..self.offset + count]);
2433        self.offset += count;
2434        if count < buffer.len() {
2435            self.truncated = true;
2436        }
2437
2438        Ok(count)
2439    }
2440}
2441
2442fn avro_value_to_datum(value: &AvroValue) -> Result<AvroDatum> {
2443    let branch = match value {
2444        AvroValue::Null => AvroDatum::Union(0, Box::new(AvroDatum::Null)),
2445        AvroValue::Boolean(value) => AvroDatum::Union(
2446            1,
2447            Box::new(AvroDatum::Record(vec![(
2448                "boolean".to_string(),
2449                AvroDatum::Boolean(*value),
2450            )])),
2451        ),
2452        AvroValue::Long(value) => AvroDatum::Union(
2453            2,
2454            Box::new(AvroDatum::Record(vec![(
2455                "long".to_string(),
2456                AvroDatum::Long(*value),
2457            )])),
2458        ),
2459        AvroValue::Double(value) => {
2460            if !value.is_finite() {
2461                return Err(Error::Codec(
2462                    "non_finite_float: Avro Value doubles must be finite".to_string(),
2463                ));
2464            }
2465            AvroDatum::Union(
2466                3,
2467                Box::new(AvroDatum::Record(vec![(
2468                    "double".to_string(),
2469                    AvroDatum::Double(*value),
2470                )])),
2471            )
2472        }
2473        AvroValue::Bytes(value) => AvroDatum::Union(
2474            4,
2475            Box::new(AvroDatum::Record(vec![(
2476                "bytes".to_string(),
2477                AvroDatum::Bytes(value.clone()),
2478            )])),
2479        ),
2480        AvroValue::String(value) => AvroDatum::Union(
2481            5,
2482            Box::new(AvroDatum::Record(vec![(
2483                "string".to_string(),
2484                AvroDatum::String(value.clone()),
2485            )])),
2486        ),
2487        AvroValue::Array(values) => AvroDatum::Union(
2488            6,
2489            Box::new(AvroDatum::Record(vec![(
2490                "items".to_string(),
2491                AvroDatum::Array(
2492                    values
2493                        .iter()
2494                        .map(avro_value_to_datum)
2495                        .collect::<Result<Vec<_>>>()?,
2496                ),
2497            )])),
2498        ),
2499        AvroValue::Map(values) => AvroDatum::Union(
2500            7,
2501            Box::new(AvroDatum::Record(vec![(
2502                "entries".to_string(),
2503                AvroDatum::Array(
2504                    values
2505                        .iter()
2506                        .map(|(key, value)| {
2507                            Ok(AvroDatum::Record(vec![
2508                                ("key".to_string(), AvroDatum::String(key.clone())),
2509                                ("value".to_string(), avro_value_to_datum(value)?),
2510                            ]))
2511                        })
2512                        .collect::<Result<Vec<_>>>()?,
2513                ),
2514            )])),
2515        ),
2516    };
2517    Ok(AvroDatum::Record(vec![("value".to_string(), branch)]))
2518}
2519
2520fn avro_value_from_datum(datum: AvroDatum) -> Result<AvroValue> {
2521    let AvroDatum::Record(mut outer) = datum else {
2522        return Err(Error::Codec(
2523            "invalid_payload_framing: datum is not a Value record".to_string(),
2524        ));
2525    };
2526    let (_, branch) = outer
2527        .pop()
2528        .filter(|(name, _)| name == "value")
2529        .ok_or_else(|| Error::Codec("invalid_payload_framing: Value field missing".to_string()))?;
2530    let AvroDatum::Union(_, branch) = branch else {
2531        return Err(Error::Codec(
2532            "invalid_payload_framing: invalid Value union".to_string(),
2533        ));
2534    };
2535    match *branch {
2536        AvroDatum::Null => Ok(AvroValue::Null),
2537        AvroDatum::Record(mut fields) => {
2538            let (name, value) = fields.pop().ok_or_else(|| {
2539                Error::Codec("invalid_payload_framing: empty Value branch".to_string())
2540            })?;
2541            match (name.as_str(), value) {
2542                ("boolean", AvroDatum::Boolean(value)) => Ok(AvroValue::Boolean(value)),
2543                ("long", AvroDatum::Long(value)) => Ok(AvroValue::Long(value)),
2544                ("double", AvroDatum::Double(value)) if value.is_finite() => {
2545                    Ok(AvroValue::Double(value))
2546                }
2547                ("bytes", AvroDatum::Bytes(value)) => Ok(AvroValue::Bytes(value)),
2548                ("string", AvroDatum::String(value)) => Ok(AvroValue::String(value)),
2549                ("items", AvroDatum::Array(values)) => values
2550                    .into_iter()
2551                    .map(avro_value_from_datum)
2552                    .collect::<Result<Vec<_>>>()
2553                    .map(AvroValue::Array),
2554                ("entries", AvroDatum::Map(values)) => values
2555                    .into_iter()
2556                    .map(|(key, value)| Ok((key, avro_value_from_datum(value)?)))
2557                    .collect::<Result<BTreeMap<_, _>>>()
2558                    .map(AvroValue::Map),
2559                _ => Err(Error::Codec(
2560                    "invalid_payload_framing: unknown Value branch".to_string(),
2561                )),
2562            }
2563        }
2564        _ => Err(Error::Codec(
2565            "invalid_payload_framing: invalid Value branch".to_string(),
2566        )),
2567    }
2568}
2569
2570fn avro_value_schema() -> Result<&'static Schema> {
2571    match AVRO_VALUE_SCHEMA.get_or_init(|| {
2572        Schema::parse_str(AVRO_VALUE_SCHEMA_JSON)
2573            .map_err(|err| format!("could not parse Avro Value schema: {err}"))
2574    }) {
2575        Ok(schema) => Ok(schema),
2576        Err(message) => Err(Error::Codec(message.clone())),
2577    }
2578}
2579
2580fn avro_value_ordered_map_encoding_schema() -> Result<&'static Schema> {
2581    match AVRO_VALUE_ORDERED_MAP_ENCODING_SCHEMA.get_or_init(|| {
2582        // Apache Avro's Value::Map uses a randomized HashMap. Arrays and maps
2583        // have the same block representation when each ordered array record
2584        // contains the map key followed by its value, so this encoding-only
2585        // adaptation lets the official encoder retain BTreeMap wire order.
2586        let mut schema: Value = serde_json::from_str(AVRO_VALUE_SCHEMA_JSON)
2587            .map_err(|err| format!("could not read packaged Avro Value schema: {err}"))?;
2588        let entries_schema = schema
2589            .pointer_mut("/fields/0/type/7/fields/0/type")
2590            .ok_or_else(|| "packaged Avro Value map schema is missing".to_string())?;
2591        if *entries_schema != json!({"type": "map", "values": "Value"}) {
2592            return Err("packaged Avro Value map schema changed unexpectedly".to_string());
2593        }
2594        *entries_schema = json!({
2595            "type": "array",
2596            "items": {
2597                "type": "record",
2598                "name": "MapEntry",
2599                "fields": [
2600                    {"name": "key", "type": "string"},
2601                    {"name": "value", "type": "Value"}
2602                ]
2603            }
2604        });
2605        Schema::parse_str(&schema.to_string())
2606            .map_err(|err| format!("could not parse ordered-map Avro Value schema: {err}"))
2607    }) {
2608        Ok(schema) => Ok(schema),
2609        Err(message) => Err(Error::Codec(message.clone())),
2610    }
2611}
2612
2613#[derive(Clone, Debug)]
2614pub struct Client {
2615    http: reqwest::Client,
2616    base_url: String,
2617    token: Option<String>,
2618    control_token: Option<String>,
2619    worker_token: Option<String>,
2620    namespace: String,
2621}
2622
2623impl Client {
2624    pub fn new(base_url: impl Into<String>) -> Result<Self> {
2625        Self::builder(base_url).build()
2626    }
2627
2628    pub fn builder(base_url: impl Into<String>) -> ClientBuilder {
2629        ClientBuilder {
2630            base_url: base_url.into(),
2631            token: None,
2632            control_token: None,
2633            worker_token: None,
2634            namespace: "default".to_string(),
2635            timeout: Duration::from_secs(60),
2636        }
2637    }
2638
2639    pub async fn health(&self) -> Result<Value> {
2640        self.request_json(
2641            reqwest::Method::GET,
2642            "/health",
2643            RequestProtocol::ControlPlane,
2644            Option::<&Value>::None,
2645        )
2646        .await
2647    }
2648
2649    pub async fn cluster_info(&self) -> Result<Value> {
2650        self.request_json(
2651            reqwest::Method::GET,
2652            "/cluster/info",
2653            RequestProtocol::ControlPlane,
2654            Option::<&Value>::None,
2655        )
2656        .await
2657    }
2658
2659    pub async fn start_workflow<T: Serialize>(
2660        &self,
2661        workflow_type: &str,
2662        task_queue: &str,
2663        workflow_id: &str,
2664        input: T,
2665    ) -> Result<WorkflowHandle> {
2666        self.start_workflow_with_options(
2667            workflow_type,
2668            task_queue,
2669            workflow_id,
2670            WorkflowStartOptions::default(),
2671            input,
2672        )
2673        .await
2674    }
2675
2676    /// Start a workflow with explicit server-enforced execution and run
2677    /// deadlines.
2678    pub async fn start_workflow_with_options<T: Serialize>(
2679        &self,
2680        workflow_type: &str,
2681        task_queue: &str,
2682        workflow_id: &str,
2683        options: WorkflowStartOptions,
2684        input: T,
2685    ) -> Result<WorkflowHandle> {
2686        options.validate()?;
2687        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2688        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2689        let body = json!({
2690            "workflow_id": workflow_id,
2691            "workflow_type": workflow_type,
2692            "task_queue": task_queue,
2693            "input": input_envelope,
2694            "execution_timeout_seconds": options.execution_timeout_seconds,
2695            "run_timeout_seconds": options.run_timeout_seconds
2696        });
2697
2698        let data: Value = self
2699            .request_json(
2700                reqwest::Method::POST,
2701                "/workflows",
2702                RequestProtocol::ControlPlane,
2703                Some(&body),
2704            )
2705            .await?;
2706
2707        Ok(WorkflowHandle {
2708            client: self.clone(),
2709            workflow_id: data
2710                .get("workflow_id")
2711                .and_then(Value::as_str)
2712                .unwrap_or(workflow_id)
2713                .to_string(),
2714            run_id: data
2715                .get("run_id")
2716                .and_then(Value::as_str)
2717                .map(str::to_string),
2718            workflow_type: data
2719                .get("workflow_type")
2720                .and_then(Value::as_str)
2721                .unwrap_or(workflow_type)
2722                .to_string(),
2723        })
2724    }
2725
2726    pub async fn signal_workflow<T: Serialize>(
2727        &self,
2728        workflow_id: &str,
2729        signal_name: &str,
2730        input: T,
2731    ) -> Result<Value> {
2732        self.signal_workflow_target(workflow_id, None, signal_name, input)
2733            .await
2734    }
2735
2736    /// Append one idempotently identified item to an instance-scoped input stream.
2737    pub async fn append_message_stream<T: Serialize>(
2738        &self,
2739        workflow_id: &str,
2740        stream_name: &str,
2741        message_id: &str,
2742        input: T,
2743    ) -> Result<Value> {
2744        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2745        let body = json!({
2746            "message_id": message_id,
2747            "input": encode_typed_envelope(&input, DEFAULT_CODEC)?
2748        });
2749        self.request_json(
2750            reqwest::Method::POST,
2751            &format!("/workflows/{workflow_id}/message-streams/{stream_name}/messages"),
2752            RequestProtocol::ControlPlane,
2753            Some(&body),
2754        )
2755        .await
2756    }
2757
2758    /// Signal only if `run_id` is still the current run for this instance.
2759    pub async fn signal_workflow_run<T: Serialize>(
2760        &self,
2761        workflow_id: &str,
2762        run_id: &str,
2763        signal_name: &str,
2764        input: T,
2765    ) -> Result<Value> {
2766        self.signal_workflow_target(workflow_id, Some(run_id), signal_name, input)
2767            .await
2768    }
2769
2770    async fn signal_workflow_target<T: Serialize>(
2771        &self,
2772        workflow_id: &str,
2773        run_id: Option<&str>,
2774        signal_name: &str,
2775        input: T,
2776    ) -> Result<Value> {
2777        validate_user_signal_name(signal_name)?;
2778        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2779        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2780        let body = json!({
2781            "input": input_envelope
2782        });
2783        let path = match run_id {
2784            Some(run_id) => {
2785                format!("/workflows/{workflow_id}/runs/{run_id}/signal/{signal_name}")
2786            }
2787            None => format!("/workflows/{workflow_id}/signal/{signal_name}"),
2788        };
2789        self.request_json(
2790            reqwest::Method::POST,
2791            &path,
2792            RequestProtocol::ControlPlane,
2793            Some(&body),
2794        )
2795        .await
2796    }
2797
2798    /// Request cooperative cancellation of the current run for an instance.
2799    pub async fn cancel_workflow(
2800        &self,
2801        workflow_id: &str,
2802        options: WorkflowCommandOptions,
2803    ) -> Result<WorkflowCommandResult> {
2804        self.workflow_command(workflow_id, None, WorkflowCommandKind::Cancel, options)
2805            .await
2806    }
2807
2808    /// Request cooperative cancellation only if `run_id` is still current.
2809    pub async fn cancel_workflow_run(
2810        &self,
2811        workflow_id: &str,
2812        run_id: &str,
2813        options: WorkflowCommandOptions,
2814    ) -> Result<WorkflowCommandResult> {
2815        self.workflow_command(
2816            workflow_id,
2817            Some(run_id),
2818            WorkflowCommandKind::Cancel,
2819            options,
2820        )
2821        .await
2822    }
2823
2824    /// Forcefully terminate the current run for an instance.
2825    pub async fn terminate_workflow(
2826        &self,
2827        workflow_id: &str,
2828        options: WorkflowCommandOptions,
2829    ) -> Result<WorkflowCommandResult> {
2830        self.workflow_command(workflow_id, None, WorkflowCommandKind::Terminate, options)
2831            .await
2832    }
2833
2834    /// Forcefully terminate only if `run_id` is still current.
2835    pub async fn terminate_workflow_run(
2836        &self,
2837        workflow_id: &str,
2838        run_id: &str,
2839        options: WorkflowCommandOptions,
2840    ) -> Result<WorkflowCommandResult> {
2841        self.workflow_command(
2842            workflow_id,
2843            Some(run_id),
2844            WorkflowCommandKind::Terminate,
2845            options,
2846        )
2847        .await
2848    }
2849
2850    async fn workflow_command(
2851        &self,
2852        workflow_id: &str,
2853        run_id: Option<&str>,
2854        command: WorkflowCommandKind,
2855        options: WorkflowCommandOptions,
2856    ) -> Result<WorkflowCommandResult> {
2857        let path = match run_id {
2858            Some(run_id) => format!(
2859                "/workflows/{workflow_id}/runs/{run_id}/{}",
2860                command.as_str()
2861            ),
2862            None => format!("/workflows/{workflow_id}/{}", command.as_str()),
2863        };
2864        let data = match self
2865            .request_json(
2866                reqwest::Method::POST,
2867                &path,
2868                RequestProtocol::ControlPlane,
2869                Some(&options),
2870            )
2871            .await
2872        {
2873            Ok(data) => data,
2874            Err(Error::Http { status, body }) => {
2875                return Err(Error::WorkflowCommandRejected(workflow_command_rejection(
2876                    command,
2877                    status,
2878                    body,
2879                    workflow_id,
2880                    run_id,
2881                )));
2882            }
2883            Err(error) => return Err(error),
2884        };
2885
2886        Ok(workflow_command_result(command, data, workflow_id, run_id))
2887    }
2888
2889    /// Execute a named, read-only query against a running or completed workflow.
2890    ///
2891    /// Arguments and results use the platform payload envelope. Server and
2892    /// worker rejections are returned as [`Error::QueryFailed`] with a stable
2893    /// reason, HTTP status, and original response body.
2894    pub async fn query_workflow<T: Serialize>(
2895        &self,
2896        workflow_id: &str,
2897        query_name: &str,
2898        input: T,
2899    ) -> Result<Value> {
2900        self.query_workflow_target(workflow_id, None, query_name, input)
2901            .await
2902    }
2903
2904    /// Query only if `run_id` is still current, preventing accidental retargeting.
2905    pub async fn query_workflow_run<T: Serialize>(
2906        &self,
2907        workflow_id: &str,
2908        run_id: &str,
2909        query_name: &str,
2910        input: T,
2911    ) -> Result<Value> {
2912        self.query_workflow_target(workflow_id, Some(run_id), query_name, input)
2913            .await
2914    }
2915
2916    /// Query a workflow and return the lossless fixed Avro Value result.
2917    pub async fn query_workflow_avro_value<T: Serialize>(
2918        &self,
2919        workflow_id: &str,
2920        query_name: &str,
2921        input: T,
2922    ) -> Result<AvroValue> {
2923        self.query_workflow_avro_value_target(workflow_id, None, query_name, input)
2924            .await
2925    }
2926
2927    /// Query a selected run and return the lossless fixed Avro Value result.
2928    pub async fn query_workflow_run_avro_value<T: Serialize>(
2929        &self,
2930        workflow_id: &str,
2931        run_id: &str,
2932        query_name: &str,
2933        input: T,
2934    ) -> Result<AvroValue> {
2935        self.query_workflow_avro_value_target(workflow_id, Some(run_id), query_name, input)
2936            .await
2937    }
2938
2939    async fn query_workflow_avro_value_target<T: Serialize>(
2940        &self,
2941        workflow_id: &str,
2942        run_id: Option<&str>,
2943        query_name: &str,
2944        input: T,
2945    ) -> Result<AvroValue> {
2946        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2947        let body = json!({"input": encode_typed_envelope(&input, DEFAULT_CODEC)?});
2948        let path = match run_id {
2949            Some(run_id) => {
2950                format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
2951            }
2952            None => format!("/workflows/{workflow_id}/query/{query_name}"),
2953        };
2954        let response: Value = match self
2955            .request_json(
2956                reqwest::Method::POST,
2957                &path,
2958                RequestProtocol::ControlPlane,
2959                Some(&body),
2960            )
2961            .await
2962        {
2963            Ok(response) => response,
2964            Err(Error::Http { status, body }) => {
2965                return Err(Error::QueryFailed(query_failure(status, body)));
2966            }
2967            Err(error) => return Err(error),
2968        };
2969
2970        let envelope = response
2971            .get("result_envelope")
2972            .filter(|envelope| !envelope.is_null())
2973            .ok_or_else(|| {
2974                Error::Codec(
2975                    "missing_payload_envelope: typed query result requires result_envelope"
2976                        .to_string(),
2977                )
2978            })?;
2979        decode_wire_avro_value(envelope, DEFAULT_CODEC)
2980    }
2981
2982    async fn query_workflow_target<T: Serialize>(
2983        &self,
2984        workflow_id: &str,
2985        run_id: Option<&str>,
2986        query_name: &str,
2987        input: T,
2988    ) -> Result<Value> {
2989        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2990        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2991        let body = json!({
2992            "input": input_envelope
2993        });
2994        let path = match run_id {
2995            Some(run_id) => {
2996                format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
2997            }
2998            None => format!("/workflows/{workflow_id}/query/{query_name}"),
2999        };
3000        let response: Value = match self
3001            .request_json(
3002                reqwest::Method::POST,
3003                &path,
3004                RequestProtocol::ControlPlane,
3005                Some(&body),
3006            )
3007            .await
3008        {
3009            Ok(response) => response,
3010            Err(Error::Http { status, body }) => {
3011                return Err(Error::QueryFailed(query_failure(status, body)));
3012            }
3013            Err(error) => return Err(error),
3014        };
3015
3016        if let Some(envelope) = response
3017            .get("result_envelope")
3018            .filter(|envelope| !envelope.is_null())
3019        {
3020            return decode_wire_value(envelope, DEFAULT_CODEC);
3021        }
3022
3023        Ok(response.get("result").cloned().unwrap_or(Value::Null))
3024    }
3025
3026    /// Send a synchronous update using fixed Avro Value arguments.
3027    pub async fn update_workflow<T: Serialize>(
3028        &self,
3029        workflow_id: &str,
3030        update_name: &str,
3031        input: T,
3032        request_id: Option<&str>,
3033    ) -> Result<Value> {
3034        let response = self
3035            .update_workflow_response(workflow_id, update_name, input, request_id)
3036            .await?;
3037        if let Some(envelope) = response
3038            .get("result_envelope")
3039            .filter(|envelope| !envelope.is_null())
3040        {
3041            return decode_wire_value(envelope, DEFAULT_CODEC);
3042        }
3043        Ok(response.get("result").cloned().unwrap_or(response))
3044    }
3045
3046    /// Send a synchronous update and retain a bytes-capable Avro result.
3047    pub async fn update_workflow_avro_value<T: Serialize>(
3048        &self,
3049        workflow_id: &str,
3050        update_name: &str,
3051        input: T,
3052        request_id: Option<&str>,
3053    ) -> Result<AvroValue> {
3054        let response = self
3055            .update_workflow_response(workflow_id, update_name, input, request_id)
3056            .await?;
3057        let envelope = response
3058            .get("result_envelope")
3059            .filter(|envelope| !envelope.is_null())
3060            .ok_or_else(|| {
3061                Error::Codec(
3062                    "missing_payload_envelope: typed update result requires result_envelope"
3063                        .to_string(),
3064                )
3065            })?;
3066        decode_wire_avro_value(envelope, DEFAULT_CODEC)
3067    }
3068
3069    async fn update_workflow_response<T: Serialize>(
3070        &self,
3071        workflow_id: &str,
3072        update_name: &str,
3073        input: T,
3074        request_id: Option<&str>,
3075    ) -> Result<Value> {
3076        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
3077        let mut body = json!({
3078            "input": encode_typed_envelope(&input, DEFAULT_CODEC)?,
3079            "wait_for": "completed",
3080        });
3081        if let Some(request_id) = request_id {
3082            body["request_id"] = json!(request_id);
3083        }
3084        self.request_json(
3085            reqwest::Method::POST,
3086            &format!("/workflows/{workflow_id}/update/{update_name}"),
3087            RequestProtocol::ControlPlane,
3088            Some(&body),
3089        )
3090        .await
3091    }
3092
3093    pub async fn describe_workflow(&self, workflow_id: &str) -> Result<WorkflowDescription> {
3094        let path = format!("/workflows/{workflow_id}");
3095        let mut data: WorkflowDescription = self
3096            .request_json(
3097                reqwest::Method::GET,
3098                &path,
3099                RequestProtocol::ControlPlane,
3100                Option::<&Value>::None,
3101            )
3102            .await?;
3103        data.decode_payloads()?;
3104        Ok(data)
3105    }
3106
3107    /// Describe one selected run, including historical terminal runs.
3108    pub async fn describe_workflow_run(
3109        &self,
3110        workflow_id: &str,
3111        run_id: &str,
3112    ) -> Result<WorkflowDescription> {
3113        let path = format!("/workflows/{workflow_id}/runs/{run_id}");
3114        let mut data: WorkflowDescription = self
3115            .request_json(
3116                reqwest::Method::GET,
3117                &path,
3118                RequestProtocol::ControlPlane,
3119                Option::<&Value>::None,
3120            )
3121            .await?;
3122        data.decode_payloads()?;
3123        Ok(data)
3124    }
3125
3126    fn workflow_stream_path(workflow_id: &str, run_id: &str, stream_name: Option<&str>) -> String {
3127        let mut path = format!(
3128            "/workflows/{}/runs/{}/streams",
3129            percent_encode_path_segment(workflow_id),
3130            percent_encode_path_segment(run_id),
3131        );
3132        if let Some(stream_name) = stream_name {
3133            path.push('/');
3134            path.push_str(&percent_encode_path_segment(stream_name));
3135        }
3136        path
3137    }
3138
3139    /// List the run-scoped output streams already opened by a workflow.
3140    pub async fn list_workflow_streams(
3141        &self,
3142        workflow_id: &str,
3143        run_id: &str,
3144    ) -> Result<Vec<WorkflowStreamDescription>> {
3145        let response: WorkflowStreamListResponse = self
3146            .request_json(
3147                reqwest::Method::GET,
3148                &Self::workflow_stream_path(workflow_id, run_id, None),
3149                RequestProtocol::ControlPlane,
3150                Option::<&Value>::None,
3151            )
3152            .await?;
3153        Ok(response.streams)
3154    }
3155
3156    /// Describe stream lifecycle, offsets, pending count, and terminal error.
3157    pub async fn describe_workflow_stream(
3158        &self,
3159        workflow_id: &str,
3160        run_id: &str,
3161        stream_name: &str,
3162    ) -> Result<WorkflowStreamDescription> {
3163        let response: WorkflowStreamDescriptionResponse = self
3164            .request_json(
3165                reqwest::Method::GET,
3166                &Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3167                RequestProtocol::ControlPlane,
3168                Option::<&Value>::None,
3169            )
3170            .await?;
3171        Ok(response.stream)
3172    }
3173
3174    /// Read one bounded page beginning at a zero-based offset.
3175    ///
3176    /// Delivery is at least once: persist `next_offset` only after processing
3177    /// the page. The future is cancellation-safe; dropping it cancels the
3178    /// in-flight request. Long polling is capped at 60 seconds by the SDK and
3179    /// service contract.
3180    pub async fn subscribe_workflow_stream(
3181        &self,
3182        workflow_id: &str,
3183        run_id: &str,
3184        stream_name: &str,
3185        from_offset: u64,
3186        max_items: usize,
3187        wait: Duration,
3188    ) -> Result<WorkflowStreamPage> {
3189        let max_items = max_items.clamp(1, 500);
3190        let wait_seconds = wait.as_secs().min(MAX_LONG_POLL_TIMEOUT_SECONDS);
3191        let path = format!(
3192            "{}/items?from={from_offset}&max_items={max_items}&wait_seconds={wait_seconds}",
3193            Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3194        );
3195        let response: WorkflowStreamPageResponse = self
3196            .request_json_with_timeout(
3197                reqwest::Method::GET,
3198                &path,
3199                RequestProtocol::ControlPlane,
3200                Option::<&Value>::None,
3201                Duration::from_secs(wait_seconds.saturating_add(5).max(5)),
3202            )
3203            .await?;
3204
3205        let items = response
3206            .items
3207            .into_iter()
3208            .map(|raw| {
3209                let offset = raw.get("offset").and_then(Value::as_u64).unwrap_or(0);
3210                let envelope = raw.get("payload").cloned();
3211                let payload = envelope
3212                    .as_ref()
3213                    .filter(|value| value.get("blob").is_some())
3214                    .map(|value| decode_wire_avro_value(value, DEFAULT_CODEC))
3215                    .transpose()?
3216                    .map(AvroValue::into_json)
3217                    .transpose()?;
3218                Ok(WorkflowStreamItem {
3219                    offset,
3220                    payload,
3221                    payload_envelope: envelope,
3222                    payload_reference: raw
3223                        .get("payload_reference")
3224                        .and_then(Value::as_str)
3225                        .map(str::to_string),
3226                    payload_codec: raw
3227                        .get("payload_codec")
3228                        .and_then(Value::as_str)
3229                        .map(str::to_string),
3230                    idempotency_key: raw
3231                        .get("idempotency_key")
3232                        .and_then(Value::as_str)
3233                        .map(str::to_string),
3234                    item_type: raw
3235                        .get("item_type")
3236                        .and_then(Value::as_str)
3237                        .map(str::to_string),
3238                    content_type: raw
3239                        .get("content_type")
3240                        .and_then(Value::as_str)
3241                        .map(str::to_string),
3242                    origin: raw
3243                        .get("origin")
3244                        .and_then(Value::as_str)
3245                        .map(str::to_string),
3246                    origin_reference: raw
3247                        .get("origin_reference")
3248                        .and_then(Value::as_str)
3249                        .map(str::to_string),
3250                    emitted_at: raw
3251                        .get("emitted_at")
3252                        .and_then(Value::as_str)
3253                        .map(str::to_string),
3254                    raw,
3255                })
3256            })
3257            .collect::<Result<Vec<_>>>()?;
3258        Ok(WorkflowStreamPage {
3259            stream: response.stream,
3260            items,
3261            next_offset: response.next_offset,
3262            terminal: response.terminal,
3263        })
3264    }
3265
3266    /// Append inline Avro envelopes or opaque external payload references.
3267    pub async fn append_workflow_stream(
3268        &self,
3269        workflow_id: &str,
3270        run_id: &str,
3271        stream_name: &str,
3272        items: &[WorkflowStreamAppendItem],
3273        max_pending_items: Option<u64>,
3274    ) -> Result<WorkflowStreamAppendResult> {
3275        if items.is_empty() {
3276            return Err(Error::Codec(
3277                "workflow_stream_items_empty: append requires at least one item".to_string(),
3278            ));
3279        }
3280        let mut body = json!({
3281            "items": items
3282                .iter()
3283                .map(|item| item.wire_value(None))
3284                .collect::<Vec<_>>(),
3285        });
3286        if let Some(max_pending_items) = max_pending_items {
3287            if max_pending_items == 0 {
3288                return Err(Error::Codec(
3289                    "workflow_stream_pending_limit_invalid: max_pending_items must be positive"
3290                        .to_string(),
3291                ));
3292            }
3293            body["max_pending_items"] = json!(max_pending_items);
3294        }
3295        let response: WorkflowStreamAppendResponse = self
3296            .request_json(
3297                reqwest::Method::POST,
3298                &format!(
3299                    "{}/items",
3300                    Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3301                ),
3302                RequestProtocol::ControlPlane,
3303                Some(&body),
3304            )
3305            .await?;
3306        Ok(WorkflowStreamAppendResult {
3307            stream: response.stream,
3308            accepted_offsets: response.accepted_offsets,
3309            accepted: response.accepted,
3310            deduped: response.deduped,
3311        })
3312    }
3313
3314    /// Close a stream, or mark it errored when `error_reason` is supplied.
3315    pub async fn close_workflow_stream(
3316        &self,
3317        workflow_id: &str,
3318        run_id: &str,
3319        stream_name: &str,
3320        error_reason: Option<&str>,
3321        retention_seconds: Option<u64>,
3322    ) -> Result<WorkflowStreamDescription> {
3323        let mut body = json!({});
3324        if let Some(error_reason) = error_reason {
3325            body["error_reason"] = json!(error_reason);
3326        }
3327        if let Some(retention_seconds) = retention_seconds {
3328            if retention_seconds == 0 {
3329                return Err(Error::Codec(
3330                    "workflow_stream_retention_invalid: retention_seconds must be positive"
3331                        .to_string(),
3332                ));
3333            }
3334            body["retention_seconds"] = json!(retention_seconds);
3335        }
3336        let response: WorkflowStreamDescriptionResponse = self
3337            .request_json(
3338                reqwest::Method::POST,
3339                &format!(
3340                    "{}/close",
3341                    Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3342                ),
3343                RequestProtocol::ControlPlane,
3344                Some(&body),
3345            )
3346            .await?;
3347        Ok(response.stream)
3348    }
3349
3350    pub async fn register_worker(
3351        &self,
3352        worker_id: &str,
3353        task_queue: &str,
3354        supported_workflow_types: Vec<String>,
3355        supported_activity_types: Vec<String>,
3356        max_concurrent_workflow_tasks: usize,
3357        max_concurrent_activity_tasks: usize,
3358    ) -> Result<RegisterWorkerResponse> {
3359        self.register_worker_with_capabilities(
3360            worker_id,
3361            task_queue,
3362            supported_workflow_types,
3363            supported_activity_types,
3364            max_concurrent_workflow_tasks,
3365            max_concurrent_activity_tasks,
3366            Vec::new(),
3367        )
3368        .await
3369    }
3370
3371    /// Register a worker and explicitly advertise additive worker capabilities.
3372    pub async fn register_worker_with_capabilities(
3373        &self,
3374        worker_id: &str,
3375        task_queue: &str,
3376        supported_workflow_types: Vec<String>,
3377        supported_activity_types: Vec<String>,
3378        max_concurrent_workflow_tasks: usize,
3379        max_concurrent_activity_tasks: usize,
3380        capabilities: Vec<String>,
3381    ) -> Result<RegisterWorkerResponse> {
3382        self.register_worker_with_command_contracts(
3383            worker_id,
3384            task_queue,
3385            supported_workflow_types,
3386            supported_activity_types,
3387            max_concurrent_workflow_tasks,
3388            max_concurrent_activity_tasks,
3389            capabilities,
3390            Value::Object(serde_json::Map::new()),
3391        )
3392        .await
3393    }
3394
3395    /// Register a worker and advertise its named query and update handlers.
3396    ///
3397    /// This Rust SDK cannot execute synchronous pre-accept update validation,
3398    /// so a workflow contract with a non-empty or malformed
3399    /// `update_validators` declaration returns
3400    /// [`Error::UnsupportedUpdateValidators`] before registration transport.
3401    #[allow(clippy::too_many_arguments)]
3402    pub async fn register_worker_with_command_contracts(
3403        &self,
3404        worker_id: &str,
3405        task_queue: &str,
3406        supported_workflow_types: Vec<String>,
3407        supported_activity_types: Vec<String>,
3408        max_concurrent_workflow_tasks: usize,
3409        max_concurrent_activity_tasks: usize,
3410        capabilities: Vec<String>,
3411        workflow_command_contracts: Value,
3412    ) -> Result<RegisterWorkerResponse> {
3413        if let Some(contracts) = workflow_command_contracts.as_object() {
3414            for (workflow_type, contract) in contracts {
3415                let Some(update_validators) = contract.get("update_validators") else {
3416                    continue;
3417                };
3418                if !update_validators
3419                    .as_array()
3420                    .is_some_and(|validators| validators.is_empty())
3421                {
3422                    return Err(Error::UnsupportedUpdateValidators {
3423                        workflow_type: workflow_type.clone(),
3424                    });
3425                }
3426            }
3427        }
3428
3429        let mut body = json!({
3430            "worker_id": worker_id,
3431            "task_queue": task_queue,
3432            "runtime": "rust",
3433            "sdk_version": SDK_VERSION,
3434            "supported_workflow_types": supported_workflow_types,
3435            "supported_activity_types": supported_activity_types,
3436            "capabilities": capabilities,
3437            "capability_manifest": portable_worker_affinity_capability_manifest(),
3438            "max_concurrent_workflow_tasks": max_concurrent_workflow_tasks,
3439            "max_concurrent_activity_tasks": max_concurrent_activity_tasks
3440        });
3441        if workflow_command_contracts
3442            .as_object()
3443            .is_some_and(|contracts| !contracts.is_empty())
3444        {
3445            body["workflow_command_contracts"] = workflow_command_contracts;
3446        }
3447
3448        self.request_json(
3449            reqwest::Method::POST,
3450            "/worker/register",
3451            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3452            Some(&body),
3453        )
3454        .await
3455    }
3456
3457    /// Gracefully remove one worker's registration through the worker plane.
3458    ///
3459    /// This operation is separate from operator-facing worker management. It
3460    /// uses worker-protocol authentication and returns the server's lease
3461    /// recovery result.
3462    pub async fn deregister_worker_registration(
3463        &self,
3464        worker_id: &str,
3465    ) -> Result<WorkerDeregistrationEnvelope> {
3466        let path = format!(
3467            "/worker/registrations/{}",
3468            percent_encode_path_segment(worker_id)
3469        );
3470        self.request_json(
3471            reqwest::Method::DELETE,
3472            &path,
3473            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3474            Option::<&Value>::None,
3475        )
3476        .await
3477    }
3478
3479    /// Long-poll for an ephemeral, read-only workflow query task.
3480    pub async fn poll_query_task(
3481        &self,
3482        worker_id: &str,
3483        task_queue: &str,
3484        timeout: Duration,
3485    ) -> Result<Option<QueryTask>> {
3486        Ok(self
3487            .poll_query_task_response(worker_id, task_queue, timeout)
3488            .await?
3489            .task)
3490    }
3491
3492    /// Poll a query task while preserving server stop and drain metadata.
3493    pub async fn poll_query_task_response(
3494        &self,
3495        worker_id: &str,
3496        task_queue: &str,
3497        timeout: Duration,
3498    ) -> Result<PollQueryTaskResponse> {
3499        let poll_request_id = unique_request_id("rust-query-poll");
3500        self.poll_query_task_response_with_request_id(
3501            worker_id,
3502            task_queue,
3503            timeout,
3504            &poll_request_id,
3505            1,
3506        )
3507        .await
3508    }
3509
3510    async fn poll_query_task_response_with_request_id(
3511        &self,
3512        worker_id: &str,
3513        task_queue: &str,
3514        timeout: Duration,
3515        poll_request_id: &str,
3516        transport_retries: usize,
3517    ) -> Result<PollQueryTaskResponse> {
3518        let timeout_seconds = long_poll_timeout_seconds(timeout);
3519        let body = json!({
3520            "worker_id": worker_id,
3521            "task_queue": task_queue,
3522            "poll_request_id": poll_request_id,
3523            "timeout_seconds": timeout_seconds,
3524        });
3525        self.poll_request_json(
3526            "/worker/query-tasks/poll",
3527            RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3528            &body,
3529            timeout + Duration::from_secs(5),
3530            transport_retries,
3531        )
3532        .await
3533    }
3534
3535    /// Complete a query task without appending workflow history.
3536    pub async fn complete_query_task<T: Serialize>(
3537        &self,
3538        query_task_id: &str,
3539        lease_owner: &str,
3540        query_task_attempt: u64,
3541        result: T,
3542        codec: &str,
3543    ) -> Result<Value> {
3544        let typed_result = AvroValue::from_serialize(&result)?;
3545        let result_envelope = encode_typed_envelope(&typed_result, codec)?;
3546        self.complete_query_task_with_envelope(
3547            query_task_id,
3548            lease_owner,
3549            query_task_attempt,
3550            typed_result.into_json()?,
3551            result_envelope,
3552        )
3553        .await
3554    }
3555
3556    async fn complete_query_task_with_envelope(
3557        &self,
3558        query_task_id: &str,
3559        lease_owner: &str,
3560        query_task_attempt: u64,
3561        result: Value,
3562        result_envelope: Value,
3563    ) -> Result<Value> {
3564        let body = json!({
3565            "lease_owner": lease_owner,
3566            "query_task_attempt": query_task_attempt,
3567            "result": result,
3568            "result_envelope": result_envelope,
3569        });
3570        let path = format!("/worker/query-tasks/{query_task_id}/complete");
3571        let response = self
3572            .request_json(
3573                reqwest::Method::POST,
3574                &path,
3575                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3576                Some(&body),
3577            )
3578            .await;
3579        query_task_response(response)
3580    }
3581
3582    /// Report a stable machine-readable query-task failure.
3583    pub async fn fail_query_task(
3584        &self,
3585        query_task_id: &str,
3586        lease_owner: &str,
3587        query_task_attempt: u64,
3588        message: impl Into<String>,
3589        reason: impl Into<String>,
3590        failure_type: impl Into<String>,
3591    ) -> Result<Value> {
3592        let body = json!({
3593            "lease_owner": lease_owner,
3594            "query_task_attempt": query_task_attempt,
3595            "failure": {
3596                "message": message.into(),
3597                "reason": reason.into(),
3598                "type": failure_type.into(),
3599            }
3600        });
3601        let path = format!("/worker/query-tasks/{query_task_id}/fail");
3602        let response = self
3603            .request_json(
3604                reqwest::Method::POST,
3605                &path,
3606                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3607                Some(&body),
3608            )
3609            .await;
3610        query_task_response(response)
3611    }
3612
3613    pub async fn heartbeat_worker(
3614        &self,
3615        worker_id: &str,
3616        workflow_available: usize,
3617        activity_available: usize,
3618    ) -> Result<Value> {
3619        let body = json!({
3620            "worker_id": worker_id,
3621            "task_slots": {
3622                "workflow_available": workflow_available,
3623                "activity_available": activity_available
3624            },
3625            "process_metrics": {
3626                "process_id": std::process::id(),
3627                "process_uptime_seconds": 0
3628            }
3629        });
3630
3631        self.request_json(
3632            reqwest::Method::POST,
3633            "/worker/heartbeat",
3634            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3635            Some(&body),
3636        )
3637        .await
3638    }
3639
3640    pub async fn poll_workflow_task(
3641        &self,
3642        worker_id: &str,
3643        task_queue: &str,
3644        timeout: Duration,
3645    ) -> Result<Option<WorkflowTask>> {
3646        Ok(self
3647            .poll_workflow_task_response(worker_id, task_queue, timeout)
3648            .await?
3649            .task)
3650    }
3651
3652    pub async fn poll_workflow_task_response(
3653        &self,
3654        worker_id: &str,
3655        task_queue: &str,
3656        timeout: Duration,
3657    ) -> Result<PollWorkflowTaskResponse> {
3658        let poll_request_id = unique_request_id("rust-workflow-poll");
3659        self.poll_workflow_task_response_with_request_id(
3660            worker_id,
3661            task_queue,
3662            timeout,
3663            &poll_request_id,
3664            1,
3665        )
3666        .await
3667    }
3668
3669    async fn poll_workflow_task_response_with_request_id(
3670        &self,
3671        worker_id: &str,
3672        task_queue: &str,
3673        timeout: Duration,
3674        poll_request_id: &str,
3675        transport_retries: usize,
3676    ) -> Result<PollWorkflowTaskResponse> {
3677        let body = json!({
3678            "worker_id": worker_id,
3679            "task_queue": task_queue,
3680            "poll_request_id": poll_request_id,
3681            "timeout_seconds": long_poll_timeout_seconds(timeout),
3682        });
3683        let mut data: PollWorkflowTaskResponse = self
3684            .poll_request_json(
3685                "/worker/workflow-tasks/poll",
3686                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3687                &body,
3688                timeout + Duration::from_secs(5),
3689                transport_retries,
3690            )
3691            .await?;
3692
3693        if let Some(task) = data.task.as_mut() {
3694            self.fetch_remaining_workflow_history(worker_id, task)
3695                .await?;
3696        }
3697
3698        Ok(data)
3699    }
3700
3701    async fn fetch_remaining_workflow_history(
3702        &self,
3703        worker_id: &str,
3704        task: &mut WorkflowTask,
3705    ) -> Result<()> {
3706        let mut next_token = task.next_history_page_token.clone();
3707
3708        while let Some(token) = next_token.take().filter(|token| !token.is_empty()) {
3709            let lease_owner = task
3710                .lease_owner
3711                .clone()
3712                .unwrap_or_else(|| worker_id.to_string());
3713            let page = self
3714                .workflow_task_history_page(
3715                    &task.task_id,
3716                    &lease_owner,
3717                    task.workflow_task_attempt,
3718                    &token,
3719                )
3720                .await?;
3721
3722            task.append_history_page(page);
3723
3724            if task.next_history_page_token.as_deref() == Some(token.as_str()) {
3725                return Err(Error::Codec(
3726                    "workflow history pagination returned the same page token".to_string(),
3727                ));
3728            }
3729
3730            next_token = task.next_history_page_token.clone();
3731        }
3732
3733        Ok(())
3734    }
3735
3736    async fn workflow_task_history_page(
3737        &self,
3738        task_id: &str,
3739        lease_owner: &str,
3740        workflow_task_attempt: u64,
3741        next_history_page_token: &str,
3742    ) -> Result<WorkflowTaskHistoryPage> {
3743        let body = json!({
3744            "lease_owner": lease_owner,
3745            "workflow_task_attempt": workflow_task_attempt,
3746            "next_history_page_token": next_history_page_token
3747        });
3748        let path = format!("/worker/workflow-tasks/{task_id}/history");
3749
3750        self.request_json(
3751            reqwest::Method::POST,
3752            &path,
3753            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3754            Some(&body),
3755        )
3756        .await
3757    }
3758
3759    pub async fn complete_workflow_task(
3760        &self,
3761        task_id: &str,
3762        lease_owner: &str,
3763        workflow_task_attempt: u64,
3764        commands: Vec<Value>,
3765    ) -> Result<Value> {
3766        self.complete_workflow_task_with_message_streams(
3767            task_id,
3768            lease_owner,
3769            workflow_task_attempt,
3770            commands,
3771            Vec::new(),
3772            Vec::new(),
3773        )
3774        .await
3775    }
3776
3777    async fn complete_workflow_task_with_message_streams(
3778        &self,
3779        task_id: &str,
3780        lease_owner: &str,
3781        workflow_task_attempt: u64,
3782        commands: Vec<Value>,
3783        message_stream_cursors: Vec<Value>,
3784        message_stream_waits: Vec<Value>,
3785    ) -> Result<Value> {
3786        validate_workflow_task_commands(&commands)?;
3787        let has_message_stream_metadata =
3788            !message_stream_cursors.is_empty() || !message_stream_waits.is_empty();
3789        if has_message_stream_metadata
3790            && !worker_protocol_supports_message_streams(WORKER_PROTOCOL_VERSION)
3791        {
3792            return Err(Error::Codec(
3793                "message_streams_unavailable: message stream completion metadata requires worker protocol 1.15 or newer"
3794                    .to_string(),
3795            ));
3796        }
3797        let protocol_version = workflow_completion_protocol_version_with_message_streams(
3798            &commands,
3799            has_message_stream_metadata,
3800        );
3801        let mut body = json!({
3802            "lease_owner": lease_owner,
3803            "workflow_task_attempt": workflow_task_attempt,
3804            "commands": commands
3805        });
3806        if !message_stream_cursors.is_empty() {
3807            body["message_stream_cursors"] = Value::Array(message_stream_cursors);
3808        }
3809        if !message_stream_waits.is_empty() {
3810            body["message_stream_waits"] = Value::Array(message_stream_waits);
3811        }
3812        let path = format!("/worker/workflow-tasks/{task_id}/complete");
3813        self.request_json(
3814            reqwest::Method::POST,
3815            &path,
3816            RequestProtocol::Worker(protocol_version),
3817            Some(&body),
3818        )
3819        .await
3820    }
3821
3822    pub async fn fail_workflow_task(
3823        &self,
3824        task_id: &str,
3825        lease_owner: &str,
3826        workflow_task_attempt: u64,
3827        message: impl Into<String>,
3828    ) -> Result<Value> {
3829        self.fail_workflow_task_with_type(
3830            task_id,
3831            lease_owner,
3832            workflow_task_attempt,
3833            message,
3834            "RustWorkflowTaskFailure",
3835        )
3836        .await
3837    }
3838
3839    async fn fail_workflow_task_with_type(
3840        &self,
3841        task_id: &str,
3842        lease_owner: &str,
3843        workflow_task_attempt: u64,
3844        message: impl Into<String>,
3845        failure_type: &str,
3846    ) -> Result<Value> {
3847        let body = json!({
3848            "lease_owner": lease_owner,
3849            "workflow_task_attempt": workflow_task_attempt,
3850            "failure": {
3851                "message": message.into(),
3852                "type": failure_type
3853            }
3854        });
3855        let path = format!("/worker/workflow-tasks/{task_id}/fail");
3856        self.request_json(
3857            reqwest::Method::POST,
3858            &path,
3859            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3860            Some(&body),
3861        )
3862        .await
3863    }
3864
3865    pub async fn poll_activity_task(
3866        &self,
3867        worker_id: &str,
3868        task_queue: &str,
3869        timeout: Duration,
3870    ) -> Result<Option<ActivityTask>> {
3871        Ok(self
3872            .poll_activity_task_response(worker_id, task_queue, timeout)
3873            .await?
3874            .task)
3875    }
3876
3877    /// Poll an activity task while preserving server stop and drain metadata.
3878    pub async fn poll_activity_task_response(
3879        &self,
3880        worker_id: &str,
3881        task_queue: &str,
3882        timeout: Duration,
3883    ) -> Result<PollActivityTaskResponse> {
3884        let poll_request_id = unique_request_id("rust-activity-poll");
3885        self.poll_activity_task_response_with_request_id(
3886            worker_id,
3887            task_queue,
3888            timeout,
3889            &poll_request_id,
3890            1,
3891        )
3892        .await
3893    }
3894
3895    async fn poll_activity_task_response_with_request_id(
3896        &self,
3897        worker_id: &str,
3898        task_queue: &str,
3899        timeout: Duration,
3900        poll_request_id: &str,
3901        transport_retries: usize,
3902    ) -> Result<PollActivityTaskResponse> {
3903        let body = json!({
3904            "worker_id": worker_id,
3905            "task_queue": task_queue,
3906            "poll_request_id": poll_request_id,
3907            "timeout_seconds": long_poll_timeout_seconds(timeout),
3908        });
3909        let data: PollActivityTaskResponse = self
3910            .poll_request_json(
3911                "/worker/activity-tasks/poll",
3912                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3913                &body,
3914                timeout + Duration::from_secs(5),
3915                transport_retries,
3916            )
3917            .await?;
3918        Ok(data)
3919    }
3920
3921    pub async fn complete_activity_task<T: Serialize>(
3922        &self,
3923        task_id: &str,
3924        activity_attempt_id: &str,
3925        lease_owner: &str,
3926        result: T,
3927        codec: &str,
3928    ) -> Result<Value> {
3929        let result = encode_typed_envelope(&AvroValue::from_serialize(&result)?, codec)?;
3930        let body = json!({
3931            "activity_attempt_id": activity_attempt_id,
3932            "lease_owner": lease_owner,
3933            "result": result
3934        });
3935        let path = format!("/worker/activity-tasks/{task_id}/complete");
3936        activity_task_response(
3937            self.request_json(
3938                reqwest::Method::POST,
3939                &path,
3940                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3941                Some(&body),
3942            )
3943            .await,
3944            "complete",
3945            task_id,
3946            activity_attempt_id,
3947        )
3948    }
3949
3950    pub async fn fail_activity_task(
3951        &self,
3952        task_id: &str,
3953        activity_attempt_id: &str,
3954        lease_owner: &str,
3955        message: impl Into<String>,
3956        non_retryable: bool,
3957    ) -> Result<Value> {
3958        let body = json!({
3959            "activity_attempt_id": activity_attempt_id,
3960            "lease_owner": lease_owner,
3961            "failure": {
3962                "message": message.into(),
3963                "type": "RustActivityFailure",
3964                "non_retryable": non_retryable
3965            }
3966        });
3967        let path = format!("/worker/activity-tasks/{task_id}/fail");
3968        activity_task_response(
3969            self.request_json(
3970                reqwest::Method::POST,
3971                &path,
3972                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3973                Some(&body),
3974            )
3975            .await,
3976            "fail",
3977            task_id,
3978            activity_attempt_id,
3979        )
3980    }
3981
3982    pub async fn heartbeat_activity_task<T: Serialize>(
3983        &self,
3984        task_id: &str,
3985        activity_attempt_id: &str,
3986        lease_owner: &str,
3987        details: T,
3988    ) -> Result<ActivityHeartbeatResponse> {
3989        let details = encode_typed_envelope(&AvroValue::from_serialize(&details)?, DEFAULT_CODEC)?;
3990        let body = json!({
3991            "activity_attempt_id": activity_attempt_id,
3992            "lease_owner": lease_owner,
3993            "details": details
3994        });
3995        let path = format!("/worker/activity-tasks/{task_id}/heartbeat");
3996        activity_task_response(
3997            self.request_json(
3998                reqwest::Method::POST,
3999                &path,
4000                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
4001                Some(&body),
4002            )
4003            .await,
4004            "heartbeat",
4005            task_id,
4006            activity_attempt_id,
4007        )
4008    }
4009
4010    async fn request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
4011        &self,
4012        method: reqwest::Method,
4013        path: &str,
4014        protocol: RequestProtocol,
4015        body: Option<&B>,
4016    ) -> Result<T> {
4017        self.request_json_with_timeout(method, path, protocol, body, Duration::from_secs(60))
4018            .await
4019    }
4020
4021    async fn request_json_with_timeout<T: DeserializeOwned, B: Serialize + ?Sized>(
4022        &self,
4023        method: reqwest::Method,
4024        path: &str,
4025        protocol: RequestProtocol,
4026        body: Option<&B>,
4027        timeout: Duration,
4028    ) -> Result<T> {
4029        let auth_token = self.auth_token(protocol)?;
4030        let mut request = self
4031            .http
4032            .request(method, format!("{}/api{}", self.base_url, path))
4033            .timeout(timeout)
4034            .header(reqwest::header::ACCEPT, "application/json")
4035            .header(reqwest::header::CONTENT_TYPE, "application/json")
4036            .header("X-Namespace", &self.namespace);
4037
4038        match protocol {
4039            RequestProtocol::Worker(version) => {
4040                request = request.header("X-Durable-Workflow-Protocol-Version", version);
4041            }
4042            RequestProtocol::ControlPlane => {
4043                request = request.header(
4044                    "X-Durable-Workflow-Control-Plane-Version",
4045                    CONTROL_PLANE_VERSION,
4046                );
4047            }
4048        }
4049
4050        if let Some(token) = auth_token {
4051            request = request.bearer_auth(token);
4052        }
4053
4054        if let Some(body) = body {
4055            request = request.json(body);
4056        }
4057
4058        let response = request.send().await?;
4059        let status = response.status();
4060        let bytes = response.bytes().await?;
4061
4062        if !status.is_success() {
4063            let body = String::from_utf8_lossy(&bytes).to_string();
4064            if let Some(protocol) = protocol_failure(status, &body) {
4065                return Err(Error::Protocol(protocol));
4066            }
4067            return Err(Error::Http { status, body });
4068        }
4069
4070        if bytes.is_empty() {
4071            return Ok(serde_json::from_value(Value::Null)?);
4072        }
4073
4074        Ok(serde_json::from_slice(&bytes)?)
4075    }
4076
4077    async fn poll_request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
4078        &self,
4079        path: &str,
4080        protocol: RequestProtocol,
4081        body: &B,
4082        timeout: Duration,
4083        max_retries: usize,
4084    ) -> Result<T> {
4085        let mut retries = 0;
4086
4087        loop {
4088            let response = self
4089                .request_json_with_timeout(
4090                    reqwest::Method::POST,
4091                    path,
4092                    protocol,
4093                    Some(body),
4094                    timeout,
4095                )
4096                .await;
4097
4098            match response {
4099                Err(Error::Transport(_)) if retries < max_retries => retries += 1,
4100                response => return worker_poll_response(response),
4101            }
4102        }
4103    }
4104
4105    fn auth_token(&self, protocol: RequestProtocol) -> Result<Option<&str>> {
4106        match protocol {
4107            RequestProtocol::Worker(_) => {
4108                if let Some(token) = self.worker_token.as_deref().or(self.token.as_deref()) {
4109                    return Ok(Some(token));
4110                }
4111                if self.control_token.is_some() {
4112                    return Err(Error::MissingRoleCredentials {
4113                        role: "worker",
4114                        opposite_role: "control",
4115                    });
4116                }
4117                Ok(None)
4118            }
4119            RequestProtocol::ControlPlane => {
4120                if let Some(token) = self.control_token.as_deref().or(self.token.as_deref()) {
4121                    return Ok(Some(token));
4122                }
4123                if self.worker_token.is_some() {
4124                    return Err(Error::MissingRoleCredentials {
4125                        role: "control",
4126                        opposite_role: "worker",
4127                    });
4128                }
4129                Ok(None)
4130            }
4131        }
4132    }
4133}
4134
4135fn query_failure(status: reqwest::StatusCode, raw_body: String) -> QueryFailure {
4136    let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
4137    let reason = body
4138        .get("reason")
4139        .and_then(Value::as_str)
4140        .unwrap_or("query_rejected")
4141        .to_string();
4142    let message = body
4143        .get("message")
4144        .or_else(|| body.get("error"))
4145        .and_then(Value::as_str)
4146        .unwrap_or("workflow query was rejected")
4147        .to_string();
4148
4149    QueryFailure {
4150        status: status.as_u16(),
4151        reason,
4152        message,
4153        body,
4154    }
4155}
4156
4157fn workflow_command_result(
4158    command: WorkflowCommandKind,
4159    data: Value,
4160    workflow_id: &str,
4161    run_id: Option<&str>,
4162) -> WorkflowCommandResult {
4163    WorkflowCommandResult {
4164        command,
4165        workflow_id: data
4166            .get("workflow_id")
4167            .and_then(Value::as_str)
4168            .unwrap_or(workflow_id)
4169            .to_string(),
4170        run_id: data
4171            .get("run_id")
4172            .and_then(Value::as_str)
4173            .or(run_id)
4174            .map(str::to_string),
4175        outcome: data
4176            .get("outcome")
4177            .and_then(Value::as_str)
4178            .map(str::to_string),
4179        reason: data
4180            .get("reason")
4181            .and_then(Value::as_str)
4182            .map(str::to_string),
4183        command_status: data
4184            .get("command_status")
4185            .and_then(Value::as_str)
4186            .map(str::to_string),
4187        raw: data,
4188    }
4189}
4190
4191fn workflow_command_rejection(
4192    command: WorkflowCommandKind,
4193    status: reqwest::StatusCode,
4194    raw_body: String,
4195    workflow_id: &str,
4196    run_id: Option<&str>,
4197) -> WorkflowCommandRejection {
4198    let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
4199    WorkflowCommandRejection {
4200        command,
4201        status: status.as_u16(),
4202        reason: body
4203            .get("reason")
4204            .and_then(Value::as_str)
4205            .unwrap_or("workflow_command_rejected")
4206            .to_string(),
4207        message: body
4208            .get("message")
4209            .or_else(|| body.get("error"))
4210            .and_then(Value::as_str)
4211            .unwrap_or("workflow lifecycle command was rejected")
4212            .to_string(),
4213        workflow_id: body
4214            .get("workflow_id")
4215            .and_then(Value::as_str)
4216            .unwrap_or(workflow_id)
4217            .to_string(),
4218        run_id: body
4219            .get("run_id")
4220            .and_then(Value::as_str)
4221            .or(run_id)
4222            .map(str::to_string),
4223        target_scope: body
4224            .get("target_scope")
4225            .and_then(Value::as_str)
4226            .map(str::to_string),
4227        body,
4228    }
4229}
4230
4231fn query_task_response(response: Result<Value>) -> Result<Value> {
4232    match response {
4233        Err(Error::Http { status, body }) => Err(Error::QueryFailed(query_failure(status, body))),
4234        response => response,
4235    }
4236}
4237
4238fn worker_poll_response<T: DeserializeOwned>(response: Result<T>) -> Result<T> {
4239    match response {
4240        Err(Error::Http { status, body })
4241            if status == reqwest::StatusCode::CONFLICT && worker_poll_body_is_stop(&body) =>
4242        {
4243            Ok(serde_json::from_str(&body)?)
4244        }
4245        response => response,
4246    }
4247}
4248
4249fn worker_poll_body_is_stop(body: &str) -> bool {
4250    serde_json::from_str::<Value>(body)
4251        .ok()
4252        .is_some_and(|body| {
4253            worker_poll_is_stop(
4254                body.get("poll_status").and_then(Value::as_str),
4255                body.get("reason").and_then(Value::as_str),
4256            )
4257        })
4258}
4259
4260fn worker_poll_is_stop(poll_status: Option<&str>, reason: Option<&str>) -> bool {
4261    matches!(poll_status, Some("draining" | "stopped"))
4262        || matches!(reason, Some("worker_draining" | "worker_stopped"))
4263}
4264
4265fn query_task_rejection_is_final(error: &Error) -> bool {
4266    matches!(
4267        error,
4268        Error::QueryFailed(failure)
4269            if QUERY_TASK_FINAL_REJECTION_REASONS.contains(&failure.reason.as_str())
4270    )
4271}
4272
4273fn activity_task_response<T>(
4274    response: Result<T>,
4275    operation: &str,
4276    task_id: &str,
4277    activity_attempt_id: &str,
4278) -> Result<T> {
4279    match response {
4280        Err(Error::Http { status, body }) => {
4281            let body = serde_json::from_str(&body).unwrap_or_else(|_| json!({"message": body}));
4282            Err(Error::ActivityTaskRejected(ActivityTaskRejection {
4283                operation: operation.to_string(),
4284                status: status.as_u16(),
4285                reason: body
4286                    .get("reason")
4287                    .and_then(Value::as_str)
4288                    .unwrap_or("activity_task_rejected")
4289                    .to_string(),
4290                task_id: body
4291                    .get("task_id")
4292                    .and_then(Value::as_str)
4293                    .unwrap_or(task_id)
4294                    .to_string(),
4295                activity_attempt_id: body
4296                    .get("activity_attempt_id")
4297                    .and_then(Value::as_str)
4298                    .unwrap_or(activity_attempt_id)
4299                    .to_string(),
4300                cancel_requested: body
4301                    .get("cancel_requested")
4302                    .and_then(Value::as_bool)
4303                    .unwrap_or(false),
4304                can_continue: body.get("can_continue").and_then(Value::as_bool),
4305                run_closed_reason: body
4306                    .get("run_closed_reason")
4307                    .and_then(Value::as_str)
4308                    .map(str::to_string),
4309                body,
4310            }))
4311        }
4312        response => response,
4313    }
4314}
4315
4316fn activity_task_rejection_is_final(error: &Error) -> bool {
4317    matches!(
4318        error,
4319        Error::ActivityTaskRejected(rejection)
4320            if matches!(
4321                rejection.reason.as_str(),
4322                "run_cancelled"
4323                    | "run_terminated"
4324                    | "attempt_closed"
4325                    | "stale_attempt"
4326                    | "activity_cancelled"
4327                    | "task_cancelled"
4328                    | "run_closed"
4329                    | "activity_not_running"
4330                    | "attempt_not_found"
4331            )
4332    )
4333}
4334
4335fn workflow_task_completion_is_terminal_timeout(
4336    error: &Error,
4337    task_id: &str,
4338    workflow_task_attempt: u64,
4339    run_id: Option<&str>,
4340) -> bool {
4341    let Error::Http { status, body } = error else {
4342        return false;
4343    };
4344    if *status != reqwest::StatusCode::CONFLICT {
4345        return false;
4346    }
4347
4348    let Some(run_id) = run_id else {
4349        return false;
4350    };
4351    let Ok(body) = serde_json::from_str::<Value>(body) else {
4352        return false;
4353    };
4354
4355    body.get("recorded").and_then(Value::as_bool) == Some(false)
4356        && body.get("reason").and_then(Value::as_str) == Some("run_timed_out")
4357        && body.get("run_status").and_then(Value::as_str) == Some("failed")
4358        && body.get("run_id").and_then(Value::as_str) == Some(run_id)
4359        && body.get("task_id").and_then(Value::as_str) == Some(task_id)
4360        && body.get("workflow_task_attempt").and_then(Value::as_u64) == Some(workflow_task_attempt)
4361}
4362
4363fn protocol_failure(status: reqwest::StatusCode, raw_body: &str) -> Option<ProtocolFailure> {
4364    let body: Value = serde_json::from_str(raw_body).ok()?;
4365    let reason = body.get("reason")?.as_str()?;
4366    if !matches!(
4367        reason,
4368        "missing_protocol_version"
4369            | "unsupported_protocol_version"
4370            | "missing_control_plane_version"
4371            | "unsupported_control_plane_version"
4372    ) {
4373        return None;
4374    }
4375
4376    Some(ProtocolFailure {
4377        status: status.as_u16(),
4378        reason: reason.to_string(),
4379        message: body
4380            .get("message")
4381            .or_else(|| body.get("error"))
4382            .and_then(Value::as_str)
4383            .unwrap_or("protocol version rejected")
4384            .to_string(),
4385        supported_version: body
4386            .get("supported_version")
4387            .and_then(Value::as_str)
4388            .map(str::to_string),
4389        requested_version: body
4390            .get("requested_version")
4391            .and_then(Value::as_str)
4392            .map(str::to_string),
4393        body,
4394    })
4395}
4396
4397fn long_poll_timeout_seconds(timeout: Duration) -> u64 {
4398    timeout
4399        .as_secs()
4400        .saturating_add(u64::from(timeout.subsec_nanos() > 0))
4401        .min(MAX_LONG_POLL_TIMEOUT_SECONDS)
4402}
4403
4404fn worker_operation_is_retryable(error: &Error) -> bool {
4405    match error {
4406        Error::Transport(error) => {
4407            error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()
4408        }
4409        Error::Http { status, .. } => {
4410            matches!(
4411                *status,
4412                reqwest::StatusCode::REQUEST_TIMEOUT | reqwest::StatusCode::TOO_MANY_REQUESTS
4413            ) || status.is_server_error()
4414        }
4415        _ => false,
4416    }
4417}
4418
4419fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration {
4420    let exponent = retry.saturating_sub(1).min(31) as u32;
4421    policy
4422        .initial_backoff
4423        .saturating_mul(1_u32 << exponent)
4424        .min(policy.max_backoff)
4425}
4426
4427#[derive(Debug)]
4428pub struct ClientBuilder {
4429    base_url: String,
4430    token: Option<String>,
4431    control_token: Option<String>,
4432    worker_token: Option<String>,
4433    namespace: String,
4434    timeout: Duration,
4435}
4436
4437impl ClientBuilder {
4438    pub fn token(mut self, token: Option<String>) -> Self {
4439        self.token = token;
4440        self
4441    }
4442
4443    pub fn control_token(mut self, token: Option<String>) -> Self {
4444        self.control_token = token;
4445        self
4446    }
4447
4448    pub fn worker_token(mut self, token: Option<String>) -> Self {
4449        self.worker_token = token;
4450        self
4451    }
4452
4453    pub fn namespace(mut self, namespace: impl Into<String>) -> Self {
4454        self.namespace = namespace.into();
4455        self
4456    }
4457
4458    pub fn timeout(mut self, timeout: Duration) -> Self {
4459        self.timeout = timeout;
4460        self
4461    }
4462
4463    pub fn build(self) -> Result<Client> {
4464        let base_url = self.base_url.trim_end_matches('/').to_string();
4465        let has_sdk_api_suffix = reqwest::Url::parse(&base_url)
4466            .map(|url| url.path().trim_end_matches('/').ends_with("/api"))
4467            .unwrap_or_else(|_| base_url.ends_with("/api"));
4468
4469        if has_sdk_api_suffix {
4470            return Err(Error::InvalidBaseUrl);
4471        }
4472
4473        Ok(Client {
4474            http: reqwest::Client::builder().timeout(self.timeout).build()?,
4475            base_url,
4476            token: self.token,
4477            control_token: self.control_token,
4478            worker_token: self.worker_token,
4479            namespace: self.namespace,
4480        })
4481    }
4482}
4483
4484#[derive(Clone, Debug)]
4485pub struct WorkflowHandle {
4486    client: Client,
4487    pub workflow_id: String,
4488    pub run_id: Option<String>,
4489    pub workflow_type: String,
4490}
4491
4492impl WorkflowHandle {
4493    /// Describe whichever run is current for this stable workflow instance.
4494    pub async fn describe(&self) -> Result<WorkflowDescription> {
4495        self.client.describe_workflow(&self.workflow_id).await
4496    }
4497
4498    /// Describe the run identity originally selected by this handle.
4499    pub async fn describe_selected_run(&self) -> Result<WorkflowDescription> {
4500        let run_id = self.run_id.as_deref().ok_or_else(|| {
4501            Error::Codec("run_id is required for selected-run description".to_string())
4502        })?;
4503        self.client
4504            .describe_workflow_run(&self.workflow_id, run_id)
4505            .await
4506    }
4507
4508    pub async fn signal<T: Serialize>(&self, signal_name: &str, input: T) -> Result<Value> {
4509        self.client
4510            .signal_workflow(&self.workflow_id, signal_name, input)
4511            .await
4512    }
4513
4514    pub async fn append_message<T: Serialize>(
4515        &self,
4516        stream_name: &str,
4517        message_id: &str,
4518        input: T,
4519    ) -> Result<Value> {
4520        self.client
4521            .append_message_stream(&self.workflow_id, stream_name, message_id, input)
4522            .await
4523    }
4524
4525    /// Signal only if this handle's selected run is still current.
4526    pub async fn signal_selected_run<T: Serialize>(
4527        &self,
4528        signal_name: &str,
4529        input: T,
4530    ) -> Result<Value> {
4531        let run_id = self.run_id.as_deref().ok_or_else(|| {
4532            Error::Codec("run_id is required for selected-run signaling".to_string())
4533        })?;
4534        self.client
4535            .signal_workflow_run(&self.workflow_id, run_id, signal_name, input)
4536            .await
4537    }
4538
4539    /// Request cooperative cancellation of whichever run is current.
4540    pub async fn cancel(&self, options: WorkflowCommandOptions) -> Result<WorkflowCommandResult> {
4541        self.client
4542            .cancel_workflow(&self.workflow_id, options)
4543            .await
4544    }
4545
4546    /// Request cancellation only if this handle's selected run is still current.
4547    pub async fn cancel_selected_run(
4548        &self,
4549        options: WorkflowCommandOptions,
4550    ) -> Result<WorkflowCommandResult> {
4551        let run_id = self.run_id.as_deref().ok_or_else(|| {
4552            Error::Codec("run_id is required for selected-run cancellation".to_string())
4553        })?;
4554        self.client
4555            .cancel_workflow_run(&self.workflow_id, run_id, options)
4556            .await
4557    }
4558
4559    /// Forcefully terminate whichever run is current.
4560    pub async fn terminate(
4561        &self,
4562        options: WorkflowCommandOptions,
4563    ) -> Result<WorkflowCommandResult> {
4564        self.client
4565            .terminate_workflow(&self.workflow_id, options)
4566            .await
4567    }
4568
4569    /// Terminate only if this handle's selected run is still current.
4570    pub async fn terminate_selected_run(
4571        &self,
4572        options: WorkflowCommandOptions,
4573    ) -> Result<WorkflowCommandResult> {
4574        let run_id = self.run_id.as_deref().ok_or_else(|| {
4575            Error::Codec("run_id is required for selected-run termination".to_string())
4576        })?;
4577        self.client
4578            .terminate_workflow_run(&self.workflow_id, run_id, options)
4579            .await
4580    }
4581
4582    /// Execute a named, read-only query against this workflow.
4583    pub async fn query<T: Serialize>(&self, query_name: &str, input: T) -> Result<Value> {
4584        self.client
4585            .query_workflow(&self.workflow_id, query_name, input)
4586            .await
4587    }
4588
4589    pub async fn query_avro_value<T: Serialize>(
4590        &self,
4591        query_name: &str,
4592        input: T,
4593    ) -> Result<AvroValue> {
4594        self.client
4595            .query_workflow_avro_value(&self.workflow_id, query_name, input)
4596            .await
4597    }
4598
4599    pub async fn update<T: Serialize>(
4600        &self,
4601        update_name: &str,
4602        input: T,
4603        request_id: Option<&str>,
4604    ) -> Result<Value> {
4605        self.client
4606            .update_workflow(&self.workflow_id, update_name, input, request_id)
4607            .await
4608    }
4609
4610    pub async fn update_avro_value<T: Serialize>(
4611        &self,
4612        update_name: &str,
4613        input: T,
4614        request_id: Option<&str>,
4615    ) -> Result<AvroValue> {
4616        self.client
4617            .update_workflow_avro_value(&self.workflow_id, update_name, input, request_id)
4618            .await
4619    }
4620
4621    /// Query only if this handle's selected run is still current.
4622    pub async fn query_selected_run<T: Serialize>(
4623        &self,
4624        query_name: &str,
4625        input: T,
4626    ) -> Result<Value> {
4627        let run_id = self
4628            .run_id
4629            .as_deref()
4630            .ok_or_else(|| Error::Codec("run_id is required for selected-run query".to_string()))?;
4631        self.client
4632            .query_workflow_run(&self.workflow_id, run_id, query_name, input)
4633            .await
4634    }
4635
4636    /// Await the final terminal outcome of the current continue-as-new chain.
4637    pub async fn result(&self, options: WorkflowResultOptions) -> Result<Value> {
4638        self.result_target(options, None).await
4639    }
4640
4641    /// Await the final result without projecting Avro bytes through JSON.
4642    pub async fn result_avro_value(&self, options: WorkflowResultOptions) -> Result<AvroValue> {
4643        self.result_avro_value_target(options, None).await
4644    }
4645
4646    /// Await the final result and decode it into a Serde application type.
4647    pub async fn result_typed<T: DeserializeOwned>(
4648        &self,
4649        options: WorkflowResultOptions,
4650    ) -> Result<T> {
4651        let result = self.result_avro_value(options).await?;
4652        decode_handler_result(result, HandlerKind::Workflow, &self.workflow_type)
4653    }
4654
4655    /// Await only the run identity originally selected by this handle.
4656    pub async fn result_selected_run(&self, options: WorkflowResultOptions) -> Result<Value> {
4657        let run_id = self.run_id.as_deref().ok_or_else(|| {
4658            Error::Codec("run_id is required for selected-run result".to_string())
4659        })?;
4660        self.result_target(options, Some(run_id)).await
4661    }
4662
4663    /// Await the selected run's result on the lossless Avro Value surface.
4664    pub async fn result_selected_run_avro_value(
4665        &self,
4666        options: WorkflowResultOptions,
4667    ) -> Result<AvroValue> {
4668        let run_id = self.run_id.as_deref().ok_or_else(|| {
4669            Error::Codec("run_id is required for selected-run result".to_string())
4670        })?;
4671        self.result_avro_value_target(options, Some(run_id)).await
4672    }
4673
4674    /// Await the selected run and decode its result into a Serde type.
4675    pub async fn result_selected_run_typed<T: DeserializeOwned>(
4676        &self,
4677        options: WorkflowResultOptions,
4678    ) -> Result<T> {
4679        let result = self.result_selected_run_avro_value(options).await?;
4680        decode_handler_result(result, HandlerKind::Workflow, &self.workflow_type)
4681    }
4682
4683    async fn result_avro_value_target(
4684        &self,
4685        options: WorkflowResultOptions,
4686        selected_run_id: Option<&str>,
4687    ) -> Result<AvroValue> {
4688        let started = Instant::now();
4689
4690        loop {
4691            let description = match selected_run_id {
4692                Some(run_id) => {
4693                    self.client
4694                        .describe_workflow_run(&self.workflow_id, run_id)
4695                        .await?
4696                }
4697                None => self.describe().await?,
4698            };
4699            if description.is_completed() {
4700                return description.output_avro_value.ok_or_else(|| {
4701                    Error::Codec(
4702                        "missing_payload_envelope: typed workflow result requires output_envelope"
4703                            .to_string(),
4704                    )
4705                });
4706            }
4707            if description.is_terminal() {
4708                let outcome =
4709                    workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
4710                return Err(match outcome.kind {
4711                    WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
4712                    WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
4713                    WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
4714                    WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
4715                });
4716            }
4717            if started.elapsed() >= options.timeout {
4718                return Err(Error::Timeout);
4719            }
4720            tokio::time::sleep(options.poll_interval).await;
4721        }
4722    }
4723
4724    async fn result_target(
4725        &self,
4726        options: WorkflowResultOptions,
4727        selected_run_id: Option<&str>,
4728    ) -> Result<Value> {
4729        let started = Instant::now();
4730
4731        loop {
4732            let description = match selected_run_id {
4733                Some(run_id) => {
4734                    self.client
4735                        .describe_workflow_run(&self.workflow_id, run_id)
4736                        .await?
4737                }
4738                None => self.describe().await?,
4739            };
4740            if description.is_completed() {
4741                return Ok(description.output.unwrap_or(Value::Null));
4742            }
4743
4744            if description.is_terminal() {
4745                let outcome =
4746                    workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
4747                return Err(match outcome.kind {
4748                    WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
4749                    WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
4750                    WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
4751                    WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
4752                });
4753            }
4754
4755            if started.elapsed() >= options.timeout {
4756                return Err(Error::WorkflowTimedOut(WorkflowTerminalOutcome {
4757                    kind: WorkflowTerminalKind::TimedOut,
4758                    workflow_id: description
4759                        .workflow_id
4760                        .clone()
4761                        .unwrap_or_else(|| self.workflow_id.clone()),
4762                    run_id: description
4763                        .run_id
4764                        .clone()
4765                        .or_else(|| selected_run_id.map(str::to_string)),
4766                    reason: "result_wait_timeout".to_string(),
4767                    failure_category: Some("client_timeout".to_string()),
4768                    failure_id: None,
4769                    exception_type: None,
4770                    exception_class: None,
4771                    non_retryable: None,
4772                    message: Some(format!(
4773                        "workflow result was not terminal within {:?}",
4774                        options.timeout
4775                    )),
4776                    exception: None,
4777                    raw: description.raw_value(),
4778                }));
4779            }
4780
4781            tokio::time::sleep(options.poll_interval).await;
4782        }
4783    }
4784}
4785
4786#[derive(Clone, Copy, Debug)]
4787pub struct WorkflowResultOptions {
4788    pub poll_interval: Duration,
4789    pub timeout: Duration,
4790}
4791
4792impl Default for WorkflowResultOptions {
4793    fn default() -> Self {
4794        Self {
4795            poll_interval: Duration::from_millis(500),
4796            timeout: Duration::from_secs(30),
4797        }
4798    }
4799}
4800
4801#[derive(Clone, Debug, Deserialize)]
4802pub struct WorkflowDescription {
4803    pub workflow_id: Option<String>,
4804    pub run_id: Option<String>,
4805    pub workflow_type: Option<String>,
4806    pub status: Option<String>,
4807    #[serde(default)]
4808    pub closed_reason: Option<String>,
4809    #[serde(default)]
4810    pub error: Option<String>,
4811    #[serde(default)]
4812    pub failure: Option<Value>,
4813    #[serde(default)]
4814    pub exception: Option<Value>,
4815    #[serde(default)]
4816    pub failures: Vec<Value>,
4817    #[serde(default)]
4818    pub output: Option<Value>,
4819    #[serde(default)]
4820    pub output_envelope: Option<Value>,
4821    #[serde(skip)]
4822    pub output_avro_value: Option<AvroValue>,
4823    #[serde(flatten)]
4824    pub raw: HashMap<String, Value>,
4825}
4826
4827/// Lifecycle and backlog metadata for one run-scoped Workflow Stream.
4828#[derive(Clone, Debug, Deserialize)]
4829pub struct WorkflowStreamDescription {
4830    pub stream_name: String,
4831    pub status: String,
4832    pub last_offset: i64,
4833    pub total_items: u64,
4834    pub pending_items: u64,
4835    #[serde(default)]
4836    pub opened_at: Option<String>,
4837    #[serde(default)]
4838    pub last_appended_at: Option<String>,
4839    #[serde(default)]
4840    pub closed_at: Option<String>,
4841    #[serde(default)]
4842    pub error_reason: Option<String>,
4843    #[serde(default)]
4844    pub retention_seconds: Option<u64>,
4845    #[serde(flatten)]
4846    pub raw: HashMap<String, Value>,
4847}
4848
4849impl WorkflowStreamDescription {
4850    pub fn is_terminal(&self) -> bool {
4851        matches!(self.status.as_str(), "closed" | "errored")
4852    }
4853}
4854
4855/// One item for direct or replay-safe append.
4856#[derive(Clone, Debug, Default)]
4857pub struct WorkflowStreamAppendItem {
4858    pub payload_envelope: Option<Value>,
4859    pub payload_reference: Option<String>,
4860    pub item_type: Option<String>,
4861    pub content_type: Option<String>,
4862    pub idempotency_key: Option<String>,
4863}
4864
4865impl WorkflowStreamAppendItem {
4866    /// Encode an inline payload with the SDK's fixed Avro Value envelope.
4867    pub fn new<T: Serialize>(payload: T) -> Result<Self> {
4868        let value = AvroValue::from_serialize(&payload)?;
4869        Ok(Self {
4870            payload_envelope: Some(encode_typed_envelope(&value, DEFAULT_CODEC)?),
4871            ..Self::default()
4872        })
4873    }
4874
4875    /// Preserve an external payload URI as an opaque service-contract reference.
4876    pub fn from_reference(reference: impl Into<String>) -> Self {
4877        Self {
4878            payload_reference: Some(reference.into()),
4879            ..Self::default()
4880        }
4881    }
4882
4883    pub fn item_type(mut self, item_type: impl Into<String>) -> Self {
4884        self.item_type = Some(item_type.into());
4885        self
4886    }
4887
4888    pub fn content_type(mut self, content_type: impl Into<String>) -> Self {
4889        self.content_type = Some(content_type.into());
4890        self
4891    }
4892
4893    pub fn idempotency_key(mut self, idempotency_key: impl Into<String>) -> Self {
4894        self.idempotency_key = Some(idempotency_key.into());
4895        self
4896    }
4897
4898    fn wire_value(&self, derived_idempotency_key: Option<String>) -> Value {
4899        let mut item = serde_json::Map::new();
4900        if let Some(payload) = &self.payload_envelope {
4901            item.insert("payload".to_string(), payload.clone());
4902            item.insert("payload_codec".to_string(), json!(DEFAULT_CODEC));
4903        }
4904        if let Some(reference) = &self.payload_reference {
4905            item.insert("payload_reference".to_string(), json!(reference));
4906        }
4907        if let Some(item_type) = &self.item_type {
4908            item.insert("item_type".to_string(), json!(item_type));
4909        }
4910        if let Some(content_type) = &self.content_type {
4911            item.insert("content_type".to_string(), json!(content_type));
4912        }
4913        if let Some(key) = derived_idempotency_key
4914            .as_ref()
4915            .or(self.idempotency_key.as_ref())
4916        {
4917            item.insert("idempotency_key".to_string(), json!(key));
4918        }
4919        Value::Object(item)
4920    }
4921}
4922
4923/// One durable item at its stable zero-based offset.
4924#[derive(Clone, Debug)]
4925pub struct WorkflowStreamItem {
4926    pub offset: u64,
4927    pub payload: Option<Value>,
4928    pub payload_envelope: Option<Value>,
4929    pub payload_reference: Option<String>,
4930    pub payload_codec: Option<String>,
4931    pub idempotency_key: Option<String>,
4932    pub item_type: Option<String>,
4933    pub content_type: Option<String>,
4934    pub origin: Option<String>,
4935    pub origin_reference: Option<String>,
4936    pub emitted_at: Option<String>,
4937    pub raw: Value,
4938}
4939
4940/// One bounded at-least-once subscription page.
4941#[derive(Clone, Debug)]
4942pub struct WorkflowStreamPage {
4943    pub stream: WorkflowStreamDescription,
4944    pub items: Vec<WorkflowStreamItem>,
4945    pub next_offset: u64,
4946    pub terminal: bool,
4947}
4948
4949/// Durable acceptance and deduplication outcome for an append request.
4950#[derive(Clone, Debug)]
4951pub struct WorkflowStreamAppendResult {
4952    pub stream: WorkflowStreamDescription,
4953    pub accepted_offsets: Vec<u64>,
4954    pub accepted: u64,
4955    pub deduped: u64,
4956}
4957
4958#[derive(Deserialize)]
4959struct WorkflowStreamListResponse {
4960    #[serde(default)]
4961    streams: Vec<WorkflowStreamDescription>,
4962}
4963
4964#[derive(Deserialize)]
4965struct WorkflowStreamDescriptionResponse {
4966    stream: WorkflowStreamDescription,
4967}
4968
4969#[derive(Deserialize)]
4970struct WorkflowStreamPageResponse {
4971    stream: WorkflowStreamDescription,
4972    #[serde(default)]
4973    items: Vec<Value>,
4974    next_offset: u64,
4975    terminal: bool,
4976}
4977
4978#[derive(Deserialize)]
4979struct WorkflowStreamAppendResponse {
4980    stream: WorkflowStreamDescription,
4981    #[serde(default)]
4982    accepted_offsets: Vec<u64>,
4983    accepted: u64,
4984    deduped: u64,
4985}
4986
4987impl WorkflowDescription {
4988    pub fn is_completed(&self) -> bool {
4989        matches!(self.status.as_deref(), Some("completed" | "Completed"))
4990    }
4991
4992    pub fn is_terminal(&self) -> bool {
4993        matches!(
4994            self.status.as_deref(),
4995            Some(
4996                "completed"
4997                    | "Completed"
4998                    | "failed"
4999                    | "Failed"
5000                    | "cancelled"
5001                    | "Cancelled"
5002                    | "terminated"
5003                    | "Terminated"
5004                    | "timed_out"
5005                    | "TimedOut",
5006            )
5007        )
5008    }
5009
5010    fn decode_payloads(&mut self) -> Result<()> {
5011        if let Some(envelope) = &self.output_envelope {
5012            let value = decode_wire_avro_value(envelope, DEFAULT_CODEC)?;
5013            self.output = Some(value.clone().into_json()?);
5014            self.output_avro_value = Some(value);
5015        }
5016
5017        Ok(())
5018    }
5019
5020    fn raw_value(&self) -> Value {
5021        let mut data = self.raw.clone();
5022        data.insert(
5023            "workflow_id".to_string(),
5024            self.workflow_id
5025                .clone()
5026                .map(Value::String)
5027                .unwrap_or(Value::Null),
5028        );
5029        data.insert(
5030            "run_id".to_string(),
5031            self.run_id
5032                .clone()
5033                .map(Value::String)
5034                .unwrap_or(Value::Null),
5035        );
5036        data.insert(
5037            "workflow_type".to_string(),
5038            self.workflow_type
5039                .clone()
5040                .map(Value::String)
5041                .unwrap_or(Value::Null),
5042        );
5043        data.insert(
5044            "status".to_string(),
5045            self.status
5046                .clone()
5047                .map(Value::String)
5048                .unwrap_or(Value::Null),
5049        );
5050        data.insert(
5051            "closed_reason".to_string(),
5052            self.closed_reason
5053                .clone()
5054                .map(Value::String)
5055                .unwrap_or(Value::Null),
5056        );
5057        if let Some(failure) = &self.failure {
5058            data.insert("failure".to_string(), failure.clone());
5059        }
5060        if let Some(exception) = &self.exception {
5061            data.insert("exception".to_string(), exception.clone());
5062        }
5063        Value::Object(data.into_iter().collect())
5064    }
5065}
5066
5067fn workflow_terminal_outcome(
5068    description: &WorkflowDescription,
5069    workflow_id: &str,
5070    run_id: Option<&str>,
5071) -> WorkflowTerminalOutcome {
5072    let terminal_kind = description
5073        .closed_reason
5074        .as_deref()
5075        .or(description.status.as_deref())
5076        .unwrap_or("failed")
5077        .to_ascii_lowercase();
5078    let kind = match terminal_kind.as_str() {
5079        "cancelled" | "canceled" => WorkflowTerminalKind::Cancelled,
5080        "terminated" => WorkflowTerminalKind::Terminated,
5081        "timed_out" | "timedout" => WorkflowTerminalKind::TimedOut,
5082        _ => WorkflowTerminalKind::Failed,
5083    };
5084    let default_reason = match kind {
5085        WorkflowTerminalKind::Failed => "workflow_failed",
5086        WorkflowTerminalKind::Cancelled => "cancelled",
5087        WorkflowTerminalKind::Terminated => "terminated",
5088        WorkflowTerminalKind::TimedOut => "timed_out",
5089    };
5090    let failure = description
5091        .failure
5092        .as_ref()
5093        .filter(|value| value.is_object());
5094    let nested_failure = failure
5095        .and_then(|value| value.get("failures"))
5096        .and_then(Value::as_array)
5097        .and_then(|failures| failures.last())
5098        .or_else(|| description.failures.last());
5099    let exception = description
5100        .exception
5101        .clone()
5102        .or_else(|| failure.and_then(|value| value.get("exception")).cloned())
5103        .or_else(|| {
5104            nested_failure
5105                .and_then(|value| value.get("exception_payload"))
5106                .cloned()
5107        });
5108    let string_field = |name: &str| {
5109        failure
5110            .and_then(|value| value.get(name))
5111            .and_then(Value::as_str)
5112            .or_else(|| {
5113                nested_failure
5114                    .and_then(|value| value.get(name))
5115                    .and_then(Value::as_str)
5116            })
5117            .map(str::to_string)
5118    };
5119    let exception_field = |name: &str| {
5120        exception
5121            .as_ref()
5122            .and_then(|value| value.get(name))
5123            .and_then(Value::as_str)
5124            .map(str::to_string)
5125    };
5126    let message = description
5127        .error
5128        .clone()
5129        .or_else(|| string_field("message"))
5130        .or_else(|| exception_field("message"));
5131    let reason = description
5132        .raw
5133        .get("reason")
5134        .and_then(Value::as_str)
5135        .map(str::to_string)
5136        .or_else(|| {
5137            failure
5138                .and_then(|value| value.get("reason"))
5139                .and_then(Value::as_str)
5140                .map(str::to_string)
5141        })
5142        .or_else(|| description.closed_reason.clone())
5143        .unwrap_or_else(|| default_reason.to_string());
5144    let failure_id = string_field("failure_id").or_else(|| {
5145        nested_failure
5146            .and_then(|value| value.get("id"))
5147            .and_then(Value::as_str)
5148            .map(str::to_string)
5149    });
5150
5151    WorkflowTerminalOutcome {
5152        kind,
5153        workflow_id: description
5154            .workflow_id
5155            .clone()
5156            .unwrap_or_else(|| workflow_id.to_string()),
5157        run_id: description
5158            .run_id
5159            .clone()
5160            .or_else(|| run_id.map(str::to_string)),
5161        reason,
5162        failure_category: string_field("failure_category")
5163            .or_else(|| Some(default_reason.to_string())),
5164        failure_id,
5165        exception_type: string_field("exception_type").or_else(|| exception_field("type")),
5166        exception_class: string_field("exception_class").or_else(|| exception_field("class")),
5167        non_retryable: failure
5168            .and_then(|value| value.get("non_retryable"))
5169            .and_then(Value::as_bool)
5170            .or_else(|| {
5171                nested_failure
5172                    .and_then(|value| value.get("non_retryable"))
5173                    .and_then(Value::as_bool)
5174            }),
5175        message,
5176        exception,
5177        raw: description.raw_value(),
5178    }
5179}
5180
5181#[derive(Clone, Debug, Deserialize)]
5182pub struct RegisterWorkerResponse {
5183    pub worker_id: String,
5184    pub registered: bool,
5185    #[serde(default)]
5186    pub heartbeat_interval_seconds: Option<u64>,
5187    #[serde(default)]
5188    pub protocol_version: Option<String>,
5189    #[serde(default)]
5190    pub server_capabilities: Option<Value>,
5191}
5192
5193/// Result of gracefully removing a worker-plane registration.
5194#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
5195pub struct WorkerDeregistrationEnvelope {
5196    pub worker_id: String,
5197    pub outcome: String,
5198    pub recovered_workflow_task_count: u64,
5199}
5200
5201#[derive(Clone, Debug, Deserialize)]
5202pub struct PollWorkflowTaskResponse {
5203    #[serde(default)]
5204    pub task: Option<WorkflowTask>,
5205    #[serde(default)]
5206    pub poll_status: Option<String>,
5207    #[serde(default)]
5208    pub reason: Option<String>,
5209    #[serde(default)]
5210    pub protocol_version: Option<String>,
5211    #[serde(default)]
5212    pub server_capabilities: Option<Value>,
5213}
5214
5215impl PollWorkflowTaskResponse {
5216    /// Classify this response without parsing server display text.
5217    pub fn outcome(&self) -> WorkerPollOutcome {
5218        worker_poll_outcome(
5219            self.task.is_some(),
5220            self.poll_status.as_deref(),
5221            self.reason.as_deref(),
5222        )
5223    }
5224}
5225
5226fn runtime_supports_workflow_memo_updates(capabilities: Option<&Value>) -> bool {
5227    let Some(capabilities) = capabilities.and_then(Value::as_object) else {
5228        return false;
5229    };
5230    let supported = capabilities
5231        .get("workflow_memo_updates")
5232        .and_then(Value::as_object)
5233        .and_then(|memo| memo.get("supported"))
5234        .and_then(Value::as_bool)
5235        == Some(true);
5236    let command_advertised = capabilities
5237        .get("supported_workflow_task_commands")
5238        .and_then(Value::as_array)
5239        .is_some_and(|commands| {
5240            commands
5241                .iter()
5242                .any(|command| command.as_str() == Some("upsert_memo"))
5243        });
5244    supported && command_advertised
5245}
5246
5247fn commands_use_workflow_memo_updates(commands: &[Value]) -> bool {
5248    commands
5249        .iter()
5250        .any(|command| command.get("type").and_then(Value::as_str) == Some("upsert_memo"))
5251}
5252
5253#[derive(Clone, Debug, Deserialize)]
5254pub struct PollActivityTaskResponse {
5255    #[serde(default)]
5256    pub task: Option<ActivityTask>,
5257    #[serde(default)]
5258    pub poll_status: Option<String>,
5259    #[serde(default)]
5260    pub reason: Option<String>,
5261}
5262
5263impl PollActivityTaskResponse {
5264    /// Classify this response without parsing server display text.
5265    pub fn outcome(&self) -> WorkerPollOutcome {
5266        worker_poll_outcome(
5267            self.task.is_some(),
5268            self.poll_status.as_deref(),
5269            self.reason.as_deref(),
5270        )
5271    }
5272}
5273
5274#[derive(Clone, Debug, Deserialize)]
5275pub struct PollQueryTaskResponse {
5276    #[serde(default)]
5277    pub task: Option<QueryTask>,
5278    #[serde(default)]
5279    pub poll_status: Option<String>,
5280    #[serde(default)]
5281    pub reason: Option<String>,
5282}
5283
5284impl PollQueryTaskResponse {
5285    /// Classify this response without parsing server display text.
5286    pub fn outcome(&self) -> WorkerPollOutcome {
5287        worker_poll_outcome(
5288            self.task.is_some(),
5289            self.poll_status.as_deref(),
5290            self.reason.as_deref(),
5291        )
5292    }
5293}
5294
5295/// Stable classification for worker poll responses.
5296#[derive(Clone, Debug, PartialEq, Eq)]
5297pub enum WorkerPollOutcome {
5298    /// A task was leased and is available on the response.
5299    Task,
5300    /// No task was leased, but the worker should continue polling.
5301    Idle {
5302        poll_status: Option<String>,
5303        reason: Option<String>,
5304    },
5305    /// The server asked this worker to stop claiming new work.
5306    Stop {
5307        poll_status: Option<String>,
5308        reason: Option<String>,
5309    },
5310}
5311
5312impl WorkerPollOutcome {
5313    pub fn should_stop(&self) -> bool {
5314        matches!(self, Self::Stop { .. })
5315    }
5316}
5317
5318fn worker_poll_outcome(
5319    has_task: bool,
5320    poll_status: Option<&str>,
5321    reason: Option<&str>,
5322) -> WorkerPollOutcome {
5323    if worker_poll_is_stop(poll_status, reason) {
5324        return WorkerPollOutcome::Stop {
5325            poll_status: poll_status.map(str::to_string),
5326            reason: reason.map(str::to_string),
5327        };
5328    }
5329
5330    if has_task {
5331        WorkerPollOutcome::Task
5332    } else {
5333        WorkerPollOutcome::Idle {
5334            poll_status: poll_status.map(str::to_string),
5335            reason: reason.map(str::to_string),
5336        }
5337    }
5338}
5339
5340/// An ephemeral server-routed query task.
5341#[derive(Clone, Debug, Deserialize)]
5342pub struct QueryTask {
5343    pub query_task_id: String,
5344    #[serde(default = "default_workflow_task_attempt")]
5345    pub query_task_attempt: u64,
5346    #[serde(default)]
5347    pub lease_owner: Option<String>,
5348    #[serde(default)]
5349    pub workflow_id: Option<String>,
5350    #[serde(default)]
5351    pub run_id: Option<String>,
5352    pub workflow_type: String,
5353    pub query_name: String,
5354    #[serde(
5355        default = "missing_task_payload_codec",
5356        deserialize_with = "deserialize_task_payload_codec"
5357    )]
5358    pub payload_codec: String,
5359    #[serde(default)]
5360    pub workflow_arguments: Option<Value>,
5361    #[serde(default)]
5362    pub query_arguments: Option<Value>,
5363    #[serde(default)]
5364    pub history_events: Vec<HistoryEvent>,
5365    #[serde(default)]
5366    pub history_export: Option<Value>,
5367    #[serde(default)]
5368    pub run_status: Option<String>,
5369}
5370
5371#[derive(Clone, Debug, Deserialize)]
5372pub struct WorkflowTask {
5373    pub task_id: String,
5374    #[serde(default)]
5375    pub workflow_command_id: Option<String>,
5376    #[serde(default)]
5377    pub workflow_id: Option<String>,
5378    #[serde(default)]
5379    pub run_id: Option<String>,
5380    pub workflow_type: String,
5381    #[serde(default)]
5382    pub cancel_requested: bool,
5383    #[serde(
5384        default = "missing_task_payload_codec",
5385        deserialize_with = "deserialize_task_payload_codec"
5386    )]
5387    pub payload_codec: String,
5388    #[serde(default)]
5389    pub arguments: Option<Value>,
5390    #[serde(default)]
5391    pub history_events: Vec<HistoryEvent>,
5392    #[serde(default)]
5393    pub total_history_events: Option<u64>,
5394    #[serde(default)]
5395    pub history_size_bytes: Option<u64>,
5396    #[serde(default)]
5397    pub continue_as_new_recommended: Option<bool>,
5398    #[serde(default)]
5399    pub history_budget_pressure: Option<String>,
5400    #[serde(default)]
5401    pub next_history_page_token: Option<String>,
5402    #[serde(default = "default_workflow_task_attempt")]
5403    pub workflow_task_attempt: u64,
5404    #[serde(default)]
5405    pub workflow_signal_id: Option<String>,
5406    #[serde(default)]
5407    pub signal_name: Option<String>,
5408    #[serde(default)]
5409    pub signal_arguments: Option<Value>,
5410    #[serde(default)]
5411    pub workflow_update_id: Option<String>,
5412    #[serde(default)]
5413    pub update_name: Option<String>,
5414    #[serde(default)]
5415    pub lease_owner: Option<String>,
5416}
5417
5418impl WorkflowTask {
5419    fn append_history_page(&mut self, page: WorkflowTaskHistoryPage) {
5420        self.history_events.extend(page.history_events);
5421
5422        if page.total_history_events.is_some() {
5423            self.total_history_events = page.total_history_events;
5424        }
5425
5426        self.next_history_page_token = page
5427            .next_history_page_token
5428            .filter(|token| !token.is_empty());
5429    }
5430}
5431
5432#[derive(Clone, Debug, Deserialize)]
5433struct WorkflowTaskHistoryPage {
5434    #[serde(default)]
5435    history_events: Vec<HistoryEvent>,
5436    #[serde(default)]
5437    total_history_events: Option<u64>,
5438    #[serde(default)]
5439    next_history_page_token: Option<String>,
5440}
5441
5442#[derive(Clone, Debug, Deserialize)]
5443pub struct ActivityTask {
5444    pub task_id: String,
5445    #[serde(default)]
5446    pub activity_attempt_id: Option<String>,
5447    #[serde(default)]
5448    pub attempt_id: Option<String>,
5449    pub activity_type: String,
5450    #[serde(
5451        default = "missing_task_payload_codec",
5452        deserialize_with = "deserialize_task_payload_codec"
5453    )]
5454    pub payload_codec: String,
5455    #[serde(default)]
5456    pub arguments: Option<Value>,
5457    #[serde(default = "default_attempt_number")]
5458    pub attempt_number: u64,
5459    #[serde(default)]
5460    pub lease_owner: Option<String>,
5461}
5462
5463#[derive(Clone, Debug, Deserialize)]
5464pub struct HistoryEvent {
5465    #[serde(alias = "type")]
5466    pub event_type: String,
5467    #[serde(default)]
5468    pub payload: Value,
5469    #[serde(flatten)]
5470    pub raw: HashMap<String, Value>,
5471}
5472
5473/// One decoded signal in the committed workflow-history snapshot.
5474#[derive(Clone, Debug, PartialEq)]
5475pub struct QuerySignal {
5476    pub id: Option<String>,
5477    pub name: String,
5478    pub arguments: Vec<Value>,
5479    avro_arguments: Vec<AvroValue>,
5480    pub workflow_sequence: Option<u64>,
5481}
5482
5483impl QuerySignal {
5484    /// Lossless fixed Avro Value arguments for this committed signal.
5485    pub fn arguments_avro_value(&self) -> &[AvroValue] {
5486        &self.avro_arguments
5487    }
5488}
5489
5490/// Immutable state supplied to a registered query handler.
5491///
5492/// This context intentionally exposes no activity, signal-wait, or command
5493/// APIs. Query handlers inspect committed history and return a value; query
5494/// completion does not append an event or advance deterministic execution.
5495#[derive(Clone, Debug)]
5496pub struct QueryContext {
5497    pub workflow_id: Option<String>,
5498    pub run_id: Option<String>,
5499    pub workflow_type: String,
5500    pub run_status: Option<String>,
5501    workflow_input: Value,
5502    workflow_input_avro_value: AvroValue,
5503    history_events: Arc<Vec<HistoryEvent>>,
5504    signal_events: Arc<Vec<QuerySignal>>,
5505}
5506
5507impl QueryContext {
5508    /// The normalized argument list used to start the workflow.
5509    pub fn workflow_input(&self) -> &Value {
5510        &self.workflow_input
5511    }
5512
5513    /// The lossless fixed Avro Value argument list used to start the workflow.
5514    pub fn workflow_input_avro_value(&self) -> &AvroValue {
5515        &self.workflow_input_avro_value
5516    }
5517
5518    /// The immutable committed history used for this query snapshot.
5519    pub fn history_events(&self) -> &[HistoryEvent] {
5520        self.history_events.as_slice()
5521    }
5522
5523    /// All decoded signals in committed workflow order.
5524    pub fn signal_events(&self) -> &[QuerySignal] {
5525        self.signal_events.as_slice()
5526    }
5527
5528    /// Decoded argument lists for each committed signal with `signal_name`.
5529    pub fn signals(&self, signal_name: &str) -> Vec<Vec<Value>> {
5530        self.signal_events
5531            .iter()
5532            .filter(|signal| signal.name == signal_name)
5533            .map(|signal| signal.arguments.clone())
5534            .collect()
5535    }
5536
5537    /// Lossless fixed Avro Value arguments for committed signals with `signal_name`.
5538    pub fn signals_avro_value(&self, signal_name: &str) -> Vec<Vec<AvroValue>> {
5539        self.signal_events
5540            .iter()
5541            .filter(|signal| signal.name == signal_name)
5542            .map(|signal| signal.avro_arguments.clone())
5543            .collect()
5544    }
5545}
5546
5547#[derive(Clone, Debug, Deserialize)]
5548pub struct ActivityHeartbeatResponse {
5549    #[serde(default)]
5550    pub cancel_requested: bool,
5551    #[serde(default)]
5552    pub heartbeat_recorded: bool,
5553    #[serde(default)]
5554    pub can_continue: Option<bool>,
5555    #[serde(default)]
5556    pub reason: Option<String>,
5557    #[serde(default)]
5558    pub run_closed_reason: Option<String>,
5559    #[serde(default)]
5560    pub run_closed_at: Option<String>,
5561    #[serde(default)]
5562    pub lease_expires_at: Option<String>,
5563    #[serde(default)]
5564    pub last_heartbeat_at: Option<String>,
5565}
5566
5567impl ActivityHeartbeatResponse {
5568    /// Whether the activity should stop instead of attempting completion.
5569    pub fn should_stop(&self) -> bool {
5570        self.cancel_requested || self.can_continue == Some(false)
5571    }
5572}
5573
5574fn missing_task_payload_codec() -> String {
5575    MISSING_TASK_PAYLOAD_CODEC.to_string()
5576}
5577
5578fn deserialize_task_payload_codec<'de, D>(deserializer: D) -> std::result::Result<String, D::Error>
5579where
5580    D: Deserializer<'de>,
5581{
5582    Ok(match Value::deserialize(deserializer)? {
5583        Value::String(codec) => codec,
5584        Value::Null => NULL_TASK_PAYLOAD_CODEC.to_string(),
5585        _ => NON_STRING_TASK_PAYLOAD_CODEC.to_string(),
5586    })
5587}
5588
5589fn default_workflow_task_attempt() -> u64 {
5590    1
5591}
5592
5593fn default_attempt_number() -> u64 {
5594    1
5595}
5596
5597type WorkflowFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5598type WorkflowHandler = Arc<dyn Fn(WorkflowContext, AvroValue) -> WorkflowFuture + Send + Sync>;
5599type ErasedWorkflowState = Arc<dyn Any + Send + Sync>;
5600type WorkflowStateSnapshot = Arc<dyn Fn() -> Result<ErasedWorkflowState> + Send + Sync>;
5601type ReplayedWorkflowHandler =
5602    Arc<dyn Fn(WorkflowContext, AvroValue) -> ReplayedWorkflowInvocation + Send + Sync>;
5603type ActivityFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5604type ActivityHandler = Arc<dyn Fn(ActivityContext, AvroValue) -> ActivityFuture + Send + Sync>;
5605type QueryFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5606type QueryHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
5607type UpdateHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
5608type ReplayedQueryHandler = Arc<
5609    dyn Fn(QueryContext, ErasedWorkflowState, AvroValue) -> std::result::Result<QueryFuture, String>
5610        + Send
5611        + Sync,
5612>;
5613type WorkerHeartbeatObserver = Arc<dyn Fn(&WorkerHeartbeatObservation) + Send + Sync>;
5614
5615struct ReplayedWorkflowInvocation {
5616    future: WorkflowFuture,
5617    snapshot: WorkflowStateSnapshot,
5618}
5619
5620#[derive(Clone)]
5621struct RegisteredWorkflow {
5622    execute: WorkflowHandler,
5623    replay: Option<ReplayedWorkflowHandler>,
5624    state_type: Option<TypeId>,
5625}
5626
5627#[derive(Debug)]
5628struct WorkflowTaskDecision {
5629    commands: Vec<Value>,
5630    message_stream_cursors: Vec<Value>,
5631    message_stream_waits: Vec<Value>,
5632}
5633
5634impl WorkflowTaskDecision {
5635    fn without_message_streams(commands: Vec<Value>) -> Self {
5636        Self {
5637            commands,
5638            message_stream_cursors: Vec::new(),
5639            message_stream_waits: Vec::new(),
5640        }
5641    }
5642}
5643
5644#[derive(Clone)]
5645enum RegisteredQuery {
5646    Snapshot(QueryHandler),
5647    Replayed {
5648        state_type: TypeId,
5649        handler: ReplayedQueryHandler,
5650    },
5651}
5652
5653#[derive(Clone, Debug)]
5654pub struct WorkerHeartbeatObservation {
5655    pub worker_id: String,
5656    pub task_queue: String,
5657    pub acknowledged_at_unix_millis: u64,
5658    pub acknowledgement: Value,
5659}
5660
5661/// Bounded retry policy for worker poll acquisition and worker heartbeats.
5662///
5663/// Expected empty long polls are normal successful responses. Transport
5664/// failures, HTTP 408/429 responses, and server errors are retried with capped
5665/// exponential backoff. Authentication, protocol, codec, and handler failures
5666/// are never retried by the worker.
5667#[derive(Clone, Copy, Debug)]
5668pub struct WorkerRetryPolicy {
5669    /// Number of retries after the initial request fails.
5670    pub max_retries: usize,
5671    /// Delay before the first retry.
5672    pub initial_backoff: Duration,
5673    /// Maximum delay between retries.
5674    pub max_backoff: Duration,
5675}
5676
5677impl Default for WorkerRetryPolicy {
5678    fn default() -> Self {
5679        Self {
5680            max_retries: 5,
5681            initial_backoff: Duration::from_millis(100),
5682            max_backoff: Duration::from_secs(5),
5683        }
5684    }
5685}
5686
5687#[derive(Clone, Copy, Debug, PartialEq, Eq)]
5688enum ManagedPollOutcome {
5689    Idle,
5690    Handled,
5691    Stop,
5692}
5693
5694#[derive(Clone)]
5695pub struct Worker {
5696    client: Client,
5697    worker_id: String,
5698    task_queue: String,
5699    workflows: HashMap<String, RegisteredWorkflow>,
5700    activities: HashMap<String, ActivityHandler>,
5701    queries: HashMap<String, HashMap<String, RegisteredQuery>>,
5702    updates: HashMap<String, HashMap<String, UpdateHandler>>,
5703    max_concurrent_workflow_tasks: usize,
5704    max_concurrent_activity_tasks: usize,
5705    poll_timeout: Duration,
5706    heartbeat_interval: Duration,
5707    retry_policy: WorkerRetryPolicy,
5708    heartbeat_observer: Option<WorkerHeartbeatObserver>,
5709}
5710
5711impl Worker {
5712    pub fn new(client: Client, task_queue: impl Into<String>) -> Self {
5713        Self {
5714            client,
5715            worker_id: default_worker_id(),
5716            task_queue: task_queue.into(),
5717            workflows: HashMap::new(),
5718            activities: HashMap::new(),
5719            queries: HashMap::new(),
5720            updates: HashMap::new(),
5721            max_concurrent_workflow_tasks: 10,
5722            max_concurrent_activity_tasks: 10,
5723            poll_timeout: Duration::from_secs(30),
5724            heartbeat_interval: Duration::from_secs(60),
5725            retry_policy: WorkerRetryPolicy::default(),
5726            heartbeat_observer: None,
5727        }
5728    }
5729
5730    pub fn worker_id(mut self, worker_id: impl Into<String>) -> Self {
5731        self.worker_id = worker_id.into();
5732        self
5733    }
5734
5735    pub fn poll_timeout(mut self, timeout: Duration) -> Self {
5736        self.poll_timeout = timeout;
5737        self
5738    }
5739
5740    pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
5741        self.heartbeat_interval = interval;
5742        self
5743    }
5744
5745    /// Configure bounded retries for task-poll acquisition and worker heartbeats.
5746    pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self {
5747        self.retry_policy = policy;
5748        self
5749    }
5750
5751    pub fn on_worker_heartbeat<F>(mut self, observer: F) -> Self
5752    where
5753        F: Fn(&WorkerHeartbeatObservation) + Send + Sync + 'static,
5754    {
5755        self.heartbeat_observer = Some(Arc::new(observer));
5756        self
5757    }
5758
5759    pub fn max_concurrent_workflow_tasks(mut self, count: usize) -> Self {
5760        self.max_concurrent_workflow_tasks = count.max(1);
5761        self
5762    }
5763
5764    pub fn max_concurrent_activity_tasks(mut self, count: usize) -> Self {
5765        self.max_concurrent_activity_tasks = count.max(1);
5766        self
5767    }
5768
5769    /// Register a workflow handler.
5770    ///
5771    /// An uncaught [`enum@Error`] returned by the handler fails the workflow run and
5772    /// is reported to clients as [`Error::WorkflowFailed`]. Errors that occur
5773    /// while acquiring or decoding a worker task remain worker-operation
5774    /// failures and do not get converted into workflow outcomes.
5775    pub fn register_workflow<F, Fut>(&mut self, workflow_type: impl Into<String>, handler: F)
5776    where
5777        F: Fn(WorkflowContext, Value) -> Fut + Send + Sync + 'static,
5778        Fut: Future<Output = Result<Value>> + Send + 'static,
5779    {
5780        let handler = Arc::new(handler);
5781        self.workflows.insert(
5782            workflow_type.into(),
5783            RegisteredWorkflow {
5784                execute: Arc::new(move |ctx, input| {
5785                    let handler = Arc::clone(&handler);
5786                    Box::pin(async move {
5787                        let result = handler(ctx, input.into_json()?).await?;
5788                        AvroValue::from_serialize(&result)
5789                    })
5790                }),
5791                replay: None,
5792                state_type: None,
5793            },
5794        );
5795    }
5796
5797    /// Register a workflow with one Serde request value and a Serde result.
5798    ///
5799    /// This is an ergonomic adapter over the same fixed Avro Value protocol as
5800    /// [`Worker::register_workflow_avro_value`]. It does not create or publish a
5801    /// workflow-specific schema. A task must contain zero arguments for a unit
5802    /// request or exactly one argument for every other request type.
5803    ///
5804    /// See the runnable
5805    /// [`hello_world` example](https://github.com/durable-workflow/sdk-rust/blob/main/examples/hello_world.rs)
5806    /// for typed workflow and activity contracts with retry and timeout policy.
5807    pub fn register_typed_workflow<I, O, F, Fut>(
5808        &mut self,
5809        workflow_type: impl Into<String>,
5810        handler: F,
5811    ) where
5812        I: DeserializeOwned + Send + 'static,
5813        O: Serialize + Send + 'static,
5814        F: Fn(WorkflowContext, I) -> Fut + Send + Sync + 'static,
5815        Fut: Future<Output = Result<O>> + Send + 'static,
5816    {
5817        let workflow_type = workflow_type.into();
5818        let handler_name = workflow_type.clone();
5819        let handler = Arc::new(handler);
5820        self.workflows.insert(
5821            workflow_type,
5822            RegisteredWorkflow {
5823                execute: Arc::new(move |ctx, input| {
5824                    let handler = Arc::clone(&handler);
5825                    let handler_name = handler_name.clone();
5826                    Box::pin(async move {
5827                        let input =
5828                            decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
5829                        let result = handler(ctx, input).await?;
5830                        encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
5831                    })
5832                }),
5833                replay: None,
5834                state_type: None,
5835            },
5836        );
5837    }
5838
5839    /// Register a workflow on the lossless fixed Avro Value surface.
5840    pub fn register_workflow_avro_value<F, Fut>(
5841        &mut self,
5842        workflow_type: impl Into<String>,
5843        handler: F,
5844    ) where
5845        F: Fn(WorkflowContext, AvroValue) -> Fut + Send + Sync + 'static,
5846        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
5847    {
5848        self.workflows.insert(
5849            workflow_type.into(),
5850            RegisteredWorkflow {
5851                execute: Arc::new(move |ctx, input| Box::pin(handler(ctx, input))),
5852                replay: None,
5853                state_type: None,
5854            },
5855        );
5856    }
5857
5858    /// Register a workflow whose typed instance state can be reconstructed for queries.
5859    ///
5860    /// `state_factory` creates a fresh instance for every normal workflow task and
5861    /// query replay. The workflow handler is the single source of truth for state
5862    /// transitions: it updates [`WorkflowInstance`] after activities and signals
5863    /// resolve. Query replay runs this same handler over committed history and
5864    /// discards any commands it would emit.
5865    pub fn register_replayed_workflow<S, Factory, F, Fut>(
5866        &mut self,
5867        workflow_type: impl Into<String>,
5868        state_factory: Factory,
5869        handler: F,
5870    ) where
5871        S: Clone + Send + Sync + 'static,
5872        Factory: Fn() -> S + Send + Sync + 'static,
5873        F: Fn(WorkflowContext, Value, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
5874        Fut: Future<Output = Result<Value>> + Send + 'static,
5875    {
5876        let state_factory = Arc::new(state_factory);
5877        let handler = Arc::new(handler);
5878
5879        let execute_factory = Arc::clone(&state_factory);
5880        let execute_handler = Arc::clone(&handler);
5881        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
5882            let state = WorkflowInstance::new(execute_factory());
5883            let handler = Arc::clone(&execute_handler);
5884            Box::pin(async move {
5885                let result = handler(ctx, input.into_json()?, state).await?;
5886                AvroValue::from_serialize(&result)
5887            }) as WorkflowFuture
5888        });
5889
5890        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
5891            let state = WorkflowInstance::new(state_factory());
5892            let snapshot_state = state.clone();
5893            let snapshot: WorkflowStateSnapshot =
5894                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
5895            let replay_handler = Arc::clone(&handler);
5896            let future = async move {
5897                let result = replay_handler(ctx, input.into_json()?, state).await?;
5898                AvroValue::from_serialize(&result)
5899            };
5900            ReplayedWorkflowInvocation {
5901                future: Box::pin(future),
5902                snapshot,
5903            }
5904        });
5905
5906        self.workflows.insert(
5907            workflow_type.into(),
5908            RegisteredWorkflow {
5909                execute,
5910                replay: Some(replay),
5911                state_type: Some(TypeId::of::<S>()),
5912            },
5913        );
5914    }
5915
5916    /// Register a replayable workflow with one Serde request value and result.
5917    ///
5918    /// Normal task execution and instance-state query replay both decode and
5919    /// encode through the fixed Avro Value codec. The state factory and handler
5920    /// otherwise follow [`Worker::register_replayed_workflow`].
5921    pub fn register_typed_replayed_workflow<I, O, S, Factory, F, Fut>(
5922        &mut self,
5923        workflow_type: impl Into<String>,
5924        state_factory: Factory,
5925        handler: F,
5926    ) where
5927        I: DeserializeOwned + Send + 'static,
5928        O: Serialize + Send + 'static,
5929        S: Clone + Send + Sync + 'static,
5930        Factory: Fn() -> S + Send + Sync + 'static,
5931        F: Fn(WorkflowContext, I, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
5932        Fut: Future<Output = Result<O>> + Send + 'static,
5933    {
5934        let workflow_type = workflow_type.into();
5935        let state_factory = Arc::new(state_factory);
5936        let handler = Arc::new(handler);
5937
5938        let execute_name = workflow_type.clone();
5939        let execute_factory = Arc::clone(&state_factory);
5940        let execute_handler = Arc::clone(&handler);
5941        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
5942            let state = WorkflowInstance::new(execute_factory());
5943            let handler = Arc::clone(&execute_handler);
5944            let handler_name = execute_name.clone();
5945            Box::pin(async move {
5946                let input = decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
5947                let result = handler(ctx, input, state).await?;
5948                encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
5949            }) as WorkflowFuture
5950        });
5951
5952        let replay_name = workflow_type.clone();
5953        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
5954            let state = WorkflowInstance::new(state_factory());
5955            let snapshot_state = state.clone();
5956            let snapshot: WorkflowStateSnapshot =
5957                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
5958            let handler = Arc::clone(&handler);
5959            let handler_name = replay_name.clone();
5960            let future = async move {
5961                let input = decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
5962                let result = handler(ctx, input, state).await?;
5963                encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
5964            };
5965            ReplayedWorkflowInvocation {
5966                future: Box::pin(future),
5967                snapshot,
5968            }
5969        });
5970
5971        self.workflows.insert(
5972            workflow_type,
5973            RegisteredWorkflow {
5974                execute,
5975                replay: Some(replay),
5976                state_type: Some(TypeId::of::<S>()),
5977            },
5978        );
5979    }
5980
5981    /// Register a replayable workflow on the lossless fixed Avro Value surface.
5982    pub fn register_replayed_workflow_avro_value<S, Factory, F, Fut>(
5983        &mut self,
5984        workflow_type: impl Into<String>,
5985        state_factory: Factory,
5986        handler: F,
5987    ) where
5988        S: Clone + Send + Sync + 'static,
5989        Factory: Fn() -> S + Send + Sync + 'static,
5990        F: Fn(WorkflowContext, AvroValue, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
5991        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
5992    {
5993        let state_factory = Arc::new(state_factory);
5994        let handler = Arc::new(handler);
5995
5996        let execute_factory = Arc::clone(&state_factory);
5997        let execute_handler = Arc::clone(&handler);
5998        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
5999            let state = WorkflowInstance::new(execute_factory());
6000            Box::pin(execute_handler(ctx, input, state)) as WorkflowFuture
6001        });
6002
6003        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6004            let state = WorkflowInstance::new(state_factory());
6005            let snapshot_state = state.clone();
6006            let snapshot: WorkflowStateSnapshot =
6007                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6008            ReplayedWorkflowInvocation {
6009                future: Box::pin(handler(ctx, input, state)),
6010                snapshot,
6011            }
6012        });
6013
6014        self.workflows.insert(
6015            workflow_type.into(),
6016            RegisteredWorkflow {
6017                execute,
6018                replay: Some(replay),
6019                state_type: Some(TypeId::of::<S>()),
6020            },
6021        );
6022    }
6023
6024    pub fn register_activity<F, Fut>(&mut self, activity_type: impl Into<String>, handler: F)
6025    where
6026        F: Fn(ActivityContext, Value) -> Fut + Send + Sync + 'static,
6027        Fut: Future<Output = Result<Value>> + Send + 'static,
6028    {
6029        let handler = Arc::new(handler);
6030        self.activities.insert(
6031            activity_type.into(),
6032            Arc::new(move |ctx, args| {
6033                let handler = Arc::clone(&handler);
6034                Box::pin(async move {
6035                    let result = handler(ctx, args.into_json()?).await?;
6036                    AvroValue::from_serialize(&result)
6037                })
6038            }),
6039        );
6040    }
6041
6042    /// Register an activity with one Serde request value and a Serde result.
6043    ///
6044    /// Inputs and results use the platform's fixed Avro Value schema. Shape
6045    /// mismatches and unsupported Serde values return [`Error::HandlerType`]
6046    /// with the activity name and Rust type.
6047    pub fn register_typed_activity<I, O, F, Fut>(
6048        &mut self,
6049        activity_type: impl Into<String>,
6050        handler: F,
6051    ) where
6052        I: DeserializeOwned + Send + 'static,
6053        O: Serialize + Send + 'static,
6054        F: Fn(ActivityContext, I) -> Fut + Send + Sync + 'static,
6055        Fut: Future<Output = Result<O>> + Send + 'static,
6056    {
6057        let activity_type = activity_type.into();
6058        let handler_name = activity_type.clone();
6059        let handler = Arc::new(handler);
6060        self.activities.insert(
6061            activity_type,
6062            Arc::new(move |ctx, input| {
6063                let handler = Arc::clone(&handler);
6064                let handler_name = handler_name.clone();
6065                Box::pin(async move {
6066                    let input =
6067                        decode_handler_input::<I>(input, HandlerKind::Activity, &handler_name)?;
6068                    let result = handler(ctx, input).await?;
6069                    encode_handler_result(&result, HandlerKind::Activity, &handler_name)
6070                })
6071            }),
6072        );
6073    }
6074
6075    /// Register an activity on the lossless fixed Avro Value surface.
6076    pub fn register_activity_avro_value<F, Fut>(
6077        &mut self,
6078        activity_type: impl Into<String>,
6079        handler: F,
6080    ) where
6081        F: Fn(ActivityContext, AvroValue) -> Fut + Send + Sync + 'static,
6082        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6083    {
6084        self.activities.insert(
6085            activity_type.into(),
6086            Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
6087        );
6088    }
6089
6090    /// Register a named, read-only query handler for a workflow type.
6091    ///
6092    /// The workflow type must also be registered with [`Worker::register_workflow`]
6093    /// before the worker runs. The handler receives only an immutable committed
6094    /// state snapshot and normalized query arguments.
6095    pub fn register_query<F, Fut>(
6096        &mut self,
6097        workflow_type: impl Into<String>,
6098        query_name: impl Into<String>,
6099        handler: F,
6100    ) where
6101        F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
6102        Fut: Future<Output = Result<Value>> + Send + 'static,
6103    {
6104        let handler = Arc::new(handler);
6105        self.queries
6106            .entry(workflow_type.into())
6107            .or_default()
6108            .insert(
6109                query_name.into(),
6110                RegisteredQuery::Snapshot(Arc::new(move |ctx, args| {
6111                    let handler = Arc::clone(&handler);
6112                    Box::pin(async move {
6113                        let result = handler(ctx, args.into_json()?).await?;
6114                        AvroValue::from_serialize(&result)
6115                    })
6116                })),
6117            );
6118    }
6119
6120    /// Register a query handler on the lossless fixed Avro Value surface.
6121    pub fn register_query_avro_value<F, Fut>(
6122        &mut self,
6123        workflow_type: impl Into<String>,
6124        query_name: impl Into<String>,
6125        handler: F,
6126    ) where
6127        F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
6128        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6129    {
6130        self.queries
6131            .entry(workflow_type.into())
6132            .or_default()
6133            .insert(
6134                query_name.into(),
6135                RegisteredQuery::Snapshot(Arc::new(move |ctx, args| Box::pin(handler(ctx, args)))),
6136            );
6137    }
6138
6139    /// Register a named query against deterministically replayed instance state.
6140    ///
6141    /// The workflow type must use [`Worker::register_replayed_workflow`] with the
6142    /// same state type `S`. The handler receives an immutable, detached state
6143    /// clone, so successful and failed queries cannot affect workflow execution
6144    /// or the state reconstructed by a later query.
6145    pub fn register_replayed_query<S, F, Fut>(
6146        &mut self,
6147        workflow_type: impl Into<String>,
6148        query_name: impl Into<String>,
6149        handler: F,
6150    ) where
6151        S: Clone + Send + Sync + 'static,
6152        F: Fn(QueryContext, Arc<S>, Value) -> Fut + Send + Sync + 'static,
6153        Fut: Future<Output = Result<Value>> + Send + 'static,
6154    {
6155        let handler = Arc::new(handler);
6156        let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
6157            let state = state.downcast::<S>().map_err(|_| {
6158                "registered query state type does not match the replayed workflow state".to_string()
6159            })?;
6160            let handler = Arc::clone(&handler);
6161            Ok(Box::pin(async move {
6162                let result = handler(ctx, state, args.into_json()?).await?;
6163                AvroValue::from_serialize(&result)
6164            }))
6165        });
6166
6167        self.queries
6168            .entry(workflow_type.into())
6169            .or_default()
6170            .insert(
6171                query_name.into(),
6172                RegisteredQuery::Replayed {
6173                    state_type: TypeId::of::<S>(),
6174                    handler: erased_handler,
6175                },
6176            );
6177    }
6178
6179    /// Register a replayed-state query on the lossless fixed Avro Value surface.
6180    pub fn register_replayed_query_avro_value<S, F, Fut>(
6181        &mut self,
6182        workflow_type: impl Into<String>,
6183        query_name: impl Into<String>,
6184        handler: F,
6185    ) where
6186        S: Clone + Send + Sync + 'static,
6187        F: Fn(QueryContext, Arc<S>, AvroValue) -> Fut + Send + Sync + 'static,
6188        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6189    {
6190        let handler = Arc::new(handler);
6191        let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
6192            let state = state.downcast::<S>().map_err(|_| {
6193                "registered query state type does not match the replayed workflow state".to_string()
6194            })?;
6195            Ok(Box::pin(handler(ctx, state, args)))
6196        });
6197
6198        self.queries
6199            .entry(workflow_type.into())
6200            .or_default()
6201            .insert(
6202                query_name.into(),
6203                RegisteredQuery::Replayed {
6204                    state_type: TypeId::of::<S>(),
6205                    handler: erased_handler,
6206                },
6207            );
6208    }
6209
6210    /// Register a synchronous workflow update handler.
6211    pub fn register_update<F, Fut>(
6212        &mut self,
6213        workflow_type: impl Into<String>,
6214        update_name: impl Into<String>,
6215        handler: F,
6216    ) where
6217        F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
6218        Fut: Future<Output = Result<Value>> + Send + 'static,
6219    {
6220        let handler = Arc::new(handler);
6221        self.updates
6222            .entry(workflow_type.into())
6223            .or_default()
6224            .insert(
6225                update_name.into(),
6226                Arc::new(move |ctx, args| {
6227                    let handler = Arc::clone(&handler);
6228                    Box::pin(async move {
6229                        let result = handler(ctx, args.into_json()?).await?;
6230                        AvroValue::from_serialize(&result)
6231                    })
6232                }),
6233            );
6234    }
6235
6236    /// Register an update handler on the lossless fixed Avro Value surface.
6237    pub fn register_update_avro_value<F, Fut>(
6238        &mut self,
6239        workflow_type: impl Into<String>,
6240        update_name: impl Into<String>,
6241        handler: F,
6242    ) where
6243        F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
6244        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6245    {
6246        self.updates
6247            .entry(workflow_type.into())
6248            .or_default()
6249            .insert(
6250                update_name.into(),
6251                Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
6252            );
6253    }
6254
6255    pub async fn register(&self) -> Result<RegisterWorkerResponse> {
6256        let mut command_contracts = serde_json::Map::new();
6257        for workflow_type in self.workflows.keys() {
6258            let mut queries = self
6259                .queries
6260                .get(workflow_type)
6261                .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
6262                .unwrap_or_default();
6263            queries.sort();
6264            let mut updates = self
6265                .updates
6266                .get(workflow_type)
6267                .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
6268                .unwrap_or_default();
6269            updates.sort();
6270            command_contracts.insert(
6271                workflow_type.clone(),
6272                json!({
6273                    "queries": queries,
6274                    "query_contracts": [],
6275                    "signals": [],
6276                    "signal_contracts": [],
6277                    "updates": updates,
6278                    "update_contracts": [],
6279                    "update_validators": [],
6280                }),
6281            );
6282        }
6283
6284        self.client
6285            .register_worker_with_command_contracts(
6286                &self.worker_id,
6287                &self.task_queue,
6288                self.workflows.keys().cloned().collect(),
6289                self.activities.keys().cloned().collect(),
6290                self.max_concurrent_workflow_tasks,
6291                self.max_concurrent_activity_tasks,
6292                [
6293                    Some(CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY.to_string()),
6294                    Some(DURABLE_SELECTION_CAPABILITY.to_string()),
6295                    Some(MEMO_UPSERTS_CAPABILITY.to_string()),
6296                    Some(TYPED_SEARCH_ATTRIBUTES_CAPABILITY.to_string()),
6297                    (!self.queries.is_empty()).then(|| QUERY_TASKS_CAPABILITY.to_string()),
6298                    (!self.updates.is_empty()).then(|| WORKFLOW_UPDATES_CAPABILITY.to_string()),
6299                    worker_protocol_supports_message_streams(WORKER_PROTOCOL_VERSION)
6300                        .then(|| MESSAGE_STREAMS_CAPABILITY.to_string()),
6301                ]
6302                .into_iter()
6303                .flatten()
6304                .collect(),
6305                Value::Object(command_contracts),
6306            )
6307            .await
6308    }
6309
6310    /// Run until shutdown or a terminal worker error occurs.
6311    ///
6312    /// Empty long-poll expirations do not stop the worker. Retryable poll and
6313    /// heartbeat failures use [`WorkerRetryPolicy`] independently, while
6314    /// authentication, protocol, and other non-retryable failures are returned.
6315    pub async fn run(&self) -> Result<()> {
6316        self.run_until(std::future::pending::<()>()).await
6317    }
6318
6319    /// Run until `shutdown` resolves or a terminal worker error occurs.
6320    ///
6321    /// This has the same liveness and terminal-error contract as [`Worker::run`].
6322    pub async fn run_until<F>(&self, shutdown: F) -> Result<()>
6323    where
6324        F: Future<Output = ()>,
6325    {
6326        let registration = self.register().await?;
6327        if !registration.registered {
6328            return Err(Error::WorkerLoop(format!(
6329                "worker registration for {:?} was not accepted",
6330                self.worker_id
6331            )));
6332        }
6333        let registered_worker_id = registration.worker_id.clone();
6334        let primary = self.run_registered_until(shutdown, registration).await;
6335        let deregistration = self
6336            .client
6337            .deregister_worker_registration(&registered_worker_id)
6338            .await;
6339
6340        match (primary, deregistration) {
6341            (Ok(()), Ok(_)) => Ok(()),
6342            (Ok(()), Err(deregistration)) => Err(deregistration),
6343            (Err(primary), Ok(_)) => Err(primary),
6344            (Err(primary), Err(deregistration)) => Err(Error::WorkerShutdown {
6345                primary: Box::new(primary),
6346                deregistration: Box::new(deregistration),
6347            }),
6348        }
6349    }
6350
6351    async fn run_registered_until<F>(
6352        &self,
6353        shutdown: F,
6354        registration: RegisterWorkerResponse,
6355    ) -> Result<()>
6356    where
6357        F: Future<Output = ()>,
6358    {
6359        let heartbeat_interval = Duration::from_secs(
6360            registration
6361                .heartbeat_interval_seconds
6362                .unwrap_or(self.heartbeat_interval.as_secs().max(1)),
6363        );
6364        // The first heartbeat is immediate. Subsequent heartbeats are scheduled
6365        // from the completion of the preceding attempt, including its bounded
6366        // retries. A fixed-epoch interval can leave an already-due tick queued
6367        // while an acknowledgement is slow, producing a catch-up heartbeat as
6368        // soon as that request completes.
6369        let heartbeat = tokio::time::sleep(Duration::ZERO);
6370        tokio::pin!(heartbeat);
6371        tokio::pin!(shutdown);
6372        let stop = Arc::new(AtomicBool::new(false));
6373        // Poll responses may already have leased server-side work by the time
6374        // they become ready, so each poller owns its responses through
6375        // completion or failure instead of racing raw polls in this select.
6376        let mut workflow_poller = (!self.workflows.is_empty()).then(|| {
6377            let worker = self.clone();
6378            let stop = Arc::clone(&stop);
6379            tokio::spawn(async move { worker.poll_workflows_until_stopped(stop).await })
6380        });
6381        let mut activity_poller = (!self.activities.is_empty()).then(|| {
6382            let worker = self.clone();
6383            let stop = Arc::clone(&stop);
6384            tokio::spawn(async move { worker.poll_activities_until_stopped(stop).await })
6385        });
6386        let mut query_poller = (!self.queries.is_empty()).then(|| {
6387            let worker = self.clone();
6388            let stop = Arc::clone(&stop);
6389            tokio::spawn(async move { worker.poll_queries_until_stopped(stop).await })
6390        });
6391
6392        loop {
6393            tokio::select! {
6394                _ = &mut shutdown => {
6395                    stop.store(true, Ordering::SeqCst);
6396                    break;
6397                }
6398                _ = &mut heartbeat => {
6399                    let result = self.retry_worker_operation(|| {
6400                        self.client.heartbeat_worker(
6401                            &self.worker_id,
6402                            self.max_concurrent_workflow_tasks,
6403                            self.max_concurrent_activity_tasks,
6404                        )
6405                    }).await;
6406                    heartbeat
6407                        .as_mut()
6408                        .reset(tokio::time::Instant::now() + heartbeat_interval);
6409                    match result {
6410                        Ok(acknowledgement) => {
6411                            if let Some(observer) = &self.heartbeat_observer {
6412                                observer(&WorkerHeartbeatObservation {
6413                                    worker_id: self.worker_id.clone(),
6414                                    task_queue: self.task_queue.clone(),
6415                                    acknowledged_at_unix_millis: SystemTime::now()
6416                                        .duration_since(UNIX_EPOCH)
6417                                        .unwrap_or_default()
6418                                        .as_millis()
6419                                        .min(u64::MAX as u128)
6420                                        as u64,
6421                                    acknowledgement,
6422                                });
6423                            }
6424                        }
6425                        Err(error) => {
6426                            stop.store(true, Ordering::SeqCst);
6427                            join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await?;
6428                            return Err(error);
6429                        }
6430                    }
6431                }
6432                result = OptionFuture::from(workflow_poller.as_mut()), if workflow_poller.is_some() => {
6433                    workflow_poller = None;
6434                    let stopped_by_server = stop.load(Ordering::SeqCst);
6435                    stop.store(true, Ordering::SeqCst);
6436                    let poller_result = optional_poller_result("workflow", result);
6437                    let join_result =
6438                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6439                    poller_result?;
6440                    join_result?;
6441                    if stopped_by_server {
6442                        return Ok(());
6443                    }
6444                    return Err(Error::WorkerLoop(
6445                        "workflow poller stopped unexpectedly".to_string(),
6446                    ));
6447                }
6448                result = OptionFuture::from(activity_poller.as_mut()), if activity_poller.is_some() => {
6449                    activity_poller = None;
6450                    let stopped_by_server = stop.load(Ordering::SeqCst);
6451                    stop.store(true, Ordering::SeqCst);
6452                    let poller_result = optional_poller_result("activity", result);
6453                    let join_result =
6454                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6455                    poller_result?;
6456                    join_result?;
6457                    if stopped_by_server {
6458                        return Ok(());
6459                    }
6460                    return Err(Error::WorkerLoop(
6461                        "activity poller stopped unexpectedly".to_string(),
6462                    ));
6463                }
6464                result = OptionFuture::from(query_poller.as_mut()), if query_poller.is_some() => {
6465                    query_poller = None;
6466                    let stopped_by_server = stop.load(Ordering::SeqCst);
6467                    stop.store(true, Ordering::SeqCst);
6468                    let poller_result = optional_poller_result("query", result);
6469                    let join_result =
6470                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6471                    poller_result?;
6472                    join_result?;
6473                    if stopped_by_server {
6474                        return Ok(());
6475                    }
6476                    return Err(Error::WorkerLoop(
6477                        "query poller stopped unexpectedly".to_string(),
6478                    ));
6479                }
6480            }
6481        }
6482
6483        join_pollers(
6484            workflow_poller.take(),
6485            activity_poller.take(),
6486            query_poller.take(),
6487        )
6488        .await
6489    }
6490
6491    /// Poll and settle at most one task from each enabled task family.
6492    ///
6493    /// A workflow may reach its server-enforced run deadline while this worker
6494    /// holds a task. When the completion endpoint authoritatively rejects that
6495    /// selected task and run with `recorded=false`, `reason=run_timed_out`, and
6496    /// terminal `run_status=failed`, the workflow tick is considered settled:
6497    /// the late command was not recorded and cannot replace the terminal run.
6498    /// Every other completion rejection remains an error. This worker-level
6499    /// race handling is distinct from [`WorkflowResultOptions::timeout`], which
6500    /// only bounds how long a client waits for a result.
6501    ///
6502    /// Direct callers of [`Client::complete_workflow_task`] continue to receive
6503    /// the original [`Error::Http`] status and response body.
6504    pub async fn run_once(&self) -> Result<usize> {
6505        let mut handled = 0;
6506        match self.poll_workflow_once().await? {
6507            ManagedPollOutcome::Handled => handled += 1,
6508            ManagedPollOutcome::Stop => return Ok(handled),
6509            ManagedPollOutcome::Idle => {}
6510        }
6511        match self.poll_activity_once().await? {
6512            ManagedPollOutcome::Handled => handled += 1,
6513            ManagedPollOutcome::Stop => return Ok(handled),
6514            ManagedPollOutcome::Idle => {}
6515        }
6516        if !self.queries.is_empty() {
6517            match self.poll_query_once().await? {
6518                ManagedPollOutcome::Handled => handled += 1,
6519                ManagedPollOutcome::Stop => return Ok(handled),
6520                ManagedPollOutcome::Idle => {}
6521            }
6522        }
6523        Ok(handled)
6524    }
6525
6526    async fn poll_workflow_once(&self) -> Result<ManagedPollOutcome> {
6527        let poll_request_id = unique_request_id("rust-workflow-poll");
6528        let response = self
6529            .retry_worker_operation(|| {
6530                self.client.poll_workflow_task_response_with_request_id(
6531                    &self.worker_id,
6532                    &self.task_queue,
6533                    self.poll_timeout,
6534                    &poll_request_id,
6535                    0,
6536                )
6537            })
6538            .await?;
6539        if response.outcome().should_stop() {
6540            return Ok(ManagedPollOutcome::Stop);
6541        }
6542        let memo_updates_supported =
6543            runtime_supports_workflow_memo_updates(response.server_capabilities.as_ref());
6544        let Some(task) = response.task else {
6545            return Ok(ManagedPollOutcome::Idle);
6546        };
6547
6548        let task_id = task.task_id.clone();
6549        let attempt = task.workflow_task_attempt;
6550        let run_id = task.run_id.clone();
6551        let lease_owner = task
6552            .lease_owner
6553            .clone()
6554            .unwrap_or_else(|| self.worker_id.clone());
6555
6556        match self.execute_workflow_task_decision(task) {
6557            Ok(decision)
6558                if commands_use_workflow_memo_updates(&decision.commands)
6559                    && !memo_updates_supported =>
6560            {
6561                self.client
6562                    .fail_workflow_task(
6563                        &task_id,
6564                        &lease_owner,
6565                        attempt,
6566                        Error::WorkflowMemoUpdatesUnavailable.to_string(),
6567                    )
6568                    .await?;
6569            }
6570            Ok(decision) if decision.commands.is_empty() => {
6571                // A replay can consume a recorded pending durable command
6572                // without producing a new command. The standalone protocol
6573                // acknowledges that state through the typed waiting outcome;
6574                // an empty completion is rejected by servers that require at
6575                // least one executable command.
6576                self.client
6577                    .fail_workflow_task_with_type(
6578                        &task_id,
6579                        &lease_owner,
6580                        attempt,
6581                        WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE,
6582                        WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE,
6583                    )
6584                    .await?;
6585            }
6586            Ok(decision) => {
6587                let completion = self
6588                    .client
6589                    .complete_workflow_task_with_message_streams(
6590                        &task_id,
6591                        &lease_owner,
6592                        attempt,
6593                        decision.commands,
6594                        decision.message_stream_cursors,
6595                        decision.message_stream_waits,
6596                    )
6597                    .await;
6598                if let Err(error) = completion {
6599                    if !workflow_task_completion_is_terminal_timeout(
6600                        &error,
6601                        &task_id,
6602                        attempt,
6603                        run_id.as_deref(),
6604                    ) {
6605                        return Err(error);
6606                    }
6607                }
6608            }
6609            Err(error) => {
6610                self.client
6611                    .fail_workflow_task(&task_id, &lease_owner, attempt, error.to_string())
6612                    .await?;
6613            }
6614        }
6615
6616        Ok(ManagedPollOutcome::Handled)
6617    }
6618
6619    async fn poll_workflows_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
6620        while !stop.load(Ordering::SeqCst) {
6621            if self.poll_workflow_once().await? == ManagedPollOutcome::Stop {
6622                stop.store(true, Ordering::SeqCst);
6623                break;
6624            }
6625        }
6626
6627        Ok(())
6628    }
6629
6630    async fn poll_activity_once(&self) -> Result<ManagedPollOutcome> {
6631        let poll_request_id = unique_request_id("rust-activity-poll");
6632        let response = self
6633            .retry_worker_operation(|| {
6634                self.client.poll_activity_task_response_with_request_id(
6635                    &self.worker_id,
6636                    &self.task_queue,
6637                    self.poll_timeout,
6638                    &poll_request_id,
6639                    0,
6640                )
6641            })
6642            .await?;
6643        if response.outcome().should_stop() {
6644            return Ok(ManagedPollOutcome::Stop);
6645        }
6646        let Some(task) = response.task else {
6647            return Ok(ManagedPollOutcome::Idle);
6648        };
6649
6650        let task_id = task.task_id.clone();
6651        let attempt_id = task
6652            .activity_attempt_id
6653            .clone()
6654            .or(task.attempt_id.clone())
6655            .unwrap_or_default();
6656        let lease_owner = task
6657            .lease_owner
6658            .clone()
6659            .unwrap_or_else(|| self.worker_id.clone());
6660        let codec = task.payload_codec.clone();
6661        let result = self.execute_activity_task(task).await;
6662        match result {
6663            Ok(value) => {
6664                let completion = self
6665                    .client
6666                    .complete_activity_task(&task_id, &attempt_id, &lease_owner, value, &codec)
6667                    .await;
6668                if let Err(error) = completion {
6669                    if !activity_task_rejection_is_final(&error) {
6670                        return Err(error);
6671                    }
6672                }
6673            }
6674            Err(error) => {
6675                let failure = self
6676                    .client
6677                    .fail_activity_task(
6678                        &task_id,
6679                        &attempt_id,
6680                        &lease_owner,
6681                        error.to_string(),
6682                        false,
6683                    )
6684                    .await;
6685                if let Err(error) = failure {
6686                    if !activity_task_rejection_is_final(&error) {
6687                        return Err(error);
6688                    }
6689                }
6690            }
6691        }
6692
6693        Ok(ManagedPollOutcome::Handled)
6694    }
6695
6696    async fn poll_activities_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
6697        while !stop.load(Ordering::SeqCst) {
6698            if self.poll_activity_once().await? == ManagedPollOutcome::Stop {
6699                stop.store(true, Ordering::SeqCst);
6700                break;
6701            }
6702        }
6703
6704        Ok(())
6705    }
6706
6707    async fn poll_query_once(&self) -> Result<ManagedPollOutcome> {
6708        let poll_request_id = unique_request_id("rust-query-poll");
6709        let response = self
6710            .retry_worker_operation(|| {
6711                self.client.poll_query_task_response_with_request_id(
6712                    &self.worker_id,
6713                    &self.task_queue,
6714                    self.poll_timeout,
6715                    &poll_request_id,
6716                    0,
6717                )
6718            })
6719            .await?;
6720        if response.outcome().should_stop() {
6721            return Ok(ManagedPollOutcome::Stop);
6722        }
6723        let Some(task) = response.task else {
6724            return Ok(ManagedPollOutcome::Idle);
6725        };
6726
6727        let query_task_id = task.query_task_id.clone();
6728        let attempt = task.query_task_attempt;
6729        let lease_owner = task
6730            .lease_owner
6731            .clone()
6732            .unwrap_or_else(|| self.worker_id.clone());
6733        let codec = task.payload_codec.clone();
6734
6735        match self.execute_query_task(task).await {
6736            Ok(value) => {
6737                let result_envelope = match encode_typed_envelope(&value, &codec) {
6738                    Ok(result_envelope) => result_envelope,
6739                    Err(error) => {
6740                        let failure = self
6741                            .client
6742                            .fail_query_task(
6743                                &query_task_id,
6744                                &lease_owner,
6745                                attempt,
6746                                error.to_string(),
6747                                "query_result_encode_failed",
6748                                "QueryResultEncodeFailed",
6749                            )
6750                            .await;
6751                        if let Err(error) = failure {
6752                            if !query_task_rejection_is_final(&error) {
6753                                return Err(error);
6754                            }
6755                        }
6756                        return Ok(ManagedPollOutcome::Handled);
6757                    }
6758                };
6759
6760                if let Err(error) = self
6761                    .client
6762                    .complete_query_task_with_envelope(
6763                        &query_task_id,
6764                        &lease_owner,
6765                        attempt,
6766                        value.clone().into_json()?,
6767                        result_envelope,
6768                    )
6769                    .await
6770                {
6771                    if !query_task_rejection_is_final(&error) {
6772                        return Err(error);
6773                    }
6774                }
6775            }
6776            Err(failure) => {
6777                let result = self
6778                    .client
6779                    .fail_query_task(
6780                        &query_task_id,
6781                        &lease_owner,
6782                        attempt,
6783                        failure.message,
6784                        failure.reason,
6785                        failure.failure_type,
6786                    )
6787                    .await;
6788                if let Err(error) = result {
6789                    if !query_task_rejection_is_final(&error) {
6790                        return Err(error);
6791                    }
6792                }
6793            }
6794        }
6795
6796        Ok(ManagedPollOutcome::Handled)
6797    }
6798
6799    async fn poll_queries_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
6800        while !stop.load(Ordering::SeqCst) {
6801            if self.poll_query_once().await? == ManagedPollOutcome::Stop {
6802                stop.store(true, Ordering::SeqCst);
6803                break;
6804            }
6805        }
6806
6807        Ok(())
6808    }
6809
6810    async fn retry_worker_operation<T, F, Fut>(&self, mut operation: F) -> Result<T>
6811    where
6812        F: FnMut() -> Fut,
6813        Fut: Future<Output = Result<T>>,
6814    {
6815        let mut retries = 0;
6816
6817        loop {
6818            match operation().await {
6819                Err(error)
6820                    if worker_operation_is_retryable(&error)
6821                        && retries < self.retry_policy.max_retries =>
6822                {
6823                    retries += 1;
6824                    tokio::time::sleep(worker_retry_delay(self.retry_policy, retries)).await;
6825                }
6826                result => return result,
6827            }
6828        }
6829    }
6830
6831    async fn execute_query_task(
6832        &self,
6833        mut task: QueryTask,
6834    ) -> std::result::Result<AvroValue, QueryTaskExecutionFailure> {
6835        validate_query_task_payloads(&task).map_err(|error| {
6836            QueryTaskExecutionFailure::new(
6837                "query_payload_decode_failed",
6838                error.to_string(),
6839                "QueryPayloadDecodeFailed",
6840            )
6841        })?;
6842
6843        if !self.workflows.contains_key(&task.workflow_type) {
6844            return Err(QueryTaskExecutionFailure::new(
6845                "query_workflow_type_not_registered",
6846                format!("no workflow registered for type {:?}", task.workflow_type),
6847                "WorkflowTypeNotRegistered",
6848            ));
6849        }
6850
6851        let Some(handlers) = self.queries.get(&task.workflow_type) else {
6852            return Err(QueryTaskExecutionFailure::new(
6853                "query_handler_unavailable",
6854                format!(
6855                    "query handlers are unavailable for workflow type {:?}",
6856                    task.workflow_type
6857                ),
6858                "QueryHandlerUnavailable",
6859            ));
6860        };
6861        let Some(query) = handlers.get(&task.query_name) else {
6862            return Err(QueryTaskExecutionFailure::new(
6863                "rejected_unknown_query",
6864                format!("unknown query {:?}", task.query_name),
6865                "QueryFailed",
6866            ));
6867        };
6868
6869        let args = decode_task_avro_arguments(task.query_arguments.as_ref(), &task.payload_codec)
6870            .map_err(|error| {
6871            QueryTaskExecutionFailure::new(
6872                "query_payload_decode_failed",
6873                format!("cannot decode query arguments: {error}"),
6874                "QueryPayloadDecodeFailed",
6875            )
6876        })?;
6877        let workflow_input_typed =
6878            decode_task_avro_arguments(task.workflow_arguments.as_ref(), &task.payload_codec)
6879                .map_err(|error| {
6880                    QueryTaskExecutionFailure::new(
6881                        "query_workflow_state_unavailable",
6882                        format!("cannot decode workflow start input: {error}"),
6883                        "QueryWorkflowStateUnavailable",
6884                    )
6885                })?;
6886        let workflow_input = workflow_input_typed.clone().into_json().map_err(|error| {
6887            QueryTaskExecutionFailure::new(
6888                "query_workflow_state_unavailable",
6889                format!("cannot project workflow start input: {error}"),
6890                "QueryWorkflowStateUnavailable",
6891            )
6892        })?;
6893        hydrate_query_history_from_export(&mut task).map_err(|error| {
6894            QueryTaskExecutionFailure::new(
6895                "query_workflow_state_unavailable",
6896                format!("cannot restore query history snapshot: {error}"),
6897                "QueryWorkflowStateUnavailable",
6898            )
6899        })?;
6900        enrich_query_history_from_export(&mut task).map_err(|error| {
6901            QueryTaskExecutionFailure::new(
6902                "query_workflow_state_unavailable",
6903                format!("cannot restore compact query history payloads: {error}"),
6904                "QueryWorkflowStateUnavailable",
6905            )
6906        })?;
6907        let signal_events = query_signal_events(&task).map_err(|error| {
6908            QueryTaskExecutionFailure::new(
6909                "query_workflow_state_unavailable",
6910                format!("cannot decode committed workflow signals: {error}"),
6911                "QueryWorkflowStateUnavailable",
6912            )
6913        })?;
6914        let history_events = Arc::new(std::mem::take(&mut task.history_events));
6915        let context = QueryContext {
6916            workflow_id: task.workflow_id,
6917            run_id: task.run_id,
6918            workflow_type: task.workflow_type.clone(),
6919            run_status: task.run_status,
6920            workflow_input,
6921            workflow_input_avro_value: workflow_input_typed.clone(),
6922            history_events: Arc::clone(&history_events),
6923            signal_events: Arc::new(signal_events),
6924        };
6925
6926        let future = match query {
6927            RegisteredQuery::Snapshot(handler) => handler(context, args),
6928            RegisteredQuery::Replayed {
6929                state_type,
6930                handler,
6931            } => {
6932                let workflow = self
6933                    .workflows
6934                    .get(&task.workflow_type)
6935                    .expect("workflow registration was checked above");
6936                if workflow.state_type != Some(*state_type) {
6937                    return Err(QueryTaskExecutionFailure::new(
6938                        "query_workflow_state_unavailable",
6939                        "replayed query state type does not match its workflow registration",
6940                        "QueryWorkflowStateUnavailable",
6941                    ));
6942                }
6943                let replay = workflow.replay.as_ref().ok_or_else(|| {
6944                    QueryTaskExecutionFailure::new(
6945                        "query_workflow_state_unavailable",
6946                        format!(
6947                            "workflow type {:?} is not registered for instance-state replay",
6948                            task.workflow_type
6949                        ),
6950                        "QueryWorkflowStateUnavailable",
6951                    )
6952                })?;
6953                let workflow_state = Arc::new(Mutex::new(
6954                    WorkflowState::new_with_identity(
6955                        history_events.as_ref().clone(),
6956                        context.workflow_id.clone(),
6957                        context.run_id.clone(),
6958                        self.task_queue.clone(),
6959                        task.payload_codec,
6960                        None,
6961                    )
6962                    .map_err(|error| {
6963                        QueryTaskExecutionFailure::new(
6964                            "query_workflow_state_unavailable",
6965                            format!("workflow replay failed before query: {error}"),
6966                            "QueryWorkflowStateUnavailable",
6967                        )
6968                    })?,
6969                ));
6970                let workflow_context = WorkflowContext {
6971                    state: workflow_state,
6972                };
6973                let mut invocation = replay(workflow_context.clone(), workflow_input_typed.clone());
6974                let mut cx = TaskContext::from_waker(noop_waker_ref());
6975                match invocation.future.as_mut().poll(&mut cx) {
6976                    Poll::Ready(Ok(_)) => {
6977                        workflow_context
6978                            .ensure_history_consumed()
6979                            .map_err(|error| {
6980                                QueryTaskExecutionFailure::new(
6981                                    "query_workflow_state_unavailable",
6982                                    format!("workflow replay failed before query: {error}"),
6983                                    "QueryWorkflowStateUnavailable",
6984                                )
6985                            })?;
6986                    }
6987                    Poll::Ready(Err(error)) => {
6988                        return Err(QueryTaskExecutionFailure::new(
6989                            "query_workflow_state_unavailable",
6990                            format!("workflow replay failed before query: {error}"),
6991                            "QueryWorkflowStateUnavailable",
6992                        ));
6993                    }
6994                    Poll::Pending => {
6995                        let commands = workflow_context.take_commands().map_err(|error| {
6996                            QueryTaskExecutionFailure::new(
6997                                "query_workflow_state_unavailable",
6998                                format!("workflow replay failed before query: {error}"),
6999                                "QueryWorkflowStateUnavailable",
7000                            )
7001                        })?;
7002                        if commands.is_empty()
7003                            && !workflow_context
7004                                .matched_recorded_pending()
7005                                .map_err(|error| {
7006                                    QueryTaskExecutionFailure::new(
7007                                        "query_workflow_state_unavailable",
7008                                        format!("workflow replay failed before query: {error}"),
7009                                        "QueryWorkflowStateUnavailable",
7010                                    )
7011                                })?
7012                        {
7013                            return Err(QueryTaskExecutionFailure::new(
7014                                "query_workflow_state_unavailable",
7015                                "workflow replay yielded without a durable command",
7016                                "QueryWorkflowStateUnavailable",
7017                            ));
7018                        }
7019                    }
7020                }
7021                let state = (invocation.snapshot)().map_err(|error| {
7022                    QueryTaskExecutionFailure::new(
7023                        "query_workflow_state_unavailable",
7024                        format!("cannot snapshot replayed workflow state: {error}"),
7025                        "QueryWorkflowStateUnavailable",
7026                    )
7027                })?;
7028                handler(context, state, args).map_err(|message| {
7029                    QueryTaskExecutionFailure::new(
7030                        "query_workflow_state_unavailable",
7031                        message,
7032                        "QueryWorkflowStateUnavailable",
7033                    )
7034                })?
7035            }
7036        };
7037
7038        future.await.map_err(|error| {
7039            QueryTaskExecutionFailure::new("query_rejected", error.to_string(), "QueryFailed")
7040        })
7041    }
7042
7043    #[cfg(test)]
7044    fn execute_workflow_task(&self, task: WorkflowTask) -> Result<Vec<Value>> {
7045        Ok(self.execute_workflow_task_decision(task)?.commands)
7046    }
7047
7048    fn execute_workflow_task_decision(&self, task: WorkflowTask) -> Result<WorkflowTaskDecision> {
7049        validate_workflow_task_payloads(&task)?;
7050
7051        if let Some(update_id) = task
7052            .workflow_update_id
7053            .as_deref()
7054            .filter(|update_id| !update_id.is_empty())
7055        {
7056            return self
7057                .execute_update_task(&task, update_id)
7058                .map(WorkflowTaskDecision::without_message_streams);
7059        }
7060
7061        let workflow = self
7062            .workflows
7063            .get(&task.workflow_type)
7064            .ok_or_else(|| Error::WorkflowNotRegistered(task.workflow_type.clone()))?;
7065        let input = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
7066        let resume_signal = decode_resume_signal(&task)?;
7067        let history_budget = WorkflowHistoryBudget {
7068            event_count: task
7069                .total_history_events
7070                .unwrap_or_else(|| u64::try_from(task.history_events.len()).unwrap_or(u64::MAX)),
7071            size_bytes: task.history_size_bytes,
7072            continue_as_new_recommended: task.continue_as_new_recommended.unwrap_or(false),
7073            pressure: task.history_budget_pressure.clone(),
7074        };
7075        let workflow_command_identity = task
7076            .workflow_command_id
7077            .clone()
7078            .filter(|identity| !identity.is_empty())
7079            .unwrap_or_default();
7080        let mut workflow_state = WorkflowState::new_with_identity(
7081            task.history_events,
7082            task.workflow_id,
7083            task.run_id,
7084            self.task_queue.clone(),
7085            task.payload_codec.clone(),
7086            resume_signal,
7087        )?;
7088        workflow_state.history_budget = history_budget;
7089        workflow_state.workflow_command_identity = workflow_command_identity;
7090        workflow_state.cancel_requested = task.cancel_requested;
7091        let state = Arc::new(Mutex::new(workflow_state));
7092        let ctx = WorkflowContext { state };
7093        let mut future = (workflow.execute)(ctx.clone(), input);
7094        let mut cx = TaskContext::from_waker(noop_waker_ref());
7095
7096        match future.as_mut().poll(&mut cx) {
7097            Poll::Ready(Ok(result)) => {
7098                ctx.ensure_history_consumed()?;
7099                let result = encode_typed_envelope(&result, &task.payload_codec)?;
7100                let mut commands = ctx.take_commands()?;
7101                commands.push(json!({
7102                    "type": "complete_workflow",
7103                    "result": result
7104                }));
7105                self.message_stream_decision(&ctx, commands)
7106            }
7107            Poll::Ready(Err(error)) => {
7108                if let Error::ContinueAsNew(request) = error {
7109                    let mut commands = ctx.take_commands()?;
7110                    if let Some(command) = ctx.continue_as_new_command(request)? {
7111                        commands.push(command);
7112                    }
7113                    ctx.ensure_history_consumed()?;
7114                    return self.message_stream_decision(&ctx, commands);
7115                }
7116                if workflow_task_integrity_error(&error) {
7117                    // Replay and protocol failures describe the workflow-task
7118                    // decision itself. Preserve their specific failure reason
7119                    // instead of replacing it with the derivative fact that
7120                    // recorded commands remain unconsumed.
7121                    return Err(error);
7122                }
7123                // A handler error must not hide a committed durable command that
7124                // upgraded workflow code no longer consumes.
7125                ctx.ensure_history_consumed()?;
7126                let mut commands = ctx.take_commands()?;
7127                commands.push(workflow_failure_command(&error));
7128                self.message_stream_decision(&ctx, commands)
7129            }
7130            Poll::Pending => {
7131                let commands = ctx.take_commands()?;
7132                if commands.is_empty() && !ctx.matched_recorded_pending()? {
7133                    Err(Error::WorkflowYieldedWithoutCommand)
7134                } else {
7135                    self.message_stream_decision(&ctx, commands)
7136                }
7137            }
7138        }
7139    }
7140
7141    fn message_stream_decision(
7142        &self,
7143        ctx: &WorkflowContext,
7144        commands: Vec<Value>,
7145    ) -> Result<WorkflowTaskDecision> {
7146        let (message_stream_cursors, message_stream_waits) = ctx.message_stream_metadata()?;
7147        Ok(WorkflowTaskDecision {
7148            commands,
7149            message_stream_cursors,
7150            message_stream_waits,
7151        })
7152    }
7153
7154    fn execute_update_task(&self, task: &WorkflowTask, update_id: &str) -> Result<Vec<Value>> {
7155        if !self.workflows.contains_key(&task.workflow_type) {
7156            return Err(Error::WorkflowNotRegistered(task.workflow_type.clone()));
7157        }
7158
7159        let accepted = task.history_events.iter().rev().find_map(|event| {
7160            (event.event_type == "UpdateAccepted"
7161                && event.payload.get("update_id").and_then(Value::as_str) == Some(update_id))
7162            .then_some(&event.payload)
7163        });
7164        let update_name = accepted
7165            .and_then(|payload| payload.get("update_name"))
7166            .and_then(Value::as_str)
7167            .or(task.update_name.as_deref())
7168            .unwrap_or_default();
7169        let Some(handler) = self
7170            .updates
7171            .get(&task.workflow_type)
7172            .and_then(|handlers| handlers.get(update_name))
7173        else {
7174            return Ok(vec![json!({
7175                "type": "fail_update",
7176                "update_id": update_id,
7177                "message": format!(
7178                    "no update handler is registered for {}.{update_name}",
7179                    task.workflow_type
7180                ),
7181                "exception_type": "UnknownUpdate",
7182                "non_retryable": true,
7183            })]);
7184        };
7185        let arguments = accepted
7186            .and_then(|payload| payload.get("arguments"))
7187            .or(task.arguments.as_ref());
7188        let arguments = decode_task_avro_arguments(arguments, &task.payload_codec)?;
7189        let context = QueryContext {
7190            workflow_id: task.workflow_id.clone(),
7191            run_id: task.run_id.clone(),
7192            workflow_type: task.workflow_type.clone(),
7193            run_status: Some("running".to_string()),
7194            workflow_input: Value::Null,
7195            workflow_input_avro_value: AvroValue::Null,
7196            history_events: Arc::new(task.history_events.clone()),
7197            signal_events: Arc::new(Vec::new()),
7198        };
7199        let mut future = handler(context, arguments);
7200        let mut cx = TaskContext::from_waker(noop_waker_ref());
7201
7202        match future.as_mut().poll(&mut cx) {
7203            Poll::Ready(Ok(result)) => Ok(vec![json!({
7204                "type": "complete_update",
7205                "update_id": update_id,
7206                "result": encode_typed_envelope(&result, &task.payload_codec)?,
7207            })]),
7208            Poll::Ready(Err(error)) => Ok(vec![json!({
7209                "type": "fail_update",
7210                "update_id": update_id,
7211                "message": error.to_string(),
7212                "exception_type": "UpdateFailed",
7213                "non_retryable": true,
7214            })]),
7215            Poll::Pending => Err(Error::WorkflowYieldedWithoutCommand),
7216        }
7217    }
7218
7219    async fn execute_activity_task(&self, task: ActivityTask) -> Result<AvroValue> {
7220        validate_activity_task_payloads(&task)?;
7221
7222        let handler = self
7223            .activities
7224            .get(&task.activity_type)
7225            .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?;
7226        let args = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
7227        let attempt_id = task
7228            .activity_attempt_id
7229            .clone()
7230            .or(task.attempt_id.clone())
7231            .unwrap_or_default();
7232        let lease_owner = task
7233            .lease_owner
7234            .clone()
7235            .unwrap_or_else(|| self.worker_id.clone());
7236        let ctx = ActivityContext {
7237            client: self.client.clone(),
7238            task_id: task.task_id,
7239            activity_attempt_id: attempt_id,
7240            lease_owner,
7241            activity_type: task.activity_type,
7242            attempt_number: task.attempt_number,
7243            task_queue: self.task_queue.clone(),
7244            worker_id: self.worker_id.clone(),
7245        };
7246
7247        handler(ctx, args).await
7248    }
7249}
7250
7251fn poller_result(
7252    kind: &str,
7253    result: std::result::Result<Result<()>, tokio::task::JoinError>,
7254) -> Result<()> {
7255    match result {
7256        Ok(result) => result,
7257        Err(error) => Err(Error::WorkerLoop(format!(
7258            "{kind} poller join error: {error}"
7259        ))),
7260    }
7261}
7262
7263fn optional_poller_result(
7264    kind: &str,
7265    result: Option<std::result::Result<Result<()>, tokio::task::JoinError>>,
7266) -> Result<()> {
7267    match result {
7268        Some(result) => poller_result(kind, result),
7269        None => Ok(()),
7270    }
7271}
7272
7273async fn join_pollers(
7274    workflow_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7275    activity_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7276    query_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7277) -> Result<()> {
7278    let mut first_error = None;
7279
7280    if let Some(handle) = workflow_poller {
7281        if let Err(error) = poller_result("workflow", handle.await) {
7282            first_error.get_or_insert(error);
7283        }
7284    }
7285
7286    if let Some(handle) = activity_poller {
7287        if let Err(error) = poller_result("activity", handle.await) {
7288            first_error.get_or_insert(error);
7289        }
7290    }
7291
7292    if let Some(handle) = query_poller {
7293        if let Err(error) = poller_result("query", handle.await) {
7294            first_error.get_or_insert(error);
7295        }
7296    }
7297
7298    if let Some(error) = first_error {
7299        Err(error)
7300    } else {
7301        Ok(())
7302    }
7303}
7304
7305fn default_worker_id() -> String {
7306    let millis = SystemTime::now()
7307        .duration_since(UNIX_EPOCH)
7308        .unwrap_or_default()
7309        .as_millis();
7310    format!("rust-worker-{}-{millis}", std::process::id())
7311}
7312
7313fn percent_encode_path_segment(segment: &str) -> String {
7314    const HEX: &[u8; 16] = b"0123456789ABCDEF";
7315    let mut encoded = String::with_capacity(segment.len());
7316
7317    for byte in segment.bytes() {
7318        if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') {
7319            encoded.push(char::from(byte));
7320        } else {
7321            encoded.push('%');
7322            encoded.push(char::from(HEX[(byte >> 4) as usize]));
7323            encoded.push(char::from(HEX[(byte & 0x0f) as usize]));
7324        }
7325    }
7326
7327    encoded
7328}
7329
7330fn unique_request_id(prefix: &str) -> String {
7331    let nanos = SystemTime::now()
7332        .duration_since(UNIX_EPOCH)
7333        .unwrap_or_default()
7334        .as_nanos();
7335    format!("{prefix}-{}-{nanos}", std::process::id())
7336}
7337
7338#[derive(Debug)]
7339struct QueryTaskExecutionFailure {
7340    reason: String,
7341    message: String,
7342    failure_type: String,
7343}
7344
7345impl QueryTaskExecutionFailure {
7346    fn new(
7347        reason: impl Into<String>,
7348        message: impl Into<String>,
7349        failure_type: impl Into<String>,
7350    ) -> Self {
7351        Self {
7352            reason: reason.into(),
7353            message: message.into(),
7354            failure_type: failure_type.into(),
7355        }
7356    }
7357}
7358
7359/// Typed local state owned by one deterministic workflow invocation.
7360///
7361/// Use [`WorkflowInstance::update`] for the same state transitions during
7362/// ordinary execution and replay. A replayed query receives a detached
7363/// immutable `Arc<S>` rather than this mutation-capable handle.
7364#[derive(Clone, Debug)]
7365pub struct WorkflowInstance<S> {
7366    state: Arc<Mutex<S>>,
7367}
7368
7369impl<S> WorkflowInstance<S> {
7370    fn new(state: S) -> Self {
7371        Self {
7372            state: Arc::new(Mutex::new(state)),
7373        }
7374    }
7375
7376    /// Read the current workflow-instance state without changing it.
7377    pub fn read<R>(&self, reader: impl FnOnce(&S) -> R) -> Result<R> {
7378        let state = self
7379            .state
7380            .lock()
7381            .map_err(|_| Error::WorkflowStatePoisoned)?;
7382        Ok(reader(&state))
7383    }
7384
7385    /// Apply one deterministic workflow-instance state transition.
7386    pub fn update<R>(&self, transition: impl FnOnce(&mut S) -> R) -> Result<R> {
7387        let mut state = self
7388            .state
7389            .lock()
7390            .map_err(|_| Error::WorkflowStatePoisoned)?;
7391        Ok(transition(&mut state))
7392    }
7393}
7394
7395impl<S: Clone> WorkflowInstance<S> {
7396    fn snapshot(&self) -> Result<S> {
7397        self.read(Clone::clone)
7398    }
7399}
7400
7401#[derive(Clone, Debug, PartialEq)]
7402pub struct MessageStreamMessage {
7403    pub stream_name: String,
7404    pub message_id: String,
7405    pub position: u64,
7406    pub arguments: Vec<AvroValue>,
7407}
7408
7409#[derive(Clone, Debug)]
7410pub struct MessageStream {
7411    ctx: WorkflowContext,
7412    name: String,
7413}
7414
7415impl MessageStream {
7416    /// Wait for one message, then return a bounded currently-available batch.
7417    pub async fn receive(&self, max_items: usize) -> Result<Vec<MessageStreamMessage>> {
7418        if !(1..=MESSAGE_STREAM_MAX_BATCH).contains(&max_items) {
7419            return Err(Error::Codec(format!(
7420                "message stream max_items must be between 1 and {MESSAGE_STREAM_MAX_BATCH}"
7421            )));
7422        }
7423        loop {
7424            if let Some(batch) = self.ctx.take_message_stream_batch(&self.name, max_items)? {
7425                return Ok(batch);
7426            }
7427
7428            self.ctx.record_message_stream_wait(&self.name)?;
7429            let replay_wait_sequence = self.ctx.next_message_stream_wait_sequence()?;
7430            let arguments = self.ctx.wait_runtime_signal(MESSAGE_STREAM_SIGNAL).await?;
7431            self.ctx.buffer_message_stream_delivery(arguments)?;
7432            if let Some(sequence) = replay_wait_sequence {
7433                self.ctx.buffer_message_stream_history_for_wait(sequence)?;
7434            }
7435        }
7436    }
7437
7438    pub async fn receive_one(&self) -> Result<MessageStreamMessage> {
7439        self.receive(1)
7440            .await?
7441            .into_iter()
7442            .next()
7443            .ok_or_else(|| Error::Codec("message stream resumed without a message".to_string()))
7444    }
7445}
7446
7447#[derive(Clone, Debug)]
7448pub struct WorkflowContext {
7449    state: Arc<Mutex<WorkflowState>>,
7450}
7451
7452fn valid_memo_key(key: &str) -> bool {
7453    let numeric_candidate = key.strip_prefix('-').unwrap_or(key);
7454
7455    !key.is_empty()
7456        && key.len() <= 64
7457        && (numeric_candidate.is_empty()
7458            || !numeric_candidate.bytes().all(|byte| byte.is_ascii_digit()))
7459        && key
7460            .bytes()
7461            .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b':' | b'-'))
7462}
7463
7464fn avro_encoded_size(value: &AvroValue) -> Result<usize> {
7465    BASE64
7466        .decode(encode_avro_value(value)?.blob)
7467        .map(|bytes| bytes.len())
7468        .map_err(|error| Error::Codec(format!("memo Avro encoding was not strict base64: {error}")))
7469}
7470
7471fn canonical_memo_entries(value: AvroValue, require_entries: bool) -> Result<AvroValue> {
7472    let AvroValue::Map(entries) = value else {
7473        return Err(Error::InvalidMemoUpdate(
7474            "entries must serialize to an Avro string-keyed map".to_string(),
7475        ));
7476    };
7477    if require_entries && entries.is_empty() {
7478        return Err(Error::InvalidMemoUpdate(
7479            "at least one entry is required".to_string(),
7480        ));
7481    }
7482    if entries.len() > MAX_MEMO_ENTRIES {
7483        return Err(Error::InvalidMemoUpdate(format!(
7484            "at most {MAX_MEMO_ENTRIES} entries are allowed"
7485        )));
7486    }
7487
7488    for (key, value) in &entries {
7489        if !valid_memo_key(&key) {
7490            return Err(Error::InvalidMemoUpdate(
7491                "keys must match ^(?!-?[0-9]+$)[A-Za-z0-9_.:-]{1,64}$".to_string(),
7492            ));
7493        }
7494        if avro_encoded_size(value)? > MAX_MEMO_VALUE_SIZE_BYTES {
7495            return Err(Error::InvalidMemoUpdate(format!(
7496                "value {key:?} exceeds the {MAX_MEMO_VALUE_SIZE_BYTES}-byte limit"
7497            )));
7498        }
7499    }
7500
7501    let value = AvroValue::Map(entries);
7502    if avro_encoded_size(&value)? > MAX_MEMO_TOTAL_SIZE_BYTES {
7503        return Err(Error::InvalidMemoUpdate(format!(
7504            "update exceeds the {MAX_MEMO_TOTAL_SIZE_BYTES}-byte total limit"
7505        )));
7506    }
7507    Ok(value)
7508}
7509
7510fn decode_memo_history_map(envelope: &Value, require_entries: bool) -> Result<AvroValue> {
7511    let object = envelope.as_object().ok_or_else(|| {
7512        Error::InvalidMemoUpdate(
7513            "history field must use the public {codec, blob} payload envelope".to_string(),
7514        )
7515    })?;
7516    if object.len() != 2 || !object.contains_key("codec") || !object.contains_key("blob") {
7517        return Err(Error::InvalidMemoUpdate(
7518            "history field must use exactly the public {codec, blob} payload envelope".to_string(),
7519        ));
7520    }
7521
7522    canonical_memo_entries(
7523        decode_wire_avro_value(envelope, DEFAULT_CODEC)?,
7524        require_entries,
7525    )
7526}
7527
7528impl WorkflowContext {
7529    pub fn message_stream(&self, name: impl Into<String>) -> Result<MessageStream> {
7530        let name = name.into();
7531        if name.is_empty()
7532            || name.len() > 128
7533            || !name.bytes().all(|byte| {
7534                byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b':' | b'-')
7535            })
7536        {
7537            return Err(Error::Codec(
7538                "message stream names must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens"
7539                    .to_string(),
7540            ));
7541        }
7542        Ok(MessageStream {
7543            ctx: self.clone(),
7544            name,
7545        })
7546    }
7547
7548    fn record_message_stream_wait(&self, name: &str) -> Result<()> {
7549        let mut state = self
7550            .state
7551            .lock()
7552            .map_err(|_| Error::WorkflowStatePoisoned)?;
7553        let position = state.message_stream_cursors.get(name).copied().unwrap_or(0);
7554        state
7555            .message_stream_waits
7556            .insert(name.to_string(), position);
7557        Ok(())
7558    }
7559
7560    fn buffer_message_stream(&self, message: MessageStreamMessage) -> Result<()> {
7561        let mut state = self
7562            .state
7563            .lock()
7564            .map_err(|_| Error::WorkflowStatePoisoned)?;
7565        let cursor = state
7566            .message_stream_cursors
7567            .get(&message.stream_name)
7568            .copied()
7569            .unwrap_or(0);
7570        if message.position <= cursor {
7571            return Ok(());
7572        }
7573        let pending = state
7574            .message_stream_messages
7575            .entry(message.stream_name.clone())
7576            .or_default();
7577        if pending.iter().any(|candidate| {
7578            candidate.position == message.position || candidate.message_id == message.message_id
7579        }) {
7580            return Ok(());
7581        }
7582        pending.push(message);
7583        pending.sort_by_key(|candidate| candidate.position);
7584        Ok(())
7585    }
7586
7587    fn buffer_message_stream_delivery(&self, arguments: Vec<Value>) -> Result<Option<String>> {
7588        if let Some(delivery) = decode_message_stream_delivery(arguments)? {
7589            match delivery {
7590                MessageStreamDelivery::Message(message) => {
7591                    let stream_name = message.stream_name.clone();
7592                    self.buffer_message_stream(message)?;
7593                    return Ok(Some(stream_name));
7594                }
7595                MessageStreamDelivery::Cursor {
7596                    stream_name,
7597                    through_position,
7598                } => self.apply_message_stream_cursor(&stream_name, through_position)?,
7599            }
7600        }
7601        Ok(None)
7602    }
7603
7604    fn next_message_stream_wait_sequence(&self) -> Result<Option<u64>> {
7605        let state = self
7606            .state
7607            .lock()
7608            .map_err(|_| Error::WorkflowStatePoisoned)?;
7609        Ok(match state.recorded_commands.get(state.command_cursor) {
7610            Some(RecordedCommand::SignalWait {
7611                sequence,
7612                signal_name,
7613                ..
7614            }) if signal_name == MESSAGE_STREAM_SIGNAL => Some(*sequence),
7615            _ => None,
7616        })
7617    }
7618
7619    fn buffer_message_stream_history_for_wait(&self, wait_sequence: u64) -> Result<()> {
7620        let (history, payload_codec) = {
7621            let state = self
7622                .state
7623                .lock()
7624                .map_err(|_| Error::WorkflowStatePoisoned)?;
7625            (
7626                Arc::clone(&state.history_events),
7627                state.payload_codec.clone(),
7628            )
7629        };
7630
7631        let Some(opened_index) = history.iter().position(|event| {
7632            event.event_type == "SignalWaitOpened"
7633                && durable_event_sequence(event) == Some(wait_sequence)
7634                && event.payload.get("signal_name").and_then(Value::as_str)
7635                    == Some(MESSAGE_STREAM_SIGNAL)
7636        }) else {
7637            return Ok(());
7638        };
7639        let boundary_index = history
7640            .iter()
7641            .enumerate()
7642            .skip(opened_index + 1)
7643            .find_map(|(index, event)| {
7644                (durable_event_sequence(event).is_some_and(|sequence| sequence > wait_sequence)
7645                    && is_authored_command_open_event(event))
7646                .then_some(index)
7647            })
7648            .unwrap_or(history.len());
7649
7650        for event in history[opened_index + 1..boundary_index]
7651            .iter()
7652            .filter(|event| {
7653                event.event_type == "SignalReceived"
7654                    && event.payload.get("signal_name").and_then(Value::as_str)
7655                        == Some(MESSAGE_STREAM_SIGNAL)
7656            })
7657        {
7658            let arguments = decode_signal_event_arguments(event, &payload_codec)?
7659                .into_iter()
7660                .map(AvroValue::into_json)
7661                .collect::<Result<Vec<_>>>()?;
7662            self.buffer_message_stream_delivery(arguments)?;
7663        }
7664        Ok(())
7665    }
7666
7667    fn apply_message_stream_cursor(&self, name: &str, through_position: u64) -> Result<()> {
7668        let mut state = self
7669            .state
7670            .lock()
7671            .map_err(|_| Error::WorkflowStatePoisoned)?;
7672        let cursor = state
7673            .message_stream_cursors
7674            .entry(name.to_string())
7675            .or_default();
7676        *cursor = (*cursor).max(through_position);
7677        if let Some(pending) = state.message_stream_messages.get_mut(name) {
7678            pending.retain(|message| message.position > through_position);
7679        }
7680        Ok(())
7681    }
7682
7683    fn take_message_stream_batch(
7684        &self,
7685        name: &str,
7686        max_items: usize,
7687    ) -> Result<Option<Vec<MessageStreamMessage>>> {
7688        let mut state = self
7689            .state
7690            .lock()
7691            .map_err(|_| Error::WorkflowStatePoisoned)?;
7692        let cursor = state.message_stream_cursors.get(name).copied().unwrap_or(0);
7693        let pending = state
7694            .message_stream_messages
7695            .entry(name.to_string())
7696            .or_default();
7697        let count = contiguous_message_stream_count(pending, cursor, max_items);
7698        if count == 0 {
7699            return Ok(None);
7700        }
7701        let batch = pending.drain(..count).collect::<Vec<_>>();
7702        let position = batch.last().map(|message| message.position).unwrap_or(0);
7703        state
7704            .message_stream_cursors
7705            .insert(name.to_string(), position);
7706        state.message_stream_waits.remove(name);
7707        Ok(Some(batch))
7708    }
7709
7710    fn message_stream_metadata(&self) -> Result<(Vec<Value>, Vec<Value>)> {
7711        let state = self
7712            .state
7713            .lock()
7714            .map_err(|_| Error::WorkflowStatePoisoned)?;
7715        let mut cursors = state.message_stream_cursors.iter().collect::<Vec<_>>();
7716        cursors.sort_by_key(|(name, _)| *name);
7717        let mut waits = state.message_stream_waits.iter().collect::<Vec<_>>();
7718        waits.sort_by_key(|(name, _)| *name);
7719        Ok((
7720            cursors
7721                .into_iter()
7722                .map(|(name, position)| json!({"stream_name": name, "through_position": position}))
7723                .collect(),
7724            waits
7725                .into_iter()
7726                .map(|(name, position)| json!({"stream_name": name, "after_position": position}))
7727                .collect(),
7728        ))
7729    }
7730    /// Identity of the parent workflow currently being replayed.
7731    pub fn workflow_identity(&self) -> Result<WorkflowIdentity> {
7732        let state = self
7733            .state
7734            .lock()
7735            .map_err(|_| Error::WorkflowStatePoisoned)?;
7736        Ok(WorkflowIdentity {
7737            workflow_id: state.workflow_id.clone(),
7738            run_id: state.run_id.clone(),
7739        })
7740    }
7741
7742    /// Return the server-published history budget for this workflow task.
7743    pub fn history_budget(&self) -> Result<WorkflowHistoryBudget> {
7744        let state = self
7745            .state
7746            .lock()
7747            .map_err(|_| Error::WorkflowStatePoisoned)?;
7748        Ok(state.history_budget.clone())
7749    }
7750
7751    /// Continue this workflow instance as a fresh run with replacement arguments.
7752    ///
7753    /// Return this value directly from the workflow handler. The worker converts
7754    /// it to the terminal protocol command only after replay has consumed every
7755    /// recorded durable command.
7756    pub fn continue_as_new<T: Serialize>(&self, args: T) -> Result<Value> {
7757        self.continue_as_new_with_options(ContinueAsNewOptions::new(), args)
7758    }
7759
7760    /// Continue as new with optional workflow-type and task-queue overrides.
7761    pub fn continue_as_new_with_options<T: Serialize>(
7762        &self,
7763        options: ContinueAsNewOptions,
7764        args: T,
7765    ) -> Result<Value> {
7766        options.validate()?;
7767        Err(Error::ContinueAsNew(ContinueAsNewRequest {
7768            arguments: normalize_avro_arguments(AvroValue::from_serialize(&args)?),
7769            options,
7770        }))
7771    }
7772
7773    pub fn activity<T: Serialize>(
7774        &self,
7775        activity_type: impl Into<String>,
7776        args: T,
7777    ) -> ActivityCall {
7778        self.activity_with_options(activity_type, ActivityOptions::new(), args)
7779    }
7780
7781    pub fn activity_on_queue<T, Q>(
7782        &self,
7783        activity_type: impl Into<String>,
7784        task_queue: Option<Q>,
7785        args: T,
7786    ) -> ActivityCall
7787    where
7788        T: Serialize,
7789        Q: Into<String>,
7790    {
7791        let mut options = ActivityOptions::new();
7792        options.task_queue = task_queue.map(Into::into);
7793        self.activity_with_options(activity_type, options, args)
7794    }
7795
7796    /// Schedule one durable activity with retry, routing, and timeout options.
7797    ///
7798    /// Options are validated before the command is emitted. Once the command is
7799    /// recorded, replay consumes the same activity lifecycle at this command
7800    /// position and never emits a duplicate schedule.
7801    ///
7802    /// ```no_run
7803    /// # use durable_workflow::{json, ActivityOptions, ActivityRetryPolicy, Error, Result, WorkflowContext};
7804    /// # use std::time::Duration;
7805    /// # async fn run(ctx: WorkflowContext) -> Result<durable_workflow::Value> {
7806    /// let result = ctx
7807    ///     .activity_with_options(
7808    ///         "charge-card",
7809    ///         ActivityOptions::new()
7810    ///             .task_queue("payments")
7811    ///             .retry_policy(
7812    ///                 ActivityRetryPolicy::new(4).exponential_backoff(
7813    ///                     Duration::from_secs(1),
7814    ///                     2,
7815    ///                     Some(Duration::from_secs(30)),
7816    ///                 ),
7817    ///             )
7818    ///             .start_to_close_timeout(Duration::from_secs(60))
7819    ///             .schedule_to_close_timeout(Duration::from_secs(180))
7820    ///             .heartbeat_timeout(Duration::from_secs(15)),
7821    ///         json!([{"order_id": "order-42"}]),
7822    ///     )
7823    ///     .await;
7824    /// match result {
7825    ///     Err(Error::ActivityFailed(failure)) => Ok(json!({
7826    ///         "reason": failure.reason,
7827    ///         "timeout_kind": failure.timeout_kind,
7828    ///     })),
7829    ///     other => other,
7830    /// }
7831    /// # }
7832    /// ```
7833    pub fn activity_with_options<T: Serialize>(
7834        &self,
7835        activity_type: impl Into<String>,
7836        options: ActivityOptions,
7837        args: T,
7838    ) -> ActivityCall {
7839        ActivityCall {
7840            ctx: self.clone(),
7841            activity_type: activity_type.into(),
7842            options,
7843            args: Some(AvroValue::from_serialize(&args)),
7844            scheduled: false,
7845            parallel_group_path: Vec::new(),
7846        }
7847    }
7848
7849    pub async fn activity_avro_value<T: Serialize>(
7850        &self,
7851        activity_type: impl Into<String>,
7852        args: T,
7853    ) -> Result<AvroValue> {
7854        let mut call = self.activity(activity_type, args);
7855        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
7856    }
7857
7858    pub async fn activity_avro_value_with_options<T: Serialize>(
7859        &self,
7860        activity_type: impl Into<String>,
7861        options: ActivityOptions,
7862        args: T,
7863    ) -> Result<AvroValue> {
7864        let mut call = self.activity_with_options(activity_type, options, args);
7865        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
7866    }
7867
7868    /// Schedule an activity with a Serde request and decode its Serde result.
7869    pub async fn activity_typed<I, O>(&self, activity_type: impl Into<String>, args: I) -> Result<O>
7870    where
7871        I: Serialize,
7872        O: DeserializeOwned,
7873    {
7874        self.activity_typed_with_options(activity_type, ActivityOptions::new(), args)
7875            .await
7876    }
7877
7878    /// Schedule an activity with options and decode its result into `O`.
7879    ///
7880    /// Both directions use the fixed Avro Value codec. In particular, this
7881    /// method does not deserialize the JSON-safe inspection projection returned
7882    /// by the dynamic [`ActivityCall`] future.
7883    pub async fn activity_typed_with_options<I, O>(
7884        &self,
7885        activity_type: impl Into<String>,
7886        options: ActivityOptions,
7887        args: I,
7888    ) -> Result<O>
7889    where
7890        I: Serialize,
7891        O: DeserializeOwned,
7892    {
7893        let activity_type = activity_type.into();
7894        let encoded = AvroValue::from_serialize(&args).map_err(|error| {
7895            handler_type_error::<I>(
7896                HandlerKind::Activity,
7897                &activity_type,
7898                HandlerValueKind::Input,
7899                error.to_string(),
7900            )
7901        });
7902        let mut call = ActivityCall {
7903            ctx: self.clone(),
7904            activity_type: activity_type.clone(),
7905            options,
7906            args: Some(encoded),
7907            scheduled: false,
7908            parallel_group_path: Vec::new(),
7909        };
7910        let result = std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await?;
7911        decode_handler_result(result, HandlerKind::Activity, &activity_type)
7912    }
7913
7914    /// Schedule and join a deterministic activity/child/timer group.
7915    ///
7916    /// Nested groups retain their input shape. Every durable leaf is scheduled
7917    /// before this future yields, results are assembled by declaration order,
7918    /// and a failure returns [`Error::ParallelFailed`] with typed cause,
7919    /// declaration path, stable group metadata, and completed siblings.
7920    pub fn parallel(&self, operations: Vec<ParallelOperation>) -> ParallelCall {
7921        ParallelCall::new(self.clone(), operations)
7922    }
7923
7924    /// Alias for [`WorkflowContext::parallel`].
7925    pub fn join(&self, operations: Vec<ParallelOperation>) -> ParallelCall {
7926        self.parallel(operations)
7927    }
7928
7929    /// Lossless fixed-Avro variant of [`WorkflowContext::parallel`].
7930    pub async fn parallel_avro_value(
7931        &self,
7932        operations: Vec<ParallelOperation>,
7933    ) -> Result<Vec<ParallelAvroResult>> {
7934        let mut call = self.parallel(operations);
7935        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
7936    }
7937
7938    /// Start every durable operation and resume from the one winner persisted
7939    /// by Server. Non-winning operations continue and remain addressable.
7940    pub fn select(&self, operations: Vec<ParallelOperation>) -> SelectCall {
7941        let operations = operations
7942            .into_iter()
7943            .enumerate()
7944            .map(|(index, operation)| (SelectionKey::Index(index), operation))
7945            .collect();
7946        SelectCall::new(self.clone(), operations)
7947    }
7948
7949    /// Named-key variant of [`WorkflowContext::select`].
7950    pub fn select_keyed<K>(&self, operations: Vec<(K, ParallelOperation)>) -> SelectCall
7951    where
7952        K: Into<SelectionKey>,
7953    {
7954        SelectCall::new(
7955            self.clone(),
7956            operations
7957                .into_iter()
7958                .map(|(key, operation)| (key.into(), operation))
7959                .collect(),
7960        )
7961    }
7962
7963    /// Create a workflow-local deterministic compensation registry.
7964    pub fn saga(&self) -> Saga {
7965        Saga::new(self.clone())
7966    }
7967
7968    /// Whether the current workflow task carries a cooperative cancel request.
7969    pub fn is_cancellation_requested(&self) -> Result<bool> {
7970        let state = self
7971            .state
7972            .lock()
7973            .map_err(|_| Error::WorkflowStatePoisoned)?;
7974        Ok(state.cancel_requested)
7975    }
7976
7977    /// Raise a typed cooperative cancellation at an author-controlled point.
7978    ///
7979    /// Passing this result to [`Saga::finish`] compensates already registered
7980    /// forward steps before the cancellation remains the initiating outcome.
7981    pub fn throw_if_cancellation_requested(&self) -> Result<()> {
7982        if self.is_cancellation_requested()? {
7983            return Err(Error::WorkflowCancellationRequested(
7984                WorkflowCancellationRequested,
7985            ));
7986        }
7987        Ok(())
7988    }
7989
7990    pub fn wait_signal(&self, signal_name: impl Into<String>) -> SignalCall {
7991        SignalCall {
7992            ctx: self.clone(),
7993            signal_name: signal_name.into(),
7994            runtime_reserved_allowed: false,
7995            opened_wait: false,
7996            matched_pending: false,
7997            parallel_group_path: Vec::new(),
7998        }
7999    }
8000
8001    fn wait_runtime_signal(&self, signal_name: impl Into<String>) -> SignalCall {
8002        SignalCall {
8003            ctx: self.clone(),
8004            signal_name: signal_name.into(),
8005            runtime_reserved_allowed: true,
8006            opened_wait: false,
8007            matched_pending: false,
8008            parallel_group_path: Vec::new(),
8009        }
8010    }
8011
8012    pub async fn wait_signal_avro_value(
8013        &self,
8014        signal_name: impl Into<String>,
8015    ) -> Result<Vec<AvroValue>> {
8016        let mut call = self.wait_signal(signal_name);
8017        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8018    }
8019
8020    /// Return every committed signal argument list with the given name.
8021    ///
8022    /// This history-backed view is deterministic and is intended for
8023    /// condition predicates that must be re-evaluated after a signal while the
8024    /// workflow is blocked on [`WorkflowContext::wait_condition`].
8025    pub fn signals(&self, signal_name: &str) -> Result<Vec<Vec<Value>>> {
8026        self.signals_avro_value(signal_name)?
8027            .into_iter()
8028            .map(|arguments| {
8029                arguments
8030                    .into_iter()
8031                    .map(AvroValue::into_json)
8032                    .collect::<Result<Vec<_>>>()
8033            })
8034            .collect()
8035    }
8036
8037    /// Lossless fixed Avro Value view of committed signals with the given name.
8038    pub fn signals_avro_value(&self, signal_name: &str) -> Result<Vec<Vec<AvroValue>>> {
8039        let state = self
8040            .state
8041            .lock()
8042            .map_err(|_| Error::WorkflowStatePoisoned)?;
8043        state
8044            .history_events
8045            .iter()
8046            .filter(|event| {
8047                event.event_type == "SignalReceived"
8048                    && event.payload.get("signal_name").and_then(Value::as_str) == Some(signal_name)
8049            })
8050            .map(|event| decode_signal_event_arguments(event, &state.payload_codec))
8051            .collect()
8052    }
8053
8054    /// Return every committed update argument list with the given name.
8055    ///
8056    /// Accepted and applied records for the same update ID are de-duplicated.
8057    /// A Server task created after an update therefore replays the workflow and
8058    /// re-evaluates an open condition without application polling.
8059    pub fn updates(&self, update_name: &str) -> Result<Vec<Vec<Value>>> {
8060        self.updates_avro_value(update_name)?
8061            .into_iter()
8062            .map(|arguments| {
8063                arguments
8064                    .into_iter()
8065                    .map(AvroValue::into_json)
8066                    .collect::<Result<Vec<_>>>()
8067            })
8068            .collect()
8069    }
8070
8071    /// Lossless fixed Avro Value view of committed updates with the given name.
8072    pub fn updates_avro_value(&self, update_name: &str) -> Result<Vec<Vec<AvroValue>>> {
8073        let state = self
8074            .state
8075            .lock()
8076            .map_err(|_| Error::WorkflowStatePoisoned)?;
8077        let mut seen = Vec::new();
8078        let mut updates = Vec::new();
8079        for event in state.history_events.iter() {
8080            if !matches!(
8081                event.event_type.as_str(),
8082                "UpdateAccepted" | "UpdateApplied"
8083            ) || event.payload.get("update_name").and_then(Value::as_str) != Some(update_name)
8084                || event.payload.get("arguments").is_none()
8085            {
8086                continue;
8087            }
8088            if let Some(update_id) = event.payload.get("update_id").and_then(Value::as_str) {
8089                if seen.iter().any(|recorded| recorded == update_id) {
8090                    continue;
8091                }
8092                seen.push(update_id.to_string());
8093            }
8094            updates.push(decode_update_event_arguments(event, &state.payload_codec)?);
8095        }
8096        Ok(updates)
8097    }
8098
8099    /// Wait for a deterministic predicate to become true or for its durable
8100    /// timeout to elapse.
8101    ///
8102    /// Prefer [`wait_condition!`] for inline predicates so changes to the Rust
8103    /// predicate tokens automatically change the recorded definition
8104    /// fingerprint. Direct callers must provide an equally stable identity in
8105    /// [`ConditionWaitOptions`].
8106    pub fn wait_condition<F>(
8107        &self,
8108        options: ConditionWaitOptions,
8109        predicate: F,
8110    ) -> ConditionWaitCall
8111    where
8112        F: Fn() -> Result<bool> + Send + 'static,
8113    {
8114        ConditionWaitCall {
8115            ctx: self.clone(),
8116            options,
8117            predicate: Box::new(predicate),
8118            occurrence_id: None,
8119            opened_wait: false,
8120            parallel_group_path: Vec::new(),
8121        }
8122    }
8123
8124    /// Wait for server-backed durable time without blocking the worker executor.
8125    ///
8126    /// Polling this future emits one `start_timer` command and yields. The
8127    /// server records the deadline, so neither worker nor server restarts reset
8128    /// the wait. Replay resolves the future only from a `TimerScheduled` and
8129    /// `TimerFired` pair at the same position in the shared durable-command
8130    /// stream, with matching sequence, timer identity, and delay. Sub-second
8131    /// durations round up because protocol deadlines use whole seconds.
8132    ///
8133    /// ```no_run
8134    /// # use durable_workflow::{json, Client, Worker};
8135    /// # use std::time::Duration;
8136    /// # fn configure(client: Client) {
8137    /// let mut worker = Worker::new(client, "rust-workers");
8138    /// worker.register_workflow("delayed-greeting", |ctx, _input| async move {
8139    ///     ctx.sleep(Duration::from_secs(5)).await?;
8140    ///     Ok(json!({"status": "timer fired"}))
8141    /// });
8142    /// # }
8143    /// ```
8144    pub fn sleep(&self, duration: Duration) -> TimerCall {
8145        let delay_seconds = duration
8146            .as_secs()
8147            .checked_add(u64::from(duration.subsec_nanos() > 0));
8148        TimerCall {
8149            ctx: self.clone(),
8150            delay_seconds,
8151            scheduled: false,
8152            matched_pending: false,
8153            parallel_group_path: Vec::new(),
8154        }
8155    }
8156
8157    /// Alias for [`WorkflowContext::sleep`] for timer-oriented workflow code.
8158    pub fn start_timer(&self, duration: Duration) -> TimerCall {
8159        self.sleep(duration)
8160    }
8161
8162    /// Evaluate a non-deterministic callback once and durably record its typed value.
8163    ///
8164    /// On replay the callback is not invoked: the value is decoded from the
8165    /// sequence-matched `SideEffectRecorded` event using the workflow's payload
8166    /// codec. Use this for UUIDs, wall-clock snapshots, random values, and other
8167    /// small values that must remain fixed for the lifetime of a workflow run.
8168    pub fn side_effect<T, F>(&self, callback: F) -> Result<T>
8169    where
8170        T: Serialize + DeserializeOwned,
8171        F: FnOnce() -> T,
8172    {
8173        {
8174            let mut state = self
8175                .state
8176                .lock()
8177                .map_err(|_| Error::WorkflowStatePoisoned)?;
8178            if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8179                return match recorded {
8180                    RecordedCommand::SideEffect { sequence, value } => {
8181                        state.command_cursor += 1;
8182                        value.deserialize().map_err(|error| {
8183                            Error::NonDeterministicReplay(ReplayFailure::new(
8184                                "side_effect_type_mismatch",
8185                                Some(sequence),
8186                                Some(std::any::type_name::<T>().to_string()),
8187                                Some(error.to_string()),
8188                                "recorded side-effect value is incompatible with the requested Rust type",
8189                            ))
8190                        })
8191                    }
8192                    other => Err(command_mismatch(&other, "side effect")),
8193                };
8194            }
8195        }
8196
8197        let value = callback();
8198        let avro_value = AvroValue::from_serialize(&value)?;
8199        let mut state = self
8200            .state
8201            .lock()
8202            .map_err(|_| Error::WorkflowStatePoisoned)?;
8203        let result = encode_typed_envelope(&avro_value, &state.payload_codec)?;
8204        state.commands.push(json!({
8205            "type": "record_side_effect",
8206            "result": result,
8207        }));
8208        Ok(value)
8209    }
8210
8211    /// Record or replay a lossless fixed Avro Value side effect.
8212    pub fn side_effect_avro_value<F>(&self, callback: F) -> Result<AvroValue>
8213    where
8214        F: FnOnce() -> AvroValue,
8215    {
8216        {
8217            let mut state = self
8218                .state
8219                .lock()
8220                .map_err(|_| Error::WorkflowStatePoisoned)?;
8221            if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8222                return match recorded {
8223                    RecordedCommand::SideEffect { value, .. } => {
8224                        state.command_cursor += 1;
8225                        Ok(value)
8226                    }
8227                    other => Err(command_mismatch(&other, "side effect")),
8228                };
8229            }
8230        }
8231
8232        let value = callback();
8233        let mut state = self
8234            .state
8235            .lock()
8236            .map_err(|_| Error::WorkflowStatePoisoned)?;
8237        let result = encode_typed_envelope(&value, &state.payload_codec)?;
8238        state.commands.push(json!({
8239            "type": "record_side_effect",
8240            "result": result,
8241        }));
8242        Ok(value)
8243    }
8244
8245    /// Append output items at a deterministic workflow command boundary.
8246    ///
8247    /// Stable idempotency keys are derived from the server-provided durable
8248    /// workflow command identity, command ordinal, and item index. Replay
8249    /// consumes the recorded side effect and never emits another append.
8250    pub fn append_workflow_stream(
8251        &self,
8252        stream_name: impl Into<String>,
8253        items: &[WorkflowStreamAppendItem],
8254        max_pending_items: Option<u64>,
8255    ) -> Result<()> {
8256        if items.is_empty() {
8257            return Err(Error::Codec(
8258                "workflow_stream_items_empty: append requires at least one item".to_string(),
8259            ));
8260        }
8261        if max_pending_items == Some(0) {
8262            return Err(Error::Codec(
8263                "workflow_stream_pending_limit_invalid: max_pending_items must be positive"
8264                    .to_string(),
8265            ));
8266        }
8267        let stream_name = stream_name.into();
8268        if stream_name.is_empty() {
8269            return Err(Error::Codec(
8270                "workflow_stream_name_invalid: stream name must not be empty".to_string(),
8271            ));
8272        }
8273
8274        let mut state = self
8275            .state
8276            .lock()
8277            .map_err(|_| Error::WorkflowStatePoisoned)?;
8278        let command_ordinal = state.workflow_stream_command_counter;
8279        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8280            state.workflow_stream_command_counter += 1;
8281            return match recorded {
8282                RecordedCommand::SideEffect { .. } => {
8283                    state.command_cursor += 1;
8284                    Ok(())
8285                }
8286                other => Err(command_mismatch(&other, "workflow stream append")),
8287            };
8288        }
8289
8290        let identity = Self::workflow_stream_command_identity(&state)?.to_string();
8291        state.workflow_stream_command_counter += 1;
8292        let wire_items = items
8293            .iter()
8294            .enumerate()
8295            .map(|(item_index, item)| {
8296                item.wire_value(Some(format!(
8297                    "dw-stream:{identity}:{command_ordinal}:{item_index}"
8298                )))
8299            })
8300            .collect::<Vec<_>>();
8301        let mut directive = json!({
8302            "operation": "append",
8303            "stream_name": stream_name,
8304            "command_identity": identity,
8305            "command_ordinal": command_ordinal,
8306            "items": wire_items,
8307        });
8308        if let Some(max_pending_items) = max_pending_items {
8309            directive["max_pending_items"] = json!(max_pending_items);
8310        }
8311        let result = encode_typed_envelope(&AvroValue::Null, &state.payload_codec)?;
8312        state.commands.push(json!({
8313            "type": "record_side_effect",
8314            "result": result,
8315            "workflow_stream": directive,
8316        }));
8317        Ok(())
8318    }
8319
8320    /// Close a run-scoped output stream at a deterministic command boundary.
8321    pub fn close_workflow_stream(
8322        &self,
8323        stream_name: impl Into<String>,
8324        retention_seconds: Option<u64>,
8325    ) -> Result<()> {
8326        self.finish_workflow_stream(stream_name.into(), None, retention_seconds)
8327    }
8328
8329    /// Mark a run-scoped output stream errored at a deterministic command boundary.
8330    pub fn error_workflow_stream(
8331        &self,
8332        stream_name: impl Into<String>,
8333        error_reason: impl Into<String>,
8334        retention_seconds: Option<u64>,
8335    ) -> Result<()> {
8336        let error_reason = error_reason.into();
8337        if error_reason.is_empty() {
8338            return Err(Error::Codec(
8339                "workflow_stream_error_invalid: error reason must not be empty".to_string(),
8340            ));
8341        }
8342        self.finish_workflow_stream(stream_name.into(), Some(error_reason), retention_seconds)
8343    }
8344
8345    fn finish_workflow_stream(
8346        &self,
8347        stream_name: String,
8348        error_reason: Option<String>,
8349        retention_seconds: Option<u64>,
8350    ) -> Result<()> {
8351        if stream_name.is_empty() {
8352            return Err(Error::Codec(
8353                "workflow_stream_name_invalid: stream name must not be empty".to_string(),
8354            ));
8355        }
8356        if retention_seconds == Some(0) {
8357            return Err(Error::Codec(
8358                "workflow_stream_retention_invalid: retention_seconds must be positive".to_string(),
8359            ));
8360        }
8361        let mut state = self
8362            .state
8363            .lock()
8364            .map_err(|_| Error::WorkflowStatePoisoned)?;
8365        let command_ordinal = state.workflow_stream_command_counter;
8366        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8367            state.workflow_stream_command_counter += 1;
8368            return match recorded {
8369                RecordedCommand::SideEffect { .. } => {
8370                    state.command_cursor += 1;
8371                    Ok(())
8372                }
8373                other => Err(command_mismatch(&other, "workflow stream close")),
8374            };
8375        }
8376        let identity = Self::workflow_stream_command_identity(&state)?.to_string();
8377        state.workflow_stream_command_counter += 1;
8378        let mut directive = json!({
8379            "operation": if error_reason.is_some() { "error" } else { "close" },
8380            "stream_name": stream_name,
8381            "command_identity": identity,
8382            "command_ordinal": command_ordinal,
8383        });
8384        if let Some(error_reason) = error_reason {
8385            directive["error_reason"] = json!(error_reason);
8386        }
8387        if let Some(retention_seconds) = retention_seconds {
8388            directive["retention_seconds"] = json!(retention_seconds);
8389        }
8390        let result = encode_typed_envelope(&AvroValue::Null, &state.payload_codec)?;
8391        state.commands.push(json!({
8392            "type": "record_side_effect",
8393            "result": result,
8394            "workflow_stream": directive,
8395        }));
8396        Ok(())
8397    }
8398
8399    fn workflow_stream_command_identity(state: &WorkflowState) -> Result<&str> {
8400        let identity = state.workflow_command_identity.as_str();
8401        if identity.is_empty() {
8402            return Err(Error::MissingWorkflowCommandIdentity);
8403        }
8404        Ok(identity)
8405    }
8406
8407    /// Validate, emit, or replay a typed workflow search-attribute update.
8408    ///
8409    /// The command is non-blocking within a workflow decision, but its
8410    /// `SearchAttributesUpserted` event occupies the same deterministic command
8411    /// stream as activities, timers, conditions, and other durable operations.
8412    pub fn upsert_search_attributes(&self, update: SearchAttributeUpdate) -> Result<()> {
8413        update.validate()?;
8414        let (attributes, attribute_types) = update.into_wire_parts();
8415        let mut state = self
8416            .state
8417            .lock()
8418            .map_err(|_| Error::WorkflowStatePoisoned)?;
8419
8420        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8421            return match recorded {
8422                RecordedCommand::SearchAttributes {
8423                    sequence,
8424                    attributes: recorded_attributes,
8425                    attribute_types: recorded_attribute_types,
8426                } => {
8427                    if recorded_attributes != attributes {
8428                        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8429                            "search_attribute_value_mismatch",
8430                            Some(sequence),
8431                            Some(recorded_attributes.to_string()),
8432                            Some(attributes.to_string()),
8433                            "search-attribute values differ from the recorded durable command",
8434                        )));
8435                    }
8436                    if let RecordedSnapshotValue::Known(recorded_types) = recorded_attribute_types {
8437                        if recorded_types != attribute_types {
8438                            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8439                                "search_attribute_type_mismatch",
8440                                Some(sequence),
8441                                Some(json!(recorded_types).to_string()),
8442                                Some(json!(attribute_types).to_string()),
8443                                "search-attribute declared types differ from the recorded durable command",
8444                            )));
8445                        }
8446                    }
8447                    state.command_cursor += 1;
8448                    Ok(())
8449                }
8450                other => Err(command_mismatch(&other, "search-attribute update")),
8451            };
8452        }
8453
8454        let mut command = serde_json::Map::from_iter([
8455            ("type".to_string(), json!("upsert_search_attributes")),
8456            ("attributes".to_string(), attributes),
8457        ]);
8458        if !attribute_types.is_empty() {
8459            command.insert("attribute_types".to_string(), json!(attribute_types));
8460        }
8461        state.commands.push(Value::Object(command));
8462        Ok(())
8463    }
8464
8465    /// Record a UUIDv4 once and return the same UUID on every replay.
8466    pub fn uuid_v4(&self) -> Result<Uuid> {
8467        self.side_effect(Uuid::new_v4)
8468    }
8469
8470    /// Select the newest supported version for a change, or replay the version
8471    /// already committed for that stable change ID.
8472    pub fn get_version(
8473        &self,
8474        change_id: impl Into<String>,
8475        min_supported: i32,
8476        max_supported: i32,
8477    ) -> Result<i32> {
8478        let change_id = change_id.into();
8479        if change_id.trim().is_empty() {
8480            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8481                "version_change_id_invalid",
8482                None,
8483                Some("non-empty change ID".to_string()),
8484                Some(change_id),
8485                "version markers require a stable non-empty change ID",
8486            )));
8487        }
8488        if min_supported > max_supported {
8489            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8490                "version_range_invalid",
8491                None,
8492                Some("min_supported <= max_supported".to_string()),
8493                Some(format!("{min_supported}..={max_supported}")),
8494                "version marker supported range is invalid",
8495            )));
8496        }
8497
8498        let mut state = self
8499            .state
8500            .lock()
8501            .map_err(|_| Error::WorkflowStatePoisoned)?;
8502        if let Some((version, sequence)) = state.version_markers.get(&change_id).copied() {
8503            ensure_version_supported(&change_id, version, min_supported, max_supported, sequence)?;
8504            return Ok(version);
8505        }
8506
8507        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8508            return match recorded {
8509                RecordedCommand::VersionMarker {
8510                    sequence,
8511                    change_id: recorded_change_id,
8512                    version,
8513                    ..
8514                } => {
8515                    if recorded_change_id != change_id {
8516                        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8517                            "version_change_id_mismatch",
8518                            Some(sequence),
8519                            Some(recorded_change_id),
8520                            Some(change_id),
8521                            "recorded version marker change ID differs from current workflow code",
8522                        )));
8523                    }
8524                    ensure_version_supported(
8525                        &change_id,
8526                        version,
8527                        min_supported,
8528                        max_supported,
8529                        sequence,
8530                    )?;
8531                    state.command_cursor += 1;
8532                    state.version_markers.insert(change_id, (version, sequence));
8533                    Ok(version)
8534                }
8535                other => Err(command_mismatch(
8536                    &other,
8537                    format!("version marker:{change_id}"),
8538                )),
8539            };
8540        }
8541
8542        let version = max_supported;
8543        state.commands.push(json!({
8544            "type": "record_version_marker",
8545            "change_id": change_id,
8546            "version": version,
8547            "min_supported": min_supported,
8548            "max_supported": max_supported,
8549        }));
8550        // Sequence numbers are assigned by the server. Zero identifies a marker
8551        // selected in this uncommitted decision batch for duplicate-call checks.
8552        state.version_markers.insert(change_id, (version, 0));
8553        Ok(version)
8554    }
8555
8556    /// Record or replay the standard `-1` (legacy) / `1` (patched) marker.
8557    pub fn patched(&self, change_id: impl Into<String>) -> Result<bool> {
8558        Ok(self.get_version(change_id, -1, 1)? == 1)
8559    }
8560
8561    /// Keep a patch marker in history after the legacy branch has been removed.
8562    pub fn deprecate_patch(&self, change_id: impl Into<String>) -> Result<()> {
8563        self.get_version(change_id, -1, 1).map(|_| ())
8564    }
8565
8566    /// Merge non-indexed workflow memo metadata through durable history.
8567    ///
8568    /// Avro `null` deletes a key. The SDK encodes the complete patch in the
8569    /// public Avro payload envelope consumed by Server and Cloud runtimes.
8570    pub fn upsert_memo<T: Serialize>(&self, entries: T) -> Result<()> {
8571        let entries = canonical_memo_entries(AvroValue::from_serialize(&entries)?, true)?;
8572        let mut state = self
8573            .state
8574            .lock()
8575            .map_err(|_| Error::WorkflowStatePoisoned)?;
8576
8577        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8578            return match recorded {
8579                RecordedCommand::Memo {
8580                    sequence,
8581                    entries: recorded_entries,
8582                } => {
8583                    if recorded_entries != entries {
8584                        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8585                            "memo_update_mismatch",
8586                            Some(sequence),
8587                            Some(format!("{recorded_entries:?}")),
8588                            Some(format!("{entries:?}")),
8589                            "recorded memo entries differ from the current workflow update",
8590                        )));
8591                    }
8592                    state.command_cursor += 1;
8593                    Ok(())
8594                }
8595                other => Err(command_mismatch(&other, "memo upsert")),
8596            };
8597        }
8598
8599        let entries_envelope = encode_typed_envelope(&entries, DEFAULT_CODEC)?;
8600        state.commands.push(json!({
8601            "type": "upsert_memo",
8602            "entries": entries_envelope,
8603        }));
8604        Ok(())
8605    }
8606
8607    /// Start a named durable child on an explicit queue and await its result.
8608    ///
8609    /// The command is recorded in the parent's sequence-ordered durable command
8610    /// stream. Replay keeps a scheduled child pending without emitting another
8611    /// start, or consumes its matching terminal `ChildRun*` outcome. Successful
8612    /// values preserve the history payload codec and include both sides of the
8613    /// durable relationship; failures are returned as
8614    /// [`Error::ChildWorkflowFailed`].
8615    ///
8616    /// ```no_run
8617    /// # use durable_workflow::{json, ChildWorkflowOptions, Client, ParentClosePolicy, Worker};
8618    /// # fn configure(client: Client) {
8619    /// let mut worker = Worker::new(client, "parent-workers");
8620    /// worker.register_workflow("order-parent", |ctx, _input| async move {
8621    ///     let child = ctx
8622    ///         .start_child_workflow(
8623    ///             "fulfil-order",
8624    ///             ChildWorkflowOptions::new("fulfilment-workers")
8625    ///                 .parent_close_policy(ParentClosePolicy::RequestCancel),
8626    ///             json!([{"order_id": "order-42"}]),
8627    ///         )
8628    ///         .await?;
8629    ///     Ok(child.result)
8630    /// });
8631    /// # }
8632    /// ```
8633    pub fn start_child_workflow<T: Serialize>(
8634        &self,
8635        workflow_type: impl Into<String>,
8636        options: ChildWorkflowOptions,
8637        args: T,
8638    ) -> ChildWorkflowCall {
8639        ChildWorkflowCall {
8640            ctx: self.clone(),
8641            workflow_type: workflow_type.into(),
8642            options,
8643            args: Some(AvroValue::from_serialize(&args)),
8644            scheduled: false,
8645            matched_pending: false,
8646            parallel_group_path: Vec::new(),
8647        }
8648    }
8649
8650    pub async fn start_child_workflow_avro_value<T: Serialize>(
8651        &self,
8652        workflow_type: impl Into<String>,
8653        options: ChildWorkflowOptions,
8654        args: T,
8655    ) -> Result<ChildWorkflowAvroResult> {
8656        let mut call = self.start_child_workflow(workflow_type, options, args);
8657        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8658    }
8659
8660    fn take_commands(&self) -> Result<Vec<Value>> {
8661        let mut state = self
8662            .state
8663            .lock()
8664            .map_err(|_| Error::WorkflowStatePoisoned)?;
8665        Ok(std::mem::take(&mut state.commands))
8666    }
8667
8668    fn continue_as_new_command(&self, request: ContinueAsNewRequest) -> Result<Option<Value>> {
8669        let mut state = self
8670            .state
8671            .lock()
8672            .map_err(|_| Error::WorkflowStatePoisoned)?;
8673
8674        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8675            return Err(command_mismatch(&recorded, "continue as new"));
8676        }
8677        if state.recorded_continue_as_new_sequence.is_some() {
8678            state.continue_as_new_consumed = true;
8679            return Ok(None);
8680        }
8681
8682        let arguments = encode_typed_envelope(&request.arguments, &state.payload_codec)?;
8683        let mut command = serde_json::Map::from_iter([
8684            ("type".to_string(), json!("continue_as_new")),
8685            ("arguments".to_string(), arguments),
8686            ("queue".to_string(), json!(state.task_queue.clone())),
8687        ]);
8688        if let Some(workflow_type) = request.options.workflow_type {
8689            command.insert("workflow_type".to_string(), json!(workflow_type));
8690        }
8691        if let Some(task_queue) = request.options.task_queue {
8692            command.insert("queue".to_string(), json!(task_queue));
8693        }
8694        Ok(Some(Value::Object(command)))
8695    }
8696
8697    fn matched_recorded_pending(&self) -> Result<bool> {
8698        let state = self
8699            .state
8700            .lock()
8701            .map_err(|_| Error::WorkflowStatePoisoned)?;
8702        Ok(state.matched_recorded_pending)
8703    }
8704
8705    fn ensure_history_consumed(&self) -> Result<()> {
8706        let state = self
8707            .state
8708            .lock()
8709            .map_err(|_| Error::WorkflowStatePoisoned)?;
8710        if let Some(command) = state.recorded_commands.get(state.command_cursor) {
8711            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8712                "recorded_commands_unconsumed",
8713                Some(command.sequence()),
8714                Some(command.shape().to_string()),
8715                Some("workflow completion".to_string()),
8716                "workflow completed before consuming all recorded durable commands",
8717            )));
8718        }
8719        if let Some(sequence) = state
8720            .recorded_continue_as_new_sequence
8721            .filter(|_| !state.continue_as_new_consumed)
8722        {
8723            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8724                "recorded_continue_as_new_unconsumed",
8725                Some(sequence),
8726                Some("continue as new".to_string()),
8727                Some("workflow completion".to_string()),
8728                "workflow completed without consuming its recorded continue-as-new transition",
8729            )));
8730        }
8731        Ok(())
8732    }
8733}
8734
8735fn contiguous_message_stream_count(
8736    pending: &[MessageStreamMessage],
8737    cursor: u64,
8738    max_items: usize,
8739) -> usize {
8740    pending
8741        .iter()
8742        .take(max_items)
8743        .enumerate()
8744        .take_while(|(offset, message)| {
8745            u64::try_from(*offset)
8746                .ok()
8747                .and_then(|offset| cursor.checked_add(offset + 1))
8748                == Some(message.position)
8749        })
8750        .count()
8751}
8752
8753fn is_authored_command_open_event(event: &HistoryEvent) -> bool {
8754    matches!(
8755        event.event_type.as_str(),
8756        "ActivityScheduled"
8757            | "TimerScheduled"
8758            | "ChildWorkflowScheduled"
8759            | "SignalWaitOpened"
8760            | "ConditionWaitOpened"
8761            | "SearchAttributesUpserted"
8762            | "SideEffectRecorded"
8763            | "VersionMarkerRecorded"
8764            | "MemoUpserted"
8765            | "WorkflowContinuedAsNew"
8766    )
8767}
8768
8769#[derive(Debug)]
8770struct WorkflowState {
8771    workflow_id: Option<String>,
8772    run_id: Option<String>,
8773    task_queue: String,
8774    payload_codec: String,
8775    history_events: Arc<Vec<HistoryEvent>>,
8776    history_budget: WorkflowHistoryBudget,
8777    cancel_requested: bool,
8778    resume_signal: Option<ResumeSignal>,
8779    recorded_commands: Vec<RecordedCommand>,
8780    selection_markers: Vec<SelectionMarker>,
8781    selection_marker_cursor: usize,
8782    cancelled_selection_members: Vec<SelectionCancellation>,
8783    recorded_continue_as_new_sequence: Option<u64>,
8784    continue_as_new_consumed: bool,
8785    command_cursor: usize,
8786    condition_wait_occurrence_counter: u64,
8787    matched_recorded_pending: bool,
8788    version_markers: HashMap<String, (i32, u64)>,
8789    workflow_command_identity: String,
8790    workflow_stream_command_counter: u64,
8791    commands: Vec<Value>,
8792    message_stream_messages: HashMap<String, Vec<MessageStreamMessage>>,
8793    message_stream_cursors: HashMap<String, u64>,
8794    message_stream_waits: HashMap<String, u64>,
8795}
8796
8797impl WorkflowState {
8798    #[cfg(test)]
8799    fn new(
8800        history: Vec<HistoryEvent>,
8801        task_queue: String,
8802        payload_codec: String,
8803        resume_signal: Option<ResumeSignal>,
8804    ) -> Result<Self> {
8805        Self::new_with_identity(
8806            history,
8807            None,
8808            None,
8809            task_queue,
8810            payload_codec,
8811            resume_signal,
8812        )
8813    }
8814
8815    fn new_with_identity(
8816        history: Vec<HistoryEvent>,
8817        workflow_id: Option<String>,
8818        run_id: Option<String>,
8819        task_queue: String,
8820        payload_codec: String,
8821        resume_signal: Option<ResumeSignal>,
8822    ) -> Result<Self> {
8823        let recorded_commands = recorded_commands(
8824            &history,
8825            &payload_codec,
8826            WorkflowIdentity {
8827                workflow_id: workflow_id.clone(),
8828                run_id: run_id.clone(),
8829            },
8830        )?;
8831        let selection_markers = recorded_selection_markers(&history)?;
8832        let cancelled_selection_members = recorded_selection_cancellations(&history)?;
8833        let recorded_continue_as_new = history
8834            .iter()
8835            .filter(|event| event.event_type == "WorkflowContinuedAsNew")
8836            .collect::<Vec<_>>();
8837        if recorded_continue_as_new.len() > 1 {
8838            return Err(invalid_recorded_history(
8839                "duplicate_continue_as_new_transition",
8840                recorded_continue_as_new
8841                    .last()
8842                    .and_then(|event| durable_event_sequence(event))
8843                    .unwrap_or(0),
8844                "one WorkflowContinuedAsNew event",
8845                &format!(
8846                    "{} WorkflowContinuedAsNew events",
8847                    recorded_continue_as_new.len()
8848                ),
8849                "workflow history records one continue-as-new transition more than once",
8850            ));
8851        }
8852        let recorded_continue_as_new_sequence = recorded_continue_as_new
8853            .first()
8854            .map(|event| {
8855                durable_event_sequence(event).ok_or_else(|| {
8856                    Error::NonDeterministicReplay(ReplayFailure::new(
8857                        "continue_as_new_sequence_missing",
8858                        None,
8859                        Some("recorded transition sequence".to_string()),
8860                        Some("missing sequence".to_string()),
8861                        "WorkflowContinuedAsNew history is missing its recorded sequence",
8862                    ))
8863                })
8864            })
8865            .transpose()?;
8866        let mut message_stream_cursors = HashMap::new();
8867        for event in &history {
8868            if !matches!(
8869                event.event_type.as_str(),
8870                "SignalReceived" | "SignalApplied"
8871            ) || event.payload.get("signal_name").and_then(Value::as_str)
8872                != Some(MESSAGE_STREAM_SIGNAL)
8873            {
8874                continue;
8875            }
8876            let arguments = decode_signal_event_arguments(event, &payload_codec)?;
8877            if arguments.len() != 1 {
8878                continue;
8879            }
8880            let envelope = arguments[0].clone().into_json()?;
8881            let Some(envelope) = envelope.as_object() else {
8882                continue;
8883            };
8884            if envelope.get("schema").and_then(Value::as_str) != Some(MESSAGE_STREAM_CURSOR_SCHEMA)
8885            {
8886                continue;
8887            }
8888            let Some(stream_name) = envelope.get("stream_name").and_then(Value::as_str) else {
8889                continue;
8890            };
8891            let Some(through_position) = envelope.get("through_position").and_then(Value::as_u64)
8892            else {
8893                continue;
8894            };
8895            let cursor = message_stream_cursors
8896                .entry(stream_name.to_string())
8897                .or_insert(0);
8898            *cursor = (*cursor).max(through_position);
8899        }
8900        let event_count = u64::try_from(history.len()).unwrap_or(u64::MAX);
8901        let cancel_requested = history.iter().any(|event| {
8902            matches!(
8903                event.event_type.as_str(),
8904                "WorkflowCancellationRequested" | "WorkflowCancelRequested"
8905            )
8906        });
8907        Ok(Self {
8908            workflow_command_identity: String::new(),
8909            workflow_stream_command_counter: 0,
8910            workflow_id,
8911            run_id,
8912            task_queue,
8913            payload_codec,
8914            history_events: Arc::new(history),
8915            history_budget: WorkflowHistoryBudget {
8916                event_count,
8917                ..WorkflowHistoryBudget::default()
8918            },
8919            cancel_requested,
8920            resume_signal,
8921            recorded_commands,
8922            selection_markers,
8923            selection_marker_cursor: 0,
8924            cancelled_selection_members,
8925            recorded_continue_as_new_sequence,
8926            continue_as_new_consumed: false,
8927            command_cursor: 0,
8928            condition_wait_occurrence_counter: 0,
8929            matched_recorded_pending: false,
8930            version_markers: HashMap::new(),
8931            commands: Vec::new(),
8932            message_stream_messages: HashMap::new(),
8933            message_stream_cursors,
8934            message_stream_waits: HashMap::new(),
8935        })
8936    }
8937}
8938
8939enum MessageStreamDelivery {
8940    Message(MessageStreamMessage),
8941    Cursor {
8942        stream_name: String,
8943        through_position: u64,
8944    },
8945}
8946
8947fn decode_message_stream_delivery(arguments: Vec<Value>) -> Result<Option<MessageStreamDelivery>> {
8948    if arguments.len() != 1 {
8949        return Ok(None);
8950    }
8951    let envelope = arguments
8952        .into_iter()
8953        .next()
8954        .expect("one argument was checked");
8955    let Some(envelope) = envelope.as_object() else {
8956        return Ok(None);
8957    };
8958    let Some(stream_name) = envelope.get("stream_name").and_then(Value::as_str) else {
8959        return Ok(None);
8960    };
8961    if envelope.get("schema").and_then(Value::as_str) == Some(MESSAGE_STREAM_CURSOR_SCHEMA) {
8962        let Some(through_position) = envelope.get("through_position").and_then(Value::as_u64)
8963        else {
8964            return Ok(None);
8965        };
8966        return Ok(Some(MessageStreamDelivery::Cursor {
8967            stream_name: stream_name.to_string(),
8968            through_position,
8969        }));
8970    }
8971    if envelope.get("schema").and_then(Value::as_str) != Some(MESSAGE_STREAM_SCHEMA) {
8972        return Ok(None);
8973    }
8974    let Some(message_id) = envelope.get("message_id").and_then(Value::as_str) else {
8975        return Ok(None);
8976    };
8977    let Some(position) = envelope
8978        .get("position")
8979        .and_then(Value::as_u64)
8980        .filter(|value| *value > 0)
8981    else {
8982        return Ok(None);
8983    };
8984    let Some(payload_envelope) = envelope.get("payload_envelope") else {
8985        return Ok(None);
8986    };
8987    let Ok(payload_envelope) = serde_json::from_value::<PayloadEnvelope>(payload_envelope.clone())
8988    else {
8989        return Ok(None);
8990    };
8991    let decoded = decode_avro_value(&payload_envelope)?;
8992    let AvroValue::Array(values) = decoded else {
8993        return Ok(None);
8994    };
8995    Ok(Some(MessageStreamDelivery::Message(MessageStreamMessage {
8996        stream_name: stream_name.to_string(),
8997        message_id: message_id.to_string(),
8998        position,
8999        arguments: values,
9000    })))
9001}
9002
9003#[derive(Clone, Debug)]
9004enum RecordedCommand {
9005    Activity {
9006        sequence: u64,
9007        activity_type: Option<String>,
9008        options: Option<RecordedActivityOptions>,
9009        outcome: Option<ActivityOutcome>,
9010        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9011    },
9012    Timer {
9013        sequence: u64,
9014        delay_seconds: u64,
9015        fired: bool,
9016        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9017    },
9018    ChildWorkflow {
9019        sequence: u64,
9020        workflow_type: Option<String>,
9021        outcome: Option<ChildWorkflowOutcome>,
9022        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9023    },
9024    SignalWait {
9025        sequence: u64,
9026        signal_name: String,
9027        value: Option<Vec<AvroValue>>,
9028        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9029    },
9030    ConditionWait {
9031        sequence: u64,
9032        occurrence_id: String,
9033        condition_key: Option<String>,
9034        predicate_identity: String,
9035        timeout_seconds: Option<u64>,
9036        result: Option<ConditionWaitResult>,
9037        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9038    },
9039    SearchAttributes {
9040        sequence: u64,
9041        attributes: Value,
9042        attribute_types: RecordedSnapshotValue<BTreeMap<String, String>>,
9043    },
9044    SideEffect {
9045        sequence: u64,
9046        value: AvroValue,
9047    },
9048    VersionMarker {
9049        sequence: u64,
9050        change_id: String,
9051        version: i32,
9052    },
9053    Memo {
9054        sequence: u64,
9055        entries: AvroValue,
9056    },
9057}
9058
9059#[derive(Clone, Debug, PartialEq, Eq)]
9060struct SelectionMarker {
9061    selection_group_id: String,
9062    selection_group_base_sequence: u64,
9063    selection_group_size: usize,
9064    member_key: SelectionKey,
9065    member_index: usize,
9066    member_base_sequence: u64,
9067    member_size: usize,
9068    operation_kind: String,
9069    operation_identity: String,
9070    outcome: String,
9071    resolution_event_id: String,
9072    resolution_event_type: String,
9073}
9074
9075#[derive(Clone, Debug, PartialEq, Eq)]
9076struct SelectionCancellation {
9077    selection_group_id: String,
9078    member_key: SelectionKey,
9079    member_index: usize,
9080    member_base_sequence: u64,
9081    member_size: usize,
9082    operation_kind: String,
9083    operation_identity: String,
9084}
9085
9086fn recorded_selection_markers(events: &[HistoryEvent]) -> Result<Vec<SelectionMarker>> {
9087    let mut markers: Vec<SelectionMarker> = Vec::new();
9088    for event in events
9089        .iter()
9090        .filter(|event| event.event_type == "SelectionResolved")
9091    {
9092        let payload = &event.payload;
9093        let base_sequence = required_selection_u64(payload, "selection_group_base_sequence")?;
9094        let group_size = required_selection_usize(payload, "selection_group_size")?;
9095        let member_base_sequence = required_selection_u64(payload, "member_base_sequence")?;
9096        let member_size = required_selection_usize(payload, "member_size")?;
9097        let member_index = required_selection_usize_allow_zero(payload, "member_index")?;
9098        let group_id = payload_string(payload, "selection_group_id").ok_or_else(|| {
9099            invalid_recorded_history(
9100                "selection_marker_invalid",
9101                base_sequence,
9102                "non-empty selection_group_id",
9103                &payload.to_string(),
9104                "selection winner history is missing its durable group identity",
9105            )
9106        })?;
9107        let expected_group_id = format!("select-calls:{base_sequence}:{group_size}");
9108        if group_id != expected_group_id {
9109            return Err(invalid_recorded_history(
9110                "selection_marker_invalid",
9111                base_sequence,
9112                &expected_group_id,
9113                &group_id,
9114                "selection winner history contains an incompatible group identity",
9115            ));
9116        }
9117        let group_end = base_sequence
9118            .checked_add(u64::try_from(group_size).unwrap_or(u64::MAX))
9119            .unwrap_or(u64::MAX);
9120        let member_end = member_base_sequence
9121            .checked_add(u64::try_from(member_size).unwrap_or(u64::MAX))
9122            .unwrap_or(u64::MAX);
9123        if member_index >= group_size
9124            || member_base_sequence < base_sequence
9125            || member_end > group_end
9126        {
9127            return Err(invalid_recorded_history(
9128                "selection_marker_invalid",
9129                base_sequence,
9130                "winner member within selection group bounds",
9131                &payload.to_string(),
9132                "selection winner history contains an invalid member range",
9133            ));
9134        }
9135        let operation_kind = payload_string(payload, "operation_kind").ok_or_else(|| {
9136            invalid_recorded_history(
9137                "selection_marker_invalid",
9138                base_sequence,
9139                "selection operation kind",
9140                &payload.to_string(),
9141                "selection winner history is missing its operation kind",
9142            )
9143        })?;
9144        if !matches!(
9145            operation_kind.as_str(),
9146            "activity" | "child" | "timer" | "signal" | "condition" | "group"
9147        ) {
9148            return Err(invalid_recorded_history(
9149                "selection_marker_invalid",
9150                base_sequence,
9151                "activity, child, timer, signal, condition, or group",
9152                &operation_kind,
9153                "selection winner history contains an unsupported operation kind",
9154            ));
9155        }
9156        let operation_identity =
9157            payload_string(payload, "operation_identity").ok_or_else(|| {
9158                invalid_recorded_history(
9159                    "selection_marker_invalid",
9160                    base_sequence,
9161                    "non-empty operation identity",
9162                    &payload.to_string(),
9163                    "selection winner history is missing its durable operation identity",
9164                )
9165            })?;
9166        let outcome = payload_string(payload, "outcome").ok_or_else(|| {
9167            invalid_recorded_history(
9168                "selection_marker_invalid",
9169                base_sequence,
9170                "completed or failed selection outcome",
9171                &payload.to_string(),
9172                "selection winner history is missing its outcome",
9173            )
9174        })?;
9175        if !matches!(outcome.as_str(), "completed" | "failed") {
9176            return Err(invalid_recorded_history(
9177                "selection_marker_invalid",
9178                base_sequence,
9179                "completed or failed selection outcome",
9180                &outcome,
9181                "selection winner history contains an unsupported outcome",
9182            ));
9183        }
9184        let marker = SelectionMarker {
9185            selection_group_id: group_id,
9186            selection_group_base_sequence: base_sequence,
9187            selection_group_size: group_size,
9188            member_key: selection_key_from_value(payload.get("member_key"), base_sequence)?,
9189            member_index,
9190            member_base_sequence,
9191            member_size,
9192            operation_kind,
9193            operation_identity,
9194            outcome,
9195            resolution_event_id: payload_string(payload, "resolution_event_id").ok_or_else(
9196                || {
9197                    invalid_recorded_history(
9198                        "selection_marker_invalid",
9199                        base_sequence,
9200                        "durable resolution_event_id",
9201                        &payload.to_string(),
9202                        "selection winner history is missing its terminal event identity",
9203                    )
9204                },
9205            )?,
9206            resolution_event_type: payload_string(payload, "resolution_event_type").ok_or_else(
9207                || {
9208                    invalid_recorded_history(
9209                        "selection_marker_invalid",
9210                        base_sequence,
9211                        "durable resolution_event_type",
9212                        &payload.to_string(),
9213                        "selection winner history is missing its terminal event type",
9214                    )
9215                },
9216            )?,
9217        };
9218        if let Some(existing) = markers
9219            .iter()
9220            .find(|existing| existing.selection_group_id == marker.selection_group_id)
9221        {
9222            if existing != &marker {
9223                return Err(invalid_recorded_history(
9224                    "selection_marker_conflict",
9225                    base_sequence,
9226                    &format!("one winner for {}", marker.selection_group_id),
9227                    &payload.to_string(),
9228                    "selection history records conflicting winners for one durable group",
9229                ));
9230            }
9231            continue;
9232        }
9233        markers.push(marker);
9234    }
9235    Ok(markers)
9236}
9237
9238fn recorded_selection_cancellations(events: &[HistoryEvent]) -> Result<Vec<SelectionCancellation>> {
9239    let mut cancelled: Vec<SelectionCancellation> = Vec::new();
9240    for event in events
9241        .iter()
9242        .filter(|event| event.event_type == "SelectionOperationCancelled")
9243    {
9244        let group_id = payload_string(&event.payload, "selection_group_id").ok_or_else(|| {
9245            invalid_recorded_history(
9246                "selection_cancellation_invalid",
9247                0,
9248                "non-empty selection_group_id",
9249                &event.payload.to_string(),
9250                "selection cancellation history is missing its group identity",
9251            )
9252        })?;
9253        let member_base_sequence = required_selection_u64(&event.payload, "member_base_sequence")?;
9254        let marker = SelectionCancellation {
9255            selection_group_id: group_id,
9256            member_key: selection_key_from_value(
9257                event.payload.get("member_key"),
9258                member_base_sequence,
9259            )?,
9260            member_index: required_selection_usize_allow_zero(&event.payload, "member_index")?,
9261            member_base_sequence,
9262            member_size: required_selection_usize(&event.payload, "member_size")?,
9263            operation_kind: payload_string(&event.payload, "operation_kind").ok_or_else(|| {
9264                invalid_recorded_history(
9265                    "selection_cancellation_invalid",
9266                    member_base_sequence,
9267                    "selection operation kind",
9268                    &event.payload.to_string(),
9269                    "selection cancellation is missing its operation kind",
9270                )
9271            })?,
9272            operation_identity: payload_string(&event.payload, "operation_identity").ok_or_else(
9273                || {
9274                    invalid_recorded_history(
9275                        "selection_cancellation_invalid",
9276                        member_base_sequence,
9277                        "selection operation identity",
9278                        &event.payload.to_string(),
9279                        "selection cancellation is missing its operation identity",
9280                    )
9281                },
9282            )?,
9283        };
9284        if let Some(existing) = cancelled.iter().find(|recorded| {
9285            recorded.selection_group_id == marker.selection_group_id
9286                && recorded.member_base_sequence == marker.member_base_sequence
9287        }) {
9288            if existing != &marker {
9289                return Err(invalid_recorded_history(
9290                    "selection_cancellation_conflict",
9291                    member_base_sequence,
9292                    "one stable SelectionOperationCancelled marker",
9293                    &event.payload.to_string(),
9294                    "selection cancellation history contains conflicting member metadata",
9295                ));
9296            }
9297        } else {
9298            cancelled.push(marker);
9299        }
9300    }
9301    Ok(cancelled)
9302}
9303
9304fn required_selection_u64(payload: &Value, field: &str) -> Result<u64> {
9305    payload
9306        .get(field)
9307        .and_then(value_as_u64)
9308        .filter(|value| *value > 0)
9309        .ok_or_else(|| {
9310            invalid_recorded_history(
9311                "selection_marker_invalid",
9312                0,
9313                &format!("positive integer {field}"),
9314                &payload.to_string(),
9315                "selection history contains invalid durable identity metadata",
9316            )
9317        })
9318}
9319
9320fn required_selection_usize(payload: &Value, field: &str) -> Result<usize> {
9321    required_selection_usize_allow_zero(payload, field).and_then(|value| {
9322        if value > 0 {
9323            Ok(value)
9324        } else {
9325            Err(invalid_recorded_history(
9326                "selection_marker_invalid",
9327                0,
9328                &format!("positive integer {field}"),
9329                &payload.to_string(),
9330                "selection history contains invalid durable identity metadata",
9331            ))
9332        }
9333    })
9334}
9335
9336fn required_selection_usize_allow_zero(payload: &Value, field: &str) -> Result<usize> {
9337    payload
9338        .get(field)
9339        .and_then(value_as_u64)
9340        .and_then(|value| usize::try_from(value).ok())
9341        .ok_or_else(|| {
9342            invalid_recorded_history(
9343                "selection_marker_invalid",
9344                0,
9345                &format!("non-negative integer {field}"),
9346                &payload.to_string(),
9347                "selection history contains invalid durable identity metadata",
9348            )
9349        })
9350}
9351
9352#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9353struct RecordedActivityOptions {
9354    task_queue: RecordedSnapshotValue<Option<String>>,
9355    execution_mode: RecordedSnapshotValue<Option<String>>,
9356    retry_policy: ActivityRetrySnapshot,
9357}
9358
9359#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9360enum RecordedSnapshotValue<T> {
9361    /// Older history did not persist this field, so it cannot constrain replay.
9362    Unknown,
9363    Known(T),
9364}
9365
9366impl<T: PartialEq> RecordedSnapshotValue<T> {
9367    fn matches_current(&self, current: &Self) -> bool {
9368        match self {
9369            Self::Unknown => true,
9370            Self::Known(recorded) => matches!(current, Self::Known(value) if value == recorded),
9371        }
9372    }
9373}
9374
9375#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9376struct ActivityRetrySnapshot {
9377    snapshot_version: RecordedSnapshotValue<Option<u64>>,
9378    max_attempts: RecordedSnapshotValue<Option<u64>>,
9379    backoff_seconds: RecordedSnapshotValue<Vec<u64>>,
9380    start_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
9381    schedule_to_start_timeout: RecordedSnapshotValue<Option<u64>>,
9382    schedule_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
9383    heartbeat_timeout: RecordedSnapshotValue<Option<u64>>,
9384    non_retryable_error_types: RecordedSnapshotValue<Vec<String>>,
9385}
9386
9387impl ActivityRetrySnapshot {
9388    fn matches_current(&self, current: &Self) -> bool {
9389        self.snapshot_version
9390            .matches_current(&current.snapshot_version)
9391            && self.max_attempts.matches_current(&current.max_attempts)
9392            && self
9393                .backoff_seconds
9394                .matches_current(&current.backoff_seconds)
9395            && self
9396                .start_to_close_timeout
9397                .matches_current(&current.start_to_close_timeout)
9398            && self
9399                .schedule_to_start_timeout
9400                .matches_current(&current.schedule_to_start_timeout)
9401            && self
9402                .schedule_to_close_timeout
9403                .matches_current(&current.schedule_to_close_timeout)
9404            && self
9405                .heartbeat_timeout
9406                .matches_current(&current.heartbeat_timeout)
9407            && self
9408                .non_retryable_error_types
9409                .matches_current(&current.non_retryable_error_types)
9410    }
9411}
9412
9413fn recorded_optional_u64(
9414    object: Option<&serde_json::Map<String, Value>>,
9415    field: &str,
9416) -> RecordedSnapshotValue<Option<u64>> {
9417    match object.and_then(|object| object.get(field)) {
9418        None => RecordedSnapshotValue::Unknown,
9419        Some(Value::Null) => RecordedSnapshotValue::Known(None),
9420        Some(value) => RecordedSnapshotValue::Known(value_as_u64(value)),
9421    }
9422}
9423
9424fn recorded_optional_string(
9425    object: &serde_json::Map<String, Value>,
9426    field: &str,
9427) -> RecordedSnapshotValue<Option<String>> {
9428    match object.get(field) {
9429        None => RecordedSnapshotValue::Unknown,
9430        Some(Value::Null) => RecordedSnapshotValue::Known(None),
9431        Some(value) => RecordedSnapshotValue::Known(value.as_str().map(str::to_string)),
9432    }
9433}
9434
9435fn recorded_activity_retry_snapshot(policy: Option<&Value>) -> ActivityRetrySnapshot {
9436    let policy = policy.and_then(Value::as_object);
9437    let backoff_seconds = policy
9438        .and_then(|policy| policy.get("backoff_seconds"))
9439        .and_then(Value::as_array)
9440        .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
9441        .map_or(RecordedSnapshotValue::Unknown, RecordedSnapshotValue::Known);
9442    let mut non_retryable_error_types = Vec::new();
9443    for error_type in policy
9444        .and_then(|policy| policy.get("non_retryable_error_types"))
9445        .and_then(Value::as_array)
9446        .into_iter()
9447        .flatten()
9448        .filter_map(Value::as_str)
9449        .map(str::trim)
9450        .filter(|error_type| !error_type.is_empty())
9451    {
9452        if !non_retryable_error_types
9453            .iter()
9454            .any(|recorded| recorded == error_type)
9455        {
9456            non_retryable_error_types.push(error_type.to_string());
9457        }
9458    }
9459
9460    ActivityRetrySnapshot {
9461        snapshot_version: recorded_optional_u64(policy, "snapshot_version"),
9462        max_attempts: recorded_optional_u64(policy, "max_attempts"),
9463        backoff_seconds,
9464        start_to_close_timeout: recorded_optional_u64(policy, "start_to_close_timeout"),
9465        schedule_to_start_timeout: recorded_optional_u64(policy, "schedule_to_start_timeout"),
9466        schedule_to_close_timeout: recorded_optional_u64(policy, "schedule_to_close_timeout"),
9467        heartbeat_timeout: recorded_optional_u64(policy, "heartbeat_timeout"),
9468        non_retryable_error_types: if policy
9469            .is_some_and(|policy| policy.contains_key("non_retryable_error_types"))
9470        {
9471            RecordedSnapshotValue::Known(non_retryable_error_types)
9472        } else {
9473            RecordedSnapshotValue::Unknown
9474        },
9475    }
9476}
9477
9478fn current_activity_retry_snapshot(options: &ValidatedActivityOptions) -> ActivityRetrySnapshot {
9479    let policy = options.retry_policy.as_ref();
9480    let max_attempts = match policy.and_then(|policy| policy.get("max_attempts")) {
9481        Some(Value::Null) => None,
9482        Some(value) => value_as_u64(value),
9483        None => Some(1),
9484    };
9485    let backoff_seconds = policy
9486        .and_then(|policy| policy.get("backoff_seconds"))
9487        .and_then(Value::as_array)
9488        .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
9489        .unwrap_or_default();
9490    let non_retryable_error_types = policy
9491        .and_then(|policy| policy.get("non_retryable_error_types"))
9492        .and_then(Value::as_array)
9493        .into_iter()
9494        .flatten()
9495        .filter_map(Value::as_str)
9496        .map(str::to_string)
9497        .collect();
9498
9499    ActivityRetrySnapshot {
9500        snapshot_version: RecordedSnapshotValue::Known(Some(1)),
9501        max_attempts: RecordedSnapshotValue::Known(max_attempts),
9502        backoff_seconds: RecordedSnapshotValue::Known(backoff_seconds),
9503        start_to_close_timeout: RecordedSnapshotValue::Known(options.start_to_close_timeout),
9504        schedule_to_start_timeout: RecordedSnapshotValue::Known(options.schedule_to_start_timeout),
9505        schedule_to_close_timeout: RecordedSnapshotValue::Known(options.schedule_to_close_timeout),
9506        heartbeat_timeout: RecordedSnapshotValue::Known(options.heartbeat_timeout),
9507        non_retryable_error_types: RecordedSnapshotValue::Known(non_retryable_error_types),
9508    }
9509}
9510
9511fn activity_options_description(options: &RecordedActivityOptions) -> String {
9512    serde_json::to_string(options).unwrap_or_else(|_| format!("{options:?}"))
9513}
9514
9515impl RecordedCommand {
9516    fn sequence(&self) -> u64 {
9517        match self {
9518            Self::Activity { sequence, .. }
9519            | Self::Timer { sequence, .. }
9520            | Self::ChildWorkflow { sequence, .. }
9521            | Self::SignalWait { sequence, .. }
9522            | Self::ConditionWait { sequence, .. }
9523            | Self::SearchAttributes { sequence, .. }
9524            | Self::SideEffect { sequence, .. }
9525            | Self::VersionMarker { sequence, .. }
9526            | Self::Memo { sequence, .. } => *sequence,
9527        }
9528    }
9529
9530    fn shape(&self) -> &'static str {
9531        match self {
9532            Self::Activity { .. } => "activity",
9533            Self::Timer { .. } => "timer",
9534            Self::ChildWorkflow { .. } => "child workflow",
9535            Self::SignalWait { .. } => "signal wait",
9536            Self::ConditionWait { .. } => "condition wait",
9537            Self::SearchAttributes { .. } => "search-attribute update",
9538            Self::SideEffect { .. } => "side effect",
9539            Self::VersionMarker { .. } => "version marker",
9540            Self::Memo { .. } => "memo upsert",
9541        }
9542    }
9543}
9544
9545fn ensure_version_supported(
9546    change_id: &str,
9547    version: i32,
9548    min_supported: i32,
9549    max_supported: i32,
9550    sequence: u64,
9551) -> Result<()> {
9552    if (min_supported..=max_supported).contains(&version) {
9553        return Ok(());
9554    }
9555    Err(Error::NonDeterministicReplay(ReplayFailure::new(
9556        "version_marker_incompatible_range",
9557        (sequence != 0).then_some(sequence),
9558        Some(format!("{min_supported}..={max_supported}")),
9559        Some(format!("{change_id}:{version}")),
9560        "recorded workflow version is outside the range supported by current code",
9561    )))
9562}
9563
9564#[derive(Clone, Debug)]
9565struct ResumeSignal {
9566    signal_name: String,
9567    arguments: Vec<AvroValue>,
9568}
9569
9570const MAX_PARALLEL_OPERATIONS: usize = 1000;
9571
9572fn parallel_group_prefix(kind: &str) -> &'static str {
9573    match kind {
9574        "activity" => "parallel-activities",
9575        "child" => "parallel-children",
9576        "timer" => "parallel-timers",
9577        _ => "parallel-calls",
9578    }
9579}
9580
9581fn parallel_group_entry(
9582    base_sequence: u64,
9583    size: usize,
9584    index: usize,
9585    kind: &str,
9586) -> ParallelGroupMetadata {
9587    ParallelGroupMetadata {
9588        parallel_group_id: format!("{}:{base_sequence}:{size}", parallel_group_prefix(kind)),
9589        parallel_group_kind: kind.to_string(),
9590        parallel_group_base_sequence: base_sequence,
9591        parallel_group_size: size,
9592        parallel_group_index: index,
9593        parallel_group_mode: None,
9594        selection_member_key: None,
9595        selection_member_index: None,
9596        selection_member_base_sequence: None,
9597        selection_member_size: None,
9598        selection_member_kind: None,
9599    }
9600}
9601
9602struct SelectionMemberMetadata {
9603    key: SelectionKey,
9604    index: usize,
9605    base_sequence: u64,
9606    size: usize,
9607    kind: String,
9608}
9609
9610fn selection_group_entry(
9611    base_sequence: u64,
9612    size: usize,
9613    index: usize,
9614    kind: &str,
9615    member: &SelectionMemberMetadata,
9616) -> ParallelGroupMetadata {
9617    ParallelGroupMetadata {
9618        parallel_group_id: format!("select-calls:{base_sequence}:{size}"),
9619        parallel_group_kind: kind.to_string(),
9620        parallel_group_base_sequence: base_sequence,
9621        parallel_group_size: size,
9622        parallel_group_index: index,
9623        parallel_group_mode: Some("select".to_string()),
9624        selection_member_key: Some(member.key.clone()),
9625        selection_member_index: Some(member.index),
9626        selection_member_base_sequence: Some(member.base_sequence),
9627        selection_member_size: Some(member.size),
9628        selection_member_kind: Some(member.kind.clone()),
9629    }
9630}
9631
9632fn apply_parallel_group_path(
9633    command: &mut serde_json::Map<String, Value>,
9634    path: &[ParallelGroupMetadata],
9635) {
9636    let Some(inner) = path.last() else {
9637        return;
9638    };
9639    command.insert(
9640        "parallel_group_id".to_string(),
9641        json!(inner.parallel_group_id),
9642    );
9643    command.insert(
9644        "parallel_group_kind".to_string(),
9645        json!(inner.parallel_group_kind),
9646    );
9647    command.insert(
9648        "parallel_group_base_sequence".to_string(),
9649        json!(inner.parallel_group_base_sequence),
9650    );
9651    command.insert(
9652        "parallel_group_size".to_string(),
9653        json!(inner.parallel_group_size),
9654    );
9655    command.insert(
9656        "parallel_group_index".to_string(),
9657        json!(inner.parallel_group_index),
9658    );
9659    if let Some(mode) = &inner.parallel_group_mode {
9660        command.insert("parallel_group_mode".to_string(), json!(mode));
9661    }
9662    if let Some(key) = &inner.selection_member_key {
9663        command.insert("selection_member_key".to_string(), json!(key));
9664    }
9665    if let Some(index) = inner.selection_member_index {
9666        command.insert("selection_member_index".to_string(), json!(index));
9667    }
9668    if let Some(base_sequence) = inner.selection_member_base_sequence {
9669        command.insert(
9670            "selection_member_base_sequence".to_string(),
9671            json!(base_sequence),
9672        );
9673    }
9674    if let Some(size) = inner.selection_member_size {
9675        command.insert("selection_member_size".to_string(), json!(size));
9676    }
9677    if let Some(kind) = &inner.selection_member_kind {
9678        command.insert("selection_member_kind".to_string(), json!(kind));
9679    }
9680    command.insert("parallel_group_path".to_string(), json!(path));
9681}
9682
9683fn ensure_parallel_path_matches(
9684    sequence: u64,
9685    recorded: Option<&[ParallelGroupMetadata]>,
9686    expected: &[ParallelGroupMetadata],
9687) -> Result<()> {
9688    match (recorded, expected.is_empty()) {
9689        (None, true) => Ok(()),
9690        (Some(recorded), false) if recorded == expected => Ok(()),
9691        (None, false) => Err(invalid_recorded_history(
9692            "parallel_group_metadata_missing",
9693            sequence,
9694            &serde_json::to_string(expected).unwrap_or_default(),
9695            "<missing>",
9696            "recorded parallel member is missing its durable group path",
9697        )),
9698        (Some(recorded), true) => Err(invalid_recorded_history(
9699            "parallel_group_shape_mismatch",
9700            sequence,
9701            "sequential command",
9702            &serde_json::to_string(recorded).unwrap_or_default(),
9703            "recorded command belonged to a parallel group but current code schedules it sequentially",
9704        )),
9705        (Some(recorded), false) => Err(invalid_recorded_history(
9706            "parallel_group_shape_mismatch",
9707            sequence,
9708            &serde_json::to_string(recorded).unwrap_or_default(),
9709            &serde_json::to_string(expected).unwrap_or_default(),
9710            "recorded parallel-group identity or path changed during replay",
9711        )),
9712    }
9713}
9714
9715#[derive(Clone, Debug)]
9716enum ParallelShape {
9717    Leaf,
9718    Group(Vec<ParallelShape>),
9719}
9720
9721struct ParallelDescriptor {
9722    operation: ParallelOperation,
9723    offset: usize,
9724    member_path: Vec<usize>,
9725    group_path: Vec<ParallelGroupMetadata>,
9726}
9727
9728fn parallel_leaf_count(operations: &[ParallelOperation]) -> usize {
9729    operations
9730        .iter()
9731        .map(|operation| match operation {
9732            ParallelOperation::Group(children) => parallel_leaf_count(children),
9733            _ => 1,
9734        })
9735        .sum()
9736}
9737
9738fn parallel_operation_kind(operation: &ParallelOperation) -> Option<&'static str> {
9739    match operation {
9740        ParallelOperation::Activity { .. } => Some("activity"),
9741        ParallelOperation::ChildWorkflow { .. } => Some("child"),
9742        ParallelOperation::Timer(_) => Some("timer"),
9743        ParallelOperation::Signal(_) => Some("signal"),
9744        ParallelOperation::Condition { .. } => Some("condition"),
9745        ParallelOperation::Group(children) => parallel_group_kind(children),
9746    }
9747}
9748
9749fn parallel_group_kind(operations: &[ParallelOperation]) -> Option<&'static str> {
9750    let mut kind = None;
9751    for operation in operations {
9752        let Some(operation_kind) = parallel_operation_kind(operation) else {
9753            continue;
9754        };
9755        match kind {
9756            None => kind = Some(operation_kind),
9757            Some(current) if current == operation_kind => {}
9758            Some(_) => return Some("mixed"),
9759        }
9760    }
9761    kind
9762}
9763
9764fn validate_parallel_operations(
9765    operations: &[ParallelOperation],
9766    member_path: &mut Vec<usize>,
9767    root: bool,
9768) -> Result<()> {
9769    let leaves = parallel_leaf_count(operations);
9770    if leaves > MAX_PARALLEL_OPERATIONS {
9771        return Err(Error::InvalidParallelGroup(ParallelGroupError {
9772            reason: "fan_out_limit_exceeded",
9773            member_path: member_path.clone(),
9774            message: format!(
9775                "group contains {leaves} durable leaves; the limit is {MAX_PARALLEL_OPERATIONS}"
9776            ),
9777        }));
9778    }
9779    if !root && operations.is_empty() {
9780        return Err(Error::InvalidParallelGroup(ParallelGroupError {
9781            reason: "nested_group_empty",
9782            member_path: member_path.clone(),
9783            message: "a nested group must contain at least one durable leaf".to_string(),
9784        }));
9785    }
9786
9787    for (index, operation) in operations.iter().enumerate() {
9788        member_path.push(index);
9789        match operation {
9790            ParallelOperation::Activity {
9791                options, arguments, ..
9792            } => {
9793                options
9794                    .validate()
9795                    .map_err(|error| Error::InvalidActivityOptions(error))?;
9796                if let Err(error) = arguments {
9797                    return Err(Error::InvalidParallelGroup(ParallelGroupError {
9798                        reason: "arguments_invalid",
9799                        member_path: member_path.clone(),
9800                        message: error.to_string(),
9801                    }));
9802                }
9803            }
9804            ParallelOperation::ChildWorkflow {
9805                options, arguments, ..
9806            } => {
9807                validate_parallel_child_options(options)?;
9808                if let Err(error) = arguments {
9809                    return Err(Error::InvalidParallelGroup(ParallelGroupError {
9810                        reason: "arguments_invalid",
9811                        member_path: member_path.clone(),
9812                        message: error.to_string(),
9813                    }));
9814                }
9815            }
9816            ParallelOperation::Timer(duration)
9817                if duration.as_secs() == u64::MAX && duration.subsec_nanos() > 0 =>
9818            {
9819                return Err(Error::TimerDurationOverflow);
9820            }
9821            ParallelOperation::Timer(_) => {}
9822            ParallelOperation::Signal(signal_name) => {
9823                validate_user_signal_name(signal_name)?;
9824                if signal_name.trim().is_empty() {
9825                    return Err(Error::InvalidParallelGroup(ParallelGroupError {
9826                        reason: "signal_name_empty",
9827                        member_path: member_path.clone(),
9828                        message: "signal wait name must not be empty".to_string(),
9829                    }));
9830                }
9831            }
9832            ParallelOperation::Condition { options, .. } => {
9833                options.validate()?;
9834            }
9835            ParallelOperation::Group(children) => {
9836                validate_parallel_operations(children, member_path, false)?;
9837            }
9838        }
9839        member_path.pop();
9840    }
9841    Ok(())
9842}
9843
9844fn validate_parallel_child_options(options: &ChildWorkflowOptions) -> Result<()> {
9845    if options.task_queue.trim().is_empty() {
9846        return Err(Error::InvalidChildWorkflowOptions(
9847            "task_queue must not be empty".to_string(),
9848        ));
9849    }
9850    for (name, value) in [
9851        (
9852            "execution_timeout_seconds",
9853            options.execution_timeout_seconds,
9854        ),
9855        ("run_timeout_seconds", options.run_timeout_seconds),
9856    ] {
9857        if value == Some(0) {
9858            return Err(Error::InvalidChildWorkflowOptions(format!(
9859                "{name} must be at least 1"
9860            )));
9861        }
9862    }
9863    if options
9864        .retry_policy
9865        .as_ref()
9866        .is_some_and(|policy| policy.max_attempts == Some(0))
9867    {
9868        return Err(Error::InvalidChildWorkflowOptions(
9869            "retry_policy.max_attempts must be at least 1".to_string(),
9870        ));
9871    }
9872    Ok(())
9873}
9874
9875fn parallel_shape(operations: &[ParallelOperation]) -> ParallelShape {
9876    ParallelShape::Group(
9877        operations
9878            .iter()
9879            .map(|operation| match operation {
9880                ParallelOperation::Group(children) => parallel_shape(children),
9881                _ => ParallelShape::Leaf,
9882            })
9883            .collect(),
9884    )
9885}
9886
9887fn parallel_descriptors(
9888    operations: Vec<ParallelOperation>,
9889    base_sequence: u64,
9890) -> Result<Vec<ParallelDescriptor>> {
9891    let size = parallel_leaf_count(&operations);
9892    let kind = parallel_group_kind(&operations).unwrap_or("activity");
9893    let mut descriptors = Vec::with_capacity(size);
9894    let mut cursor = 0;
9895
9896    for (index, operation) in operations.into_iter().enumerate() {
9897        match operation {
9898            ParallelOperation::Group(children) => {
9899                let child_base = base_sequence
9900                    .checked_add(u64::try_from(cursor).unwrap_or(u64::MAX))
9901                    .ok_or(Error::TimerDurationOverflow)?;
9902                for mut descriptor in parallel_descriptors(children, child_base)? {
9903                    let outer_index = cursor + descriptor.offset;
9904                    descriptor.group_path.insert(
9905                        0,
9906                        parallel_group_entry(base_sequence, size, outer_index, kind),
9907                    );
9908                    descriptor.member_path.insert(0, index);
9909                    descriptor.offset = outer_index;
9910                    descriptors.push(descriptor);
9911                }
9912                cursor = descriptors.len();
9913            }
9914            operation => {
9915                descriptors.push(ParallelDescriptor {
9916                    operation,
9917                    offset: cursor,
9918                    member_path: vec![index],
9919                    group_path: vec![parallel_group_entry(base_sequence, size, cursor, kind)],
9920                });
9921                cursor += 1;
9922            }
9923        }
9924    }
9925    Ok(descriptors)
9926}
9927
9928enum ParallelLeafCall {
9929    Activity(ActivityCall),
9930    ChildWorkflow(ChildWorkflowCall),
9931    Timer(TimerCall),
9932    Signal(SignalCall),
9933    Condition(ConditionWaitCall),
9934}
9935
9936fn parallel_leaf_call(
9937    ctx: &WorkflowContext,
9938    operation: ParallelOperation,
9939    parallel_group_path: Vec<ParallelGroupMetadata>,
9940) -> ParallelLeafCall {
9941    match operation {
9942        ParallelOperation::Activity {
9943            activity_type,
9944            options,
9945            arguments,
9946        } => ParallelLeafCall::Activity(ActivityCall {
9947            ctx: ctx.clone(),
9948            activity_type,
9949            options,
9950            args: Some(arguments),
9951            scheduled: false,
9952            parallel_group_path,
9953        }),
9954        ParallelOperation::ChildWorkflow {
9955            workflow_type,
9956            options,
9957            arguments,
9958        } => ParallelLeafCall::ChildWorkflow(ChildWorkflowCall {
9959            ctx: ctx.clone(),
9960            workflow_type,
9961            options,
9962            args: Some(arguments),
9963            scheduled: false,
9964            matched_pending: false,
9965            parallel_group_path,
9966        }),
9967        ParallelOperation::Timer(duration) => {
9968            let delay_seconds = duration
9969                .as_secs()
9970                .checked_add(u64::from(duration.subsec_nanos() > 0));
9971            ParallelLeafCall::Timer(TimerCall {
9972                ctx: ctx.clone(),
9973                delay_seconds,
9974                scheduled: false,
9975                matched_pending: false,
9976                parallel_group_path,
9977            })
9978        }
9979        ParallelOperation::Signal(signal_name) => ParallelLeafCall::Signal(SignalCall {
9980            ctx: ctx.clone(),
9981            signal_name,
9982            runtime_reserved_allowed: false,
9983            opened_wait: false,
9984            matched_pending: false,
9985            parallel_group_path,
9986        }),
9987        ParallelOperation::Condition { options, predicate } => {
9988            ParallelLeafCall::Condition(ConditionWaitCall {
9989                ctx: ctx.clone(),
9990                options,
9991                predicate,
9992                occurrence_id: None,
9993                opened_wait: false,
9994                parallel_group_path,
9995            })
9996        }
9997        ParallelOperation::Group(_) => {
9998            unreachable!("parallel descriptors contain only durable leaves")
9999        }
10000    }
10001}
10002
10003impl ParallelLeafCall {
10004    fn poll_avro_value(&mut self, cx: &mut TaskContext<'_>) -> Poll<Result<ParallelAvroResult>> {
10005        match self {
10006            Self::Activity(call) => Pin::new(call)
10007                .poll_avro_value(cx)
10008                .map_ok(ParallelAvroResult::Activity),
10009            Self::ChildWorkflow(call) => Pin::new(call)
10010                .poll_avro_value(cx)
10011                .map_ok(ParallelAvroResult::ChildWorkflow),
10012            Self::Timer(call) => Pin::new(call)
10013                .poll(cx)
10014                .map_ok(|()| ParallelAvroResult::Timer),
10015            Self::Signal(call) => Pin::new(call)
10016                .poll_avro_value(cx)
10017                .map_ok(ParallelAvroResult::Signal),
10018            Self::Condition(call) => Pin::new(call)
10019                .poll(cx)
10020                .map_ok(ParallelAvroResult::Condition),
10021        }
10022    }
10023}
10024
10025struct ParallelLeaf {
10026    call: ParallelLeafCall,
10027    member_path: Vec<usize>,
10028    group_path: Vec<ParallelGroupMetadata>,
10029    result: Option<ParallelAvroResult>,
10030}
10031
10032/// Future returned by [`WorkflowContext::parallel`].
10033pub struct ParallelCall {
10034    ctx: WorkflowContext,
10035    operations: Option<Vec<ParallelOperation>>,
10036    shape: Option<ParallelShape>,
10037    leaves: Vec<ParallelLeaf>,
10038}
10039
10040impl ParallelCall {
10041    fn new(ctx: WorkflowContext, operations: Vec<ParallelOperation>) -> Self {
10042        Self {
10043            ctx,
10044            operations: Some(operations),
10045            shape: None,
10046            leaves: Vec::new(),
10047        }
10048    }
10049
10050    fn initialize(&mut self) -> Result<()> {
10051        let operations = self.operations.take().unwrap_or_default();
10052        validate_parallel_operations(&operations, &mut Vec::new(), true)?;
10053        self.shape = Some(parallel_shape(&operations));
10054        if operations.is_empty() {
10055            return Ok(());
10056        }
10057
10058        let base_sequence = {
10059            let state = self
10060                .ctx
10061                .state
10062                .lock()
10063                .map_err(|_| Error::WorkflowStatePoisoned)?;
10064            if let Some(recorded) = state.recorded_commands.get(state.command_cursor) {
10065                recorded.sequence()
10066            } else {
10067                let last = state
10068                    .recorded_commands
10069                    .last()
10070                    .map(RecordedCommand::sequence)
10071                    .unwrap_or(0);
10072                last.checked_add(u64::try_from(state.commands.len()).unwrap_or(u64::MAX))
10073                    .and_then(|sequence| sequence.checked_add(1))
10074                    .ok_or_else(|| {
10075                        Error::InvalidParallelGroup(ParallelGroupError {
10076                            reason: "sequence_overflow",
10077                            member_path: Vec::new(),
10078                            message: "parallel group sequence identity overflowed u64".to_string(),
10079                        })
10080                    })?
10081            }
10082        };
10083
10084        self.leaves = parallel_descriptors(operations, base_sequence)?
10085            .into_iter()
10086            .map(|descriptor| {
10087                let call = parallel_leaf_call(
10088                    &self.ctx,
10089                    descriptor.operation,
10090                    descriptor.group_path.clone(),
10091                );
10092                ParallelLeaf {
10093                    call,
10094                    member_path: descriptor.member_path,
10095                    group_path: descriptor.group_path,
10096                    result: None,
10097                }
10098            })
10099            .collect();
10100        Ok(())
10101    }
10102
10103    fn poll_avro_value(
10104        mut self: Pin<&mut Self>,
10105        cx: &mut TaskContext<'_>,
10106    ) -> Poll<Result<Vec<ParallelAvroResult>>> {
10107        if self.operations.is_some() {
10108            if let Err(error) = self.initialize() {
10109                return Poll::Ready(Err(error));
10110            }
10111        }
10112        if self.leaves.is_empty() {
10113            return Poll::Ready(Ok(Vec::new()));
10114        }
10115
10116        let mut failures = Vec::new();
10117        let mut pending = false;
10118        for (index, leaf) in self.leaves.iter_mut().enumerate() {
10119            if leaf.result.is_some() {
10120                continue;
10121            }
10122            match leaf.call.poll_avro_value(cx) {
10123                Poll::Ready(Ok(result)) => leaf.result = Some(result),
10124                Poll::Ready(Err(error)) => failures.push((index, error)),
10125                Poll::Pending => pending = true,
10126            }
10127        }
10128
10129        if !failures.is_empty() {
10130            if let Some(position) = failures
10131                .iter()
10132                .position(|(_, error)| workflow_task_integrity_error(error))
10133            {
10134                return Poll::Ready(Err(failures.remove(position).1));
10135            }
10136            failures.sort_by_key(|(index, _)| *index);
10137            let (failed_index, cause) = failures.remove(0);
10138            let failed = &self.leaves[failed_index];
10139            let completed = self
10140                .leaves
10141                .iter()
10142                .filter_map(|leaf| {
10143                    leaf.result
10144                        .clone()
10145                        .and_then(|result| result.into_json_result().ok())
10146                        .map(|result| ParallelCompletion {
10147                            member_path: leaf.member_path.clone(),
10148                            result,
10149                        })
10150                })
10151                .collect();
10152            let group_id = failed
10153                .group_path
10154                .first()
10155                .map(|entry| entry.parallel_group_id.clone())
10156                .unwrap_or_default();
10157            return Poll::Ready(Err(Error::ParallelFailed(ParallelFailure {
10158                group_id,
10159                member_path: failed.member_path.clone(),
10160                group_path: failed.group_path.clone(),
10161                completed,
10162                cause: Box::new(cause),
10163            })));
10164        }
10165        if pending {
10166            return Poll::Pending;
10167        }
10168
10169        let mut flat_results = self
10170            .leaves
10171            .iter_mut()
10172            .map(|leaf| leaf.result.take().expect("completed parallel leaf"))
10173            .collect::<Vec<_>>()
10174            .into_iter();
10175        let results = parallel_results_for_shape(
10176            self.shape.as_ref().expect("initialized parallel shape"),
10177            &mut flat_results,
10178        );
10179        Poll::Ready(Ok(match results {
10180            ParallelAvroResult::Group(results) => results,
10181            ParallelAvroResult::Activity(_)
10182            | ParallelAvroResult::ChildWorkflow(_)
10183            | ParallelAvroResult::Timer
10184            | ParallelAvroResult::Signal(_)
10185            | ParallelAvroResult::Condition(_) => {
10186                unreachable!("root parallel shape is a group")
10187            }
10188        }))
10189    }
10190}
10191
10192fn parallel_results_for_shape(
10193    shape: &ParallelShape,
10194    flat_results: &mut impl Iterator<Item = ParallelAvroResult>,
10195) -> ParallelAvroResult {
10196    match shape {
10197        ParallelShape::Leaf => flat_results.next().expect("one result per parallel leaf"),
10198        ParallelShape::Group(children) => ParallelAvroResult::Group(
10199            children
10200                .iter()
10201                .map(|child| parallel_results_for_shape(child, flat_results))
10202                .collect(),
10203        ),
10204    }
10205}
10206
10207impl Future for ParallelCall {
10208    type Output = Result<Vec<ParallelResult>>;
10209
10210    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
10211        self.poll_avro_value(cx)
10212            .map_ok(|results| {
10213                results
10214                    .into_iter()
10215                    .map(ParallelAvroResult::into_json_result)
10216                    .collect::<Result<Vec<_>>>()
10217            })
10218            .map_ok(|result| result)
10219            .flatten_result()
10220    }
10221}
10222
10223#[derive(Clone, Debug)]
10224struct SelectionMemberPlan {
10225    key: SelectionKey,
10226    index: usize,
10227    base_sequence: u64,
10228    size: usize,
10229    kind: String,
10230    shape: ParallelShape,
10231    leaf_start: usize,
10232}
10233
10234fn selection_operation_kind(operation: &ParallelOperation) -> &'static str {
10235    match operation {
10236        ParallelOperation::Activity { .. } => "activity",
10237        ParallelOperation::ChildWorkflow { .. } => "child",
10238        ParallelOperation::Timer(_) => "timer",
10239        ParallelOperation::Signal(_) => "signal",
10240        ParallelOperation::Condition { .. } => "condition",
10241        ParallelOperation::Group(_) => "group",
10242    }
10243}
10244
10245fn selection_operation_shape(operation: &ParallelOperation) -> ParallelShape {
10246    match operation {
10247        ParallelOperation::Group(children) => parallel_shape(children),
10248        _ => ParallelShape::Leaf,
10249    }
10250}
10251
10252fn selection_descriptors(
10253    operations: Vec<(SelectionKey, ParallelOperation)>,
10254    base_sequence: u64,
10255) -> Result<(Vec<ParallelDescriptor>, Vec<SelectionMemberPlan>)> {
10256    if operations.is_empty() {
10257        return Err(Error::InvalidParallelGroup(ParallelGroupError {
10258            reason: "selection_empty",
10259            member_path: Vec::new(),
10260            message: "durable selection requires at least one operation".to_string(),
10261        }));
10262    }
10263    let operation_refs = operations
10264        .iter()
10265        .map(|(_, operation)| operation)
10266        .collect::<Vec<_>>();
10267    let total_size = operation_refs
10268        .iter()
10269        .map(|operation| match operation {
10270            ParallelOperation::Group(children) => parallel_leaf_count(children),
10271            _ => 1,
10272        })
10273        .sum::<usize>();
10274    if total_size > MAX_PARALLEL_OPERATIONS {
10275        return Err(Error::InvalidParallelGroup(ParallelGroupError {
10276            reason: "fan_out_limit_exceeded",
10277            member_path: Vec::new(),
10278            message: format!(
10279                "selection contains {total_size} durable leaves; the limit is {MAX_PARALLEL_OPERATIONS}"
10280            ),
10281        }));
10282    }
10283    let group_kind = {
10284        let mut kind = None;
10285        for operation in &operation_refs {
10286            let operation_kind = parallel_operation_kind(operation).unwrap_or("mixed");
10287            match kind {
10288                None => kind = Some(operation_kind),
10289                Some(current) if current == operation_kind => {}
10290                Some(_) => {
10291                    kind = Some("mixed");
10292                    break;
10293                }
10294            }
10295        }
10296        kind.unwrap_or("mixed")
10297    };
10298
10299    let mut descriptors = Vec::with_capacity(total_size);
10300    let mut members = Vec::with_capacity(operations.len());
10301    let mut cursor = 0usize;
10302    let mut seen_keys: Vec<SelectionKey> = Vec::new();
10303    for (member_index, (key, operation)) in operations.into_iter().enumerate() {
10304        if matches!(&key, SelectionKey::Name(value) if value.is_empty()) {
10305            return Err(Error::InvalidParallelGroup(ParallelGroupError {
10306                reason: "selection_key_invalid",
10307                member_path: vec![member_index],
10308                message: "selection member keys must be non-empty strings or non-negative integers"
10309                    .to_string(),
10310            }));
10311        }
10312        if seen_keys.contains(&key) {
10313            return Err(Error::InvalidParallelGroup(ParallelGroupError {
10314                reason: "selection_key_duplicate",
10315                member_path: vec![member_index],
10316                message: format!("selection member key {key:?} is duplicated"),
10317            }));
10318        }
10319        seen_keys.push(key.clone());
10320        let member_size = match &operation {
10321            ParallelOperation::Group(children) => parallel_leaf_count(children),
10322            _ => 1,
10323        };
10324        if member_size == 0 {
10325            return Err(Error::InvalidParallelGroup(ParallelGroupError {
10326                reason: "selection_member_empty",
10327                member_path: vec![member_index],
10328                message: "a selection member must contain at least one durable leaf".to_string(),
10329            }));
10330        }
10331        let member_base = base_sequence
10332            .checked_add(u64::try_from(cursor).unwrap_or(u64::MAX))
10333            .ok_or(Error::TimerDurationOverflow)?;
10334        let member_kind = selection_operation_kind(&operation).to_string();
10335        let member_shape = selection_operation_shape(&operation);
10336        let leaf_start = descriptors.len();
10337        match operation {
10338            ParallelOperation::Group(children) => {
10339                validate_parallel_operations(&children, &mut vec![member_index], false)?;
10340                for mut descriptor in parallel_descriptors(children, member_base)? {
10341                    let flat_index = cursor + descriptor.offset;
10342                    descriptor.group_path.insert(
10343                        0,
10344                        selection_group_entry(
10345                            base_sequence,
10346                            total_size,
10347                            flat_index,
10348                            group_kind,
10349                            &SelectionMemberMetadata {
10350                                key: key.clone(),
10351                                index: member_index,
10352                                base_sequence: member_base,
10353                                size: member_size,
10354                                kind: member_kind.clone(),
10355                            },
10356                        ),
10357                    );
10358                    descriptor.member_path.insert(0, member_index);
10359                    descriptor.offset = flat_index;
10360                    descriptors.push(descriptor);
10361                }
10362            }
10363            operation => {
10364                validate_parallel_operations(
10365                    std::slice::from_ref(&operation),
10366                    &mut Vec::new(),
10367                    true,
10368                )?;
10369                descriptors.push(ParallelDescriptor {
10370                    operation,
10371                    offset: cursor,
10372                    member_path: vec![member_index],
10373                    group_path: vec![selection_group_entry(
10374                        base_sequence,
10375                        total_size,
10376                        cursor,
10377                        group_kind,
10378                        &SelectionMemberMetadata {
10379                            key: key.clone(),
10380                            index: member_index,
10381                            base_sequence: member_base,
10382                            size: member_size,
10383                            kind: member_kind.clone(),
10384                        },
10385                    )],
10386                });
10387            }
10388        }
10389        members.push(SelectionMemberPlan {
10390            key,
10391            index: member_index,
10392            base_sequence: member_base,
10393            size: member_size,
10394            kind: member_kind,
10395            shape: member_shape,
10396            leaf_start,
10397        });
10398        cursor += member_size;
10399    }
10400    Ok((descriptors, members))
10401}
10402
10403struct SelectionLeaf {
10404    call: ParallelLeafCall,
10405    outcome: Option<Result<ParallelAvroResult>>,
10406}
10407
10408/// Stable reference to one member of a durable selection group.
10409#[derive(Clone)]
10410pub struct DurableOperationHandle {
10411    ctx: WorkflowContext,
10412    pub key: SelectionKey,
10413    pub index: usize,
10414    pub kind: String,
10415    pub identity: String,
10416    pub base_sequence: u64,
10417    pub size: usize,
10418    pub selection_group_id: String,
10419    shape: ParallelShape,
10420}
10421
10422impl std::fmt::Debug for DurableOperationHandle {
10423    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10424        formatter
10425            .debug_struct("DurableOperationHandle")
10426            .field("key", &self.key)
10427            .field("index", &self.index)
10428            .field("kind", &self.kind)
10429            .field("identity", &self.identity)
10430            .field("base_sequence", &self.base_sequence)
10431            .field("size", &self.size)
10432            .field("selection_group_id", &self.selection_group_id)
10433            .finish()
10434    }
10435}
10436
10437impl DurableOperationHandle {
10438    /// Await this member after another member has already won the selection.
10439    pub fn await_result(&self) -> DurableOperationAwaitCall {
10440        DurableOperationAwaitCall {
10441            handle: self.clone(),
10442        }
10443    }
10444
10445    /// Request cancellation without affecting siblings. The future resolves
10446    /// to unit; only `SelectionOperationCancelled` history proves cancellation
10447    /// beat a concurrently committed terminal result.
10448    pub fn cancel(&self) -> CancelDurableOperationCall {
10449        CancelDurableOperationCall {
10450            handle: self.clone(),
10451            emitted: false,
10452        }
10453    }
10454}
10455
10456/// The one winner committed for a durable selection group.
10457#[derive(Debug)]
10458pub struct SelectionResult {
10459    pub key: SelectionKey,
10460    pub index: usize,
10461    pub kind: String,
10462    pub identity: String,
10463    pub value: Option<ParallelResult>,
10464    pub failure: Option<Error>,
10465    pub winner: DurableOperationHandle,
10466    pub handles: Vec<DurableOperationHandle>,
10467}
10468
10469impl SelectionResult {
10470    pub fn succeeded(&self) -> bool {
10471        self.failure.is_none()
10472    }
10473
10474    pub fn handle(&self, key: &SelectionKey) -> Option<&DurableOperationHandle> {
10475        self.handles.iter().find(|handle| &handle.key == key)
10476    }
10477
10478    pub fn remaining(&self) -> Vec<&DurableOperationHandle> {
10479        self.handles
10480            .iter()
10481            .filter(|handle| handle.index != self.index)
10482            .collect()
10483    }
10484
10485    pub fn into_result(self) -> Result<ParallelResult> {
10486        match (self.value, self.failure) {
10487            (Some(value), None) => Ok(value),
10488            (_, Some(error)) => Err(error),
10489            _ => Err(Error::WorkerLoop(
10490                "selection result contained neither a value nor a failure".to_string(),
10491            )),
10492        }
10493    }
10494}
10495
10496/// Future returned by [`WorkflowContext::select`].
10497pub struct SelectCall {
10498    ctx: WorkflowContext,
10499    operations: Option<Vec<(SelectionKey, ParallelOperation)>>,
10500    members: Vec<SelectionMemberPlan>,
10501    leaves: Vec<SelectionLeaf>,
10502    group_id: Option<String>,
10503}
10504
10505impl SelectCall {
10506    fn new(ctx: WorkflowContext, operations: Vec<(SelectionKey, ParallelOperation)>) -> Self {
10507        Self {
10508            ctx,
10509            operations: Some(operations),
10510            members: Vec::new(),
10511            leaves: Vec::new(),
10512            group_id: None,
10513        }
10514    }
10515
10516    fn initialize(&mut self) -> Result<()> {
10517        let operations = self.operations.take().unwrap_or_default();
10518        let base_sequence = {
10519            let state = self
10520                .ctx
10521                .state
10522                .lock()
10523                .map_err(|_| Error::WorkflowStatePoisoned)?;
10524            if let Some(marker) = state.selection_markers.get(state.selection_marker_cursor) {
10525                marker.selection_group_base_sequence
10526            } else if let Some(recorded) = state.recorded_commands.get(state.command_cursor) {
10527                recorded.sequence()
10528            } else {
10529                let last = state
10530                    .recorded_commands
10531                    .last()
10532                    .map(RecordedCommand::sequence)
10533                    .unwrap_or(0);
10534                last.checked_add(u64::try_from(state.commands.len()).unwrap_or(u64::MAX))
10535                    .and_then(|sequence| sequence.checked_add(1))
10536                    .ok_or(Error::TimerDurationOverflow)?
10537            }
10538        };
10539        let (descriptors, members) = selection_descriptors(operations, base_sequence)?;
10540        let group_id = format!("select-calls:{base_sequence}:{}", descriptors.len());
10541        self.leaves = descriptors
10542            .into_iter()
10543            .map(|descriptor| SelectionLeaf {
10544                call: parallel_leaf_call(&self.ctx, descriptor.operation, descriptor.group_path),
10545                outcome: None,
10546            })
10547            .collect();
10548        self.members = members;
10549        self.group_id = Some(group_id);
10550        Ok(())
10551    }
10552}
10553
10554impl Future for SelectCall {
10555    type Output = Result<SelectionResult>;
10556
10557    fn poll(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
10558        if self.operations.is_some() {
10559            if let Err(error) = self.initialize() {
10560                return Poll::Ready(Err(error));
10561            }
10562        }
10563
10564        for leaf in &mut self.leaves {
10565            if leaf.outcome.is_some() {
10566                continue;
10567            }
10568            if let Poll::Ready(outcome) = leaf.call.poll_avro_value(cx) {
10569                if outcome
10570                    .as_ref()
10571                    .err()
10572                    .is_some_and(workflow_task_integrity_error)
10573                {
10574                    return Poll::Ready(outcome.map(|_| unreachable!()));
10575                }
10576                leaf.outcome = Some(outcome);
10577            }
10578        }
10579
10580        let all_members_terminal = self.leaves.iter().all(|leaf| leaf.outcome.is_some());
10581        let selection_member_range = self
10582            .members
10583            .first()
10584            .map(|member| member.base_sequence)
10585            .zip(self.leaves.len().try_into().ok())
10586            .map(|(base_sequence, size): (u64, u64)| {
10587                base_sequence..base_sequence.saturating_add(size)
10588            });
10589        let marker = {
10590            let mut state = match self.ctx.state.lock() {
10591                Ok(state) => state,
10592                Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
10593            };
10594            let marker = state
10595                .selection_markers
10596                .get(state.selection_marker_cursor)
10597                .cloned();
10598            if marker.is_none()
10599                && all_members_terminal
10600                && selection_member_range.as_ref().is_some_and(|member_range| {
10601                    state
10602                        .recorded_commands
10603                        .iter()
10604                        .any(|command| member_range.contains(&command.sequence()))
10605                })
10606            {
10607                // Terminal member history can be committed before the server's
10608                // SelectionResolved marker becomes visible to this task. That
10609                // marker is the durable barrier the selection is still waiting
10610                // on, so an otherwise commandless replay remains legitimately
10611                // pending instead of failing as an untracked yield.
10612                state.matched_recorded_pending = true;
10613            }
10614            marker
10615        };
10616        let Some(marker) = marker else {
10617            return Poll::Pending;
10618        };
10619        if self.group_id.as_deref() != Some(marker.selection_group_id.as_str())
10620            || marker.selection_group_size != self.leaves.len()
10621            || self.members.first().map(|member| member.base_sequence)
10622                != Some(marker.selection_group_base_sequence)
10623        {
10624            return Poll::Ready(Err(invalid_recorded_history(
10625                "selection_group_shape_mismatch",
10626                marker.selection_group_base_sequence,
10627                self.group_id
10628                    .as_deref()
10629                    .unwrap_or("initialized selection group"),
10630                &marker.selection_group_id,
10631                "recorded selection group differs from current workflow code",
10632            )));
10633        }
10634        let Some(member_position) = self.members.iter().position(|member| {
10635            member.key == marker.member_key
10636                && member.index == marker.member_index
10637                && member.base_sequence == marker.member_base_sequence
10638                && member.size == marker.member_size
10639                && member.kind == marker.operation_kind
10640        }) else {
10641            return Poll::Ready(Err(invalid_recorded_history(
10642                "selection_member_shape_mismatch",
10643                marker.member_base_sequence,
10644                "winner member matching current workflow code",
10645                &format!("{:?}", marker.member_key),
10646                "recorded selection winner differs from the authored member identity",
10647            )));
10648        };
10649        let member = self.members[member_position].clone();
10650        let (handles, resolution_sequence) = {
10651            let mut state = match self.ctx.state.lock() {
10652                Ok(state) => state,
10653                Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
10654            };
10655            let identities = self
10656                .members
10657                .iter()
10658                .map(|candidate| {
10659                    selection_operation_identity(
10660                        &state,
10661                        &candidate.kind,
10662                        candidate.base_sequence,
10663                        candidate.size,
10664                    )
10665                })
10666                .collect::<Vec<_>>();
10667            if let Some((position, missing)) = identities
10668                .iter()
10669                .enumerate()
10670                .find(|(_, identity)| identity.is_empty())
10671                .map(|(position, identity)| (position, identity.clone()))
10672            {
10673                let candidate = &self.members[position];
10674                return Poll::Ready(Err(invalid_recorded_history(
10675                    "selection_operation_identity_missing",
10676                    candidate.base_sequence,
10677                    &format!(
10678                        "durable {} resource identity from scheduled/open history",
10679                        candidate.kind
10680                    ),
10681                    &missing,
10682                    "selection member history is missing its canonical durable identity",
10683                )));
10684            }
10685            let expected_winner_identity = &identities[member_position];
10686            let resolution_sequence = match validated_selection_resolution_sequence(
10687                &state,
10688                &marker,
10689                &member,
10690                expected_winner_identity,
10691            ) {
10692                Ok(sequence) => sequence,
10693                Err(error) => return Poll::Ready(Err(error)),
10694            };
10695            let handles = self
10696                .members
10697                .iter()
10698                .zip(identities)
10699                .map(|(member, identity)| DurableOperationHandle {
10700                    ctx: self.ctx.clone(),
10701                    key: member.key.clone(),
10702                    index: member.index,
10703                    kind: member.kind.clone(),
10704                    identity,
10705                    base_sequence: member.base_sequence,
10706                    size: member.size,
10707                    selection_group_id: marker.selection_group_id.clone(),
10708                    shape: member.shape.clone(),
10709                })
10710                .collect::<Vec<_>>();
10711            if let Err(error) = validate_selection_cancellations_for_handles(&state, &handles) {
10712                return Poll::Ready(Err(error));
10713            }
10714            state.selection_marker_cursor += 1;
10715            (handles, resolution_sequence)
10716        };
10717
10718        let mut winner_failure = None;
10719        let mut flat_results = Vec::with_capacity(member.size);
10720        if marker.outcome == "failed" {
10721            let resolution_offset = match resolution_sequence
10722                .checked_sub(member.base_sequence)
10723                .and_then(|offset| usize::try_from(offset).ok())
10724            {
10725                Some(offset) if offset < member.size => offset,
10726                _ => {
10727                    return Poll::Ready(Err(invalid_recorded_history(
10728                        "selection_resolution_event_mismatch",
10729                        member.base_sequence,
10730                        "failure event within selected member bounds",
10731                        &resolution_sequence.to_string(),
10732                        "selection failure event is outside the authored member",
10733                    )))
10734                }
10735            };
10736            let leaf = &mut self.leaves[member.leaf_start + resolution_offset];
10737            match leaf.outcome.take() {
10738                Some(Err(error)) => winner_failure = Some(error),
10739                _ => {
10740                    return Poll::Ready(Err(invalid_recorded_history(
10741                        "selection_winner_outcome_mismatch",
10742                        member.base_sequence,
10743                        "exact failed terminal history referenced by SelectionResolved",
10744                        "missing or successful resolution event",
10745                        "selection winner marker disagrees with terminal operation history",
10746                    )))
10747                }
10748            }
10749        } else {
10750            for leaf in &mut self.leaves[member.leaf_start..member.leaf_start + member.size] {
10751                match leaf.outcome.take() {
10752                    Some(Ok(result)) => flat_results.push(result),
10753                    Some(Err(_)) => {
10754                        return Poll::Ready(Err(invalid_recorded_history(
10755                            "selection_winner_outcome_mismatch",
10756                            member.base_sequence,
10757                            "fully completed nested selection member",
10758                            "failed durable leaf",
10759                            "completed selection winner contains a failed leaf",
10760                        )))
10761                    }
10762                    None => {
10763                        return Poll::Ready(Err(invalid_recorded_history(
10764                            "selection_winner_unresolved",
10765                            member.base_sequence,
10766                            "terminal history for every completed winner leaf",
10767                            "pending member history",
10768                            "completed SelectionResolved member has an unfinished durable barrier",
10769                        )))
10770                    }
10771                }
10772            }
10773        }
10774        let value = if winner_failure.is_none() {
10775            let mut flat_results = flat_results.into_iter();
10776            let value = parallel_results_for_shape(&member.shape, &mut flat_results);
10777            match value.into_json_result() {
10778                Ok(value) => Some(value),
10779                Err(error) => return Poll::Ready(Err(error)),
10780            }
10781        } else {
10782            None
10783        };
10784        let winner = handles[member_position].clone();
10785        Poll::Ready(Ok(SelectionResult {
10786            key: winner.key.clone(),
10787            index: winner.index,
10788            kind: winner.kind.clone(),
10789            identity: winner.identity.clone(),
10790            value,
10791            failure: winner_failure,
10792            winner,
10793            handles,
10794        }))
10795    }
10796}
10797
10798fn selection_operation_identity(
10799    state: &WorkflowState,
10800    kind: &str,
10801    base_sequence: u64,
10802    size: usize,
10803) -> String {
10804    if kind == "group" {
10805        return format!("group:{base_sequence}:{size}");
10806    }
10807    let fields: &[&str] = match kind {
10808        "activity" => &["activity_execution_id"],
10809        "child" => &["child_workflow_run_id"],
10810        "timer" => &["timer_id"],
10811        "signal" => &["signal_wait_id"],
10812        "condition" => &["condition_wait_id"],
10813        _ => &[],
10814    };
10815    for sequence in base_sequence..base_sequence.saturating_add(size as u64) {
10816        for event in state
10817            .history_events
10818            .iter()
10819            .filter(|event| durable_event_sequence(event) == Some(sequence))
10820        {
10821            for field in fields {
10822                if let Some(identity) = event.payload.get(*field).and_then(Value::as_str) {
10823                    if !identity.is_empty() {
10824                        return identity.to_string();
10825                    }
10826                }
10827            }
10828        }
10829    }
10830    String::new()
10831}
10832
10833fn validated_selection_resolution_sequence(
10834    state: &WorkflowState,
10835    marker: &SelectionMarker,
10836    member: &SelectionMemberPlan,
10837    expected_identity: &str,
10838) -> Result<u64> {
10839    if expected_identity.is_empty() {
10840        return Err(invalid_recorded_history(
10841            "selection_operation_identity_missing",
10842            member.base_sequence,
10843            &format!(
10844                "durable {} resource identity from scheduled/open history",
10845                member.kind
10846            ),
10847            "missing operation identity",
10848            "selection member history is missing its canonical durable identity",
10849        ));
10850    }
10851    if marker.operation_identity != expected_identity {
10852        return Err(invalid_recorded_history(
10853            "selection_operation_identity_mismatch",
10854            member.base_sequence,
10855            expected_identity,
10856            &marker.operation_identity,
10857            "selection winner identity does not match durable scheduled/open history",
10858        ));
10859    }
10860
10861    let failure_types = [
10862        "ActivityFailed",
10863        "ActivityCancelled",
10864        "ActivityTimedOut",
10865        "ChildRunFailed",
10866        "ChildRunCancelled",
10867        "ChildRunTerminated",
10868    ];
10869    let success_types = [
10870        "ActivityCompleted",
10871        "ChildRunCompleted",
10872        "TimerFired",
10873        "SignalApplied",
10874        "ConditionWaitSatisfied",
10875        "ConditionWaitTimedOut",
10876    ];
10877    let terminal_types: &[&str] = if marker.outcome == "failed" {
10878        &failure_types
10879    } else {
10880        &success_types
10881    };
10882    let mut candidates = Vec::new();
10883    for event in state.history_events.iter() {
10884        let Some(sequence) = durable_event_sequence(event) else {
10885            continue;
10886        };
10887        if sequence < member.base_sequence
10888            || sequence >= member.base_sequence.saturating_add(member.size as u64)
10889            || !terminal_types.contains(&event.event_type.as_str())
10890        {
10891            continue;
10892        }
10893        let event_id = event
10894            .raw
10895            .get("id")
10896            .or_else(|| event.raw.get("event_id"))
10897            .and_then(Value::as_str)
10898            .filter(|value| !value.is_empty())
10899            .ok_or_else(|| {
10900                invalid_recorded_history(
10901                    "selection_resolution_event_id_missing",
10902                    member.base_sequence,
10903                    "terminal selection history with a durable event id",
10904                    &event.payload.to_string(),
10905                    "selection terminal history cannot be bound to its winner marker",
10906                )
10907            })?;
10908        candidates.push((event_id.to_string(), event.event_type.clone(), sequence));
10909    }
10910    let resolution = if marker.outcome == "failed" {
10911        candidates.first()
10912    } else {
10913        candidates.last()
10914    };
10915    let Some((event_id, event_type, sequence)) = resolution else {
10916        return Err(invalid_recorded_history(
10917            "selection_resolution_event_missing",
10918            member.base_sequence,
10919            "terminal history for the selected member",
10920            &format!("{:?}", marker.member_key),
10921            "selection winner marker has no matching durable terminal event",
10922        ));
10923    };
10924    if event_id != &marker.resolution_event_id || event_type != &marker.resolution_event_type {
10925        return Err(invalid_recorded_history(
10926            "selection_resolution_event_mismatch",
10927            member.base_sequence,
10928            &format!("{event_type}:{event_id}"),
10929            &format!(
10930                "{}:{}",
10931                marker.resolution_event_type, marker.resolution_event_id
10932            ),
10933            "selection winner marker does not reference the event that made its member terminal",
10934        ));
10935    }
10936    Ok(*sequence)
10937}
10938
10939fn recorded_selection_member_outcome(
10940    state: &WorkflowState,
10941    handle: &DurableOperationHandle,
10942) -> Result<Option<ParallelResult>> {
10943    for event in state.history_events.iter() {
10944        let Some(sequence) = durable_event_sequence(event) else {
10945            continue;
10946        };
10947        if sequence < handle.base_sequence
10948            || sequence >= handle.base_sequence.saturating_add(handle.size as u64)
10949            || !matches!(
10950                event.event_type.as_str(),
10951                "ActivityFailed"
10952                    | "ActivityCancelled"
10953                    | "ActivityTimedOut"
10954                    | "ChildRunFailed"
10955                    | "ChildRunCancelled"
10956                    | "ChildRunTerminated"
10957            )
10958        {
10959            continue;
10960        }
10961        let Some(command) = state
10962            .recorded_commands
10963            .iter()
10964            .find(|command| command.sequence() == sequence)
10965        else {
10966            continue;
10967        };
10968        match command {
10969            RecordedCommand::Activity {
10970                outcome: Some(Err(failure)),
10971                ..
10972            } => return Err(Error::ActivityFailed(failure.clone())),
10973            RecordedCommand::ChildWorkflow {
10974                outcome: Some(Err(failure)),
10975                ..
10976            } => return Err(Error::ChildWorkflowFailed(failure.clone())),
10977            _ => {}
10978        }
10979    }
10980
10981    let mut results = Vec::with_capacity(handle.size);
10982    for sequence in handle.base_sequence..handle.base_sequence.saturating_add(handle.size as u64) {
10983        let Some(command) = state
10984            .recorded_commands
10985            .iter()
10986            .find(|command| command.sequence() == sequence)
10987        else {
10988            return Ok(None);
10989        };
10990        let result = match command {
10991            RecordedCommand::Activity { outcome, .. } => match outcome {
10992                Some(Ok(value)) => ParallelAvroResult::Activity(value.clone()),
10993                Some(Err(failure)) => return Err(Error::ActivityFailed(failure.clone())),
10994                None => return Ok(None),
10995            },
10996            RecordedCommand::Timer { fired, .. } => {
10997                if !fired {
10998                    return Ok(None);
10999                }
11000                ParallelAvroResult::Timer
11001            }
11002            RecordedCommand::ChildWorkflow { outcome, .. } => match outcome {
11003                Some(Ok(value)) => ParallelAvroResult::ChildWorkflow(value.clone()),
11004                Some(Err(failure)) => return Err(Error::ChildWorkflowFailed(failure.clone())),
11005                None => return Ok(None),
11006            },
11007            RecordedCommand::SignalWait { value, .. } => match value {
11008                Some(value) => ParallelAvroResult::Signal(value.clone()),
11009                None => return Ok(None),
11010            },
11011            RecordedCommand::ConditionWait { result, .. } => match result {
11012                Some(result) => ParallelAvroResult::Condition(*result),
11013                None => return Ok(None),
11014            },
11015            other => {
11016                return Err(command_mismatch(
11017                    other,
11018                    format!("selected {} member", handle.kind),
11019                ))
11020            }
11021        };
11022        results.push(result);
11023    }
11024    let mut results = results.into_iter();
11025    parallel_results_for_shape(&handle.shape, &mut results)
11026        .into_json_result()
11027        .map(Some)
11028}
11029
11030fn recorded_selection_member_is_terminal(
11031    state: &WorkflowState,
11032    handle: &DurableOperationHandle,
11033) -> bool {
11034    let mut completed = 0usize;
11035    let mut all_completed = true;
11036    for sequence in handle.base_sequence..handle.base_sequence.saturating_add(handle.size as u64) {
11037        let Some(command) = state
11038            .recorded_commands
11039            .iter()
11040            .find(|command| command.sequence() == sequence)
11041        else {
11042            all_completed = false;
11043            continue;
11044        };
11045        let terminal = match command {
11046            RecordedCommand::Activity {
11047                outcome: Some(Err(_)),
11048                ..
11049            }
11050            | RecordedCommand::ChildWorkflow {
11051                outcome: Some(Err(_)),
11052                ..
11053            } => return true,
11054            RecordedCommand::Activity { outcome, .. } => outcome.is_some(),
11055            RecordedCommand::ChildWorkflow { outcome, .. } => outcome.is_some(),
11056            RecordedCommand::Timer { fired, .. } => *fired,
11057            RecordedCommand::SignalWait { value, .. } => value.is_some(),
11058            RecordedCommand::ConditionWait { result, .. } => result.is_some(),
11059            RecordedCommand::SearchAttributes { .. }
11060            | RecordedCommand::SideEffect { .. }
11061            | RecordedCommand::VersionMarker { .. }
11062            | RecordedCommand::Memo { .. } => false,
11063        };
11064        if !terminal {
11065            all_completed = false;
11066            continue;
11067        }
11068        completed += 1;
11069    }
11070    all_completed && completed == handle.size
11071}
11072
11073fn selection_cancellation_for_handle(
11074    state: &WorkflowState,
11075    handle: &DurableOperationHandle,
11076) -> Result<bool> {
11077    let Some(marker) = state.cancelled_selection_members.iter().find(|recorded| {
11078        recorded.selection_group_id == handle.selection_group_id
11079            && recorded.member_base_sequence == handle.base_sequence
11080    }) else {
11081        return Ok(false);
11082    };
11083    validate_selection_cancellation_marker(marker, handle)?;
11084    Ok(true)
11085}
11086
11087fn validate_selection_cancellations_for_handles(
11088    state: &WorkflowState,
11089    handles: &[DurableOperationHandle],
11090) -> Result<()> {
11091    let Some(group_id) = handles
11092        .first()
11093        .map(|handle| handle.selection_group_id.as_str())
11094    else {
11095        return Ok(());
11096    };
11097    for marker in state
11098        .cancelled_selection_members
11099        .iter()
11100        .filter(|marker| marker.selection_group_id == group_id)
11101    {
11102        let Some(handle) = handles
11103            .iter()
11104            .find(|handle| handle.base_sequence == marker.member_base_sequence)
11105        else {
11106            return Err(invalid_recorded_history(
11107                "selection_cancellation_member_mismatch",
11108                marker.member_base_sequence,
11109                "SelectionOperationCancelled matching an authored selection handle",
11110                &format!("{marker:?}"),
11111                "selection cancellation member base does not name an authored member",
11112            ));
11113        };
11114        validate_selection_cancellation_marker(marker, handle)?;
11115    }
11116    Ok(())
11117}
11118
11119fn validate_selection_cancellation_marker(
11120    marker: &SelectionCancellation,
11121    handle: &DurableOperationHandle,
11122) -> Result<()> {
11123    if marker.selection_group_id != handle.selection_group_id
11124        || marker.member_key != handle.key
11125        || marker.member_index != handle.index
11126        || marker.member_base_sequence != handle.base_sequence
11127        || marker.member_size != handle.size
11128        || marker.operation_kind != handle.kind
11129        || marker.operation_identity != handle.identity
11130    {
11131        return Err(invalid_recorded_history(
11132            "selection_cancellation_member_mismatch",
11133            handle.base_sequence,
11134            "SelectionOperationCancelled matching the authored selection handle",
11135            &format!("{marker:?}"),
11136            "selection cancellation history targets different authored member metadata",
11137        ));
11138    }
11139    Ok(())
11140}
11141
11142/// Future returned by [`DurableOperationHandle::await_result`].
11143pub struct DurableOperationAwaitCall {
11144    handle: DurableOperationHandle,
11145}
11146
11147impl Future for DurableOperationAwaitCall {
11148    type Output = Result<ParallelResult>;
11149
11150    fn poll(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11151        let state = match self.handle.ctx.state.lock() {
11152            Ok(state) => state,
11153            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11154        };
11155        match selection_cancellation_for_handle(&state, &self.handle) {
11156            Err(error) => return Poll::Ready(Err(error)),
11157            Ok(false) => {}
11158            Ok(true) => {
11159                return Poll::Ready(Err(Error::DurableOperationCancelled(
11160                    DurableOperationCancelled {
11161                        selection_group_id: self.handle.selection_group_id.clone(),
11162                        member_key: self.handle.key.clone(),
11163                        member_index: self.handle.index,
11164                        operation_kind: self.handle.kind.clone(),
11165                        operation_identity: self.handle.identity.clone(),
11166                    },
11167                )));
11168            }
11169        }
11170        match recorded_selection_member_outcome(&state, &self.handle) {
11171            Ok(Some(result)) => Poll::Ready(Ok(result)),
11172            Ok(None) => Poll::Pending,
11173            Err(error) => Poll::Ready(Err(error)),
11174        }
11175    }
11176}
11177
11178/// Future returned by [`DurableOperationHandle::cancel`].
11179pub struct CancelDurableOperationCall {
11180    handle: DurableOperationHandle,
11181    emitted: bool,
11182}
11183
11184impl Future for CancelDurableOperationCall {
11185    type Output = Result<()>;
11186
11187    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11188        let ctx = self.handle.ctx.clone();
11189        let mut state = match ctx.state.lock() {
11190            Ok(state) => state,
11191            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11192        };
11193        match selection_cancellation_for_handle(&state, &self.handle) {
11194            Err(error) => return Poll::Ready(Err(error)),
11195            Ok(true) => return Poll::Ready(Ok(())),
11196            Ok(false) => {}
11197        }
11198        if recorded_selection_member_is_terminal(&state, &self.handle) {
11199            return Poll::Ready(Ok(()));
11200        }
11201        if !self.emitted {
11202            state.commands.push(json!({
11203                "type": "cancel_selection_operation",
11204                "selection_group_id": self.handle.selection_group_id,
11205                "member_key": self.handle.key,
11206                "member_index": self.handle.index,
11207                "member_base_sequence": self.handle.base_sequence,
11208                "member_size": self.handle.size,
11209                "operation_kind": self.handle.kind,
11210                "operation_identity": self.handle.identity,
11211            }));
11212            self.emitted = true;
11213        }
11214        // The cancellation command is only a request. Do not expose workflow
11215        // state after cancel until committed SelectionOperationCancelled
11216        // history proves that the request won the race with member completion.
11217        Poll::Pending
11218    }
11219}
11220
11221trait PollNestedResultExt<T> {
11222    fn flatten_result(self) -> Poll<Result<T>>;
11223}
11224
11225impl<T> PollNestedResultExt<T> for Poll<Result<Result<T>>> {
11226    fn flatten_result(self) -> Poll<Result<T>> {
11227        match self {
11228            Poll::Ready(Ok(result)) => Poll::Ready(result),
11229            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
11230            Poll::Pending => Poll::Pending,
11231        }
11232    }
11233}
11234
11235struct SagaCompensation {
11236    activity_type: String,
11237    options: ActivityOptions,
11238    arguments: AvroValue,
11239    registration_order: usize,
11240}
11241
11242/// Workflow-local deterministic saga compensation helper.
11243///
11244/// Register each compensation only after its forward step succeeds. Passing
11245/// the forward `Result` to [`Saga::finish`] runs compensations sequentially in
11246/// reverse registration order after any failure, including cooperative
11247/// cancellation. Each compensation is an ordinary durable activity, so replay,
11248/// duplicate delivery, and worker restart use existing history semantics.
11249pub struct Saga {
11250    ctx: WorkflowContext,
11251    compensations: Vec<SagaCompensation>,
11252}
11253
11254impl Saga {
11255    fn new(ctx: WorkflowContext) -> Self {
11256        Self {
11257            ctx,
11258            compensations: Vec::new(),
11259        }
11260    }
11261
11262    pub fn add_compensation<T: Serialize>(
11263        &mut self,
11264        activity_type: impl Into<String>,
11265        args: T,
11266    ) -> Result<&mut Self> {
11267        self.add_compensation_with_options(activity_type, ActivityOptions::new(), args)
11268    }
11269
11270    pub fn add_compensation_with_options<T: Serialize>(
11271        &mut self,
11272        activity_type: impl Into<String>,
11273        options: ActivityOptions,
11274        args: T,
11275    ) -> Result<&mut Self> {
11276        let activity_type = activity_type.into();
11277        if activity_type.trim().is_empty() || activity_type.trim() != activity_type {
11278            return Err(Error::Codec(
11279                "saga compensation activity type must be non-empty without surrounding whitespace"
11280                    .to_string(),
11281            ));
11282        }
11283        options.validate().map_err(Error::InvalidActivityOptions)?;
11284        let arguments = AvroValue::from_serialize(&args)?;
11285        let registration_order = self.compensations.len() + 1;
11286        self.compensations.push(SagaCompensation {
11287            activity_type,
11288            options,
11289            arguments,
11290            registration_order,
11291        });
11292        Ok(self)
11293    }
11294
11295    /// Compensate `initiating_failure` and return the failure that must remain.
11296    pub async fn compensate(mut self, initiating_failure: Error) -> Error {
11297        while let Some(compensation) = self.compensations.pop() {
11298            if let Err(compensation_failure) = self
11299                .ctx
11300                .activity_with_options(
11301                    compensation.activity_type.clone(),
11302                    compensation.options,
11303                    compensation.arguments,
11304                )
11305                .await
11306            {
11307                if workflow_task_integrity_error(&compensation_failure) {
11308                    return compensation_failure;
11309                }
11310                return Error::SagaCompensationFailed(SagaCompensationFailure {
11311                    initiating_failure: Box::new(initiating_failure),
11312                    compensation_failure: Box::new(compensation_failure),
11313                    compensation_activity_type: compensation.activity_type,
11314                    compensation_registration_order: compensation.registration_order,
11315                });
11316            }
11317        }
11318        initiating_failure
11319    }
11320
11321    /// Return a successful forward value or compensate and preserve its failure.
11322    pub async fn finish<T>(self, outcome: Result<T>) -> Result<T> {
11323        match outcome {
11324            Ok(value) => Ok(value),
11325            Err(error) => Err(self.compensate(error).await),
11326        }
11327    }
11328}
11329
11330pub struct ActivityCall {
11331    ctx: WorkflowContext,
11332    activity_type: String,
11333    options: ActivityOptions,
11334    args: Option<Result<AvroValue>>,
11335    scheduled: bool,
11336    parallel_group_path: Vec<ParallelGroupMetadata>,
11337}
11338
11339impl ActivityCall {
11340    fn poll_avro_value(
11341        mut self: Pin<&mut Self>,
11342        _cx: &mut TaskContext<'_>,
11343    ) -> Poll<Result<AvroValue>> {
11344        let ctx = self.ctx.clone();
11345        let mut state = match ctx.state.lock() {
11346            Ok(state) => state,
11347            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11348        };
11349
11350        if self.scheduled {
11351            return Poll::Pending;
11352        }
11353
11354        let options = match self.options.validate() {
11355            Ok(options) => options,
11356            Err(error) => {
11357                return Poll::Ready(Err(Error::InvalidActivityOptions(error)));
11358            }
11359        };
11360        let task_queue = options
11361            .task_queue
11362            .clone()
11363            .unwrap_or_else(|| state.task_queue.clone());
11364        let current_recorded_options = RecordedActivityOptions {
11365            task_queue: RecordedSnapshotValue::Known(Some(task_queue.clone())),
11366            // Rust schedules ordinary durable activities. The server records a
11367            // non-null mode only for a specialized execution primitive.
11368            execution_mode: RecordedSnapshotValue::Known(None),
11369            retry_policy: current_activity_retry_snapshot(&options),
11370        };
11371
11372        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
11373            let sequence = recorded.sequence();
11374            match recorded {
11375                RecordedCommand::Activity {
11376                    activity_type,
11377                    options: recorded_options,
11378                    outcome,
11379                    parallel_group_path,
11380                    ..
11381                } => {
11382                    if let Err(error) = ensure_parallel_path_matches(
11383                        sequence,
11384                        parallel_group_path.as_deref(),
11385                        &self.parallel_group_path,
11386                    ) {
11387                        return Poll::Ready(Err(error));
11388                    }
11389                    if let Some(recorded_type) = activity_type {
11390                        if recorded_type != self.activity_type {
11391                            return Poll::Ready(Err(Error::NonDeterministicReplay(
11392                                ReplayFailure::new(
11393                                    "recorded_command_detail_mismatch",
11394                                    Some(sequence),
11395                                    Some(format!("activity:{recorded_type}")),
11396                                    Some(format!("activity:{}", self.activity_type)),
11397                                    "recorded activity type differs from the current workflow command",
11398                                ),
11399                            )));
11400                        }
11401                    }
11402                    if let Some(recorded_options) = recorded_options {
11403                        if !recorded_options
11404                            .task_queue
11405                            .matches_current(&current_recorded_options.task_queue)
11406                        {
11407                            return Poll::Ready(Err(Error::NonDeterministicReplay(
11408                                ReplayFailure::new(
11409                                    "activity_task_queue_mismatch",
11410                                    Some(sequence),
11411                                    Some(activity_options_description(&recorded_options)),
11412                                    Some(activity_options_description(&current_recorded_options)),
11413                                    "recorded activity task queue differs from the current workflow command",
11414                                ),
11415                            )));
11416                        }
11417                        if !recorded_options
11418                            .execution_mode
11419                            .matches_current(&current_recorded_options.execution_mode)
11420                        {
11421                            return Poll::Ready(Err(Error::NonDeterministicReplay(
11422                                ReplayFailure::new(
11423                                    "activity_execution_mode_mismatch",
11424                                    Some(sequence),
11425                                    Some(activity_options_description(&recorded_options)),
11426                                    Some(activity_options_description(&current_recorded_options)),
11427                                    "recorded activity execution mode differs from the current workflow command",
11428                                ),
11429                            )));
11430                        }
11431                        if !recorded_options
11432                            .retry_policy
11433                            .matches_current(&current_recorded_options.retry_policy)
11434                        {
11435                            return Poll::Ready(Err(Error::NonDeterministicReplay(
11436                                ReplayFailure::new(
11437                                    "activity_retry_policy_mismatch",
11438                                    Some(sequence),
11439                                    Some(activity_options_description(&recorded_options)),
11440                                    Some(activity_options_description(&current_recorded_options)),
11441                                    "recorded activity retry policy differs from the current workflow command",
11442                                ),
11443                            )));
11444                        }
11445                    }
11446                    state.command_cursor += 1;
11447                    if let Some(outcome) = outcome {
11448                        return Poll::Ready(outcome.map_err(Error::ActivityFailed));
11449                    }
11450                    state.matched_recorded_pending = true;
11451                    self.scheduled = true;
11452                    return Poll::Pending;
11453                }
11454                other => {
11455                    return Poll::Ready(Err(command_mismatch(
11456                        &other,
11457                        format!("activity:{}", self.activity_type),
11458                    )));
11459                }
11460            }
11461        }
11462
11463        if !self.scheduled {
11464            let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
11465                Ok(args) => args,
11466                Err(error) => return Poll::Ready(Err(error)),
11467            };
11468            let arguments = normalize_avro_arguments(args);
11469            let envelope = match encode_typed_envelope(&arguments, &state.payload_codec) {
11470                Ok(envelope) => envelope,
11471                Err(error) => return Poll::Ready(Err(error)),
11472            };
11473
11474            let mut command = serde_json::Map::from_iter([
11475                ("type".to_string(), json!("schedule_activity")),
11476                (
11477                    "activity_type".to_string(),
11478                    json!(self.activity_type.clone()),
11479                ),
11480                ("queue".to_string(), json!(task_queue)),
11481                ("arguments".to_string(), envelope),
11482            ]);
11483            for (field, value) in [
11484                ("start_to_close_timeout", options.start_to_close_timeout),
11485                (
11486                    "schedule_to_start_timeout",
11487                    options.schedule_to_start_timeout,
11488                ),
11489                (
11490                    "schedule_to_close_timeout",
11491                    options.schedule_to_close_timeout,
11492                ),
11493                ("heartbeat_timeout", options.heartbeat_timeout),
11494            ] {
11495                if let Some(value) = value {
11496                    command.insert(field.to_string(), json!(value));
11497                }
11498            }
11499            if let Some(retry_policy) = options.retry_policy {
11500                command.insert("retry_policy".to_string(), retry_policy);
11501            }
11502            apply_parallel_group_path(&mut command, &self.parallel_group_path);
11503            state.commands.push(Value::Object(command));
11504            self.scheduled = true;
11505        }
11506
11507        Poll::Pending
11508    }
11509}
11510
11511impl Future for ActivityCall {
11512    type Output = Result<Value>;
11513
11514    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11515        match self.poll_avro_value(cx) {
11516            Poll::Ready(Ok(value)) => Poll::Ready(value.into_json()),
11517            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
11518            Poll::Pending => Poll::Pending,
11519        }
11520    }
11521}
11522
11523/// Future returned by [`WorkflowContext::sleep`].
11524pub struct TimerCall {
11525    ctx: WorkflowContext,
11526    delay_seconds: Option<u64>,
11527    scheduled: bool,
11528    matched_pending: bool,
11529    parallel_group_path: Vec<ParallelGroupMetadata>,
11530}
11531
11532impl Future for TimerCall {
11533    type Output = Result<()>;
11534
11535    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11536        if self.matched_pending {
11537            return Poll::Pending;
11538        }
11539
11540        let ctx = self.ctx.clone();
11541        let Some(requested_delay) = self.delay_seconds else {
11542            return Poll::Ready(Err(Error::TimerDurationOverflow));
11543        };
11544        let mut state = match ctx.state.lock() {
11545            Ok(state) => state,
11546            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11547        };
11548
11549        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
11550            match recorded {
11551                RecordedCommand::Timer {
11552                    sequence,
11553                    delay_seconds,
11554                    fired,
11555                    parallel_group_path,
11556                    ..
11557                } => {
11558                    if let Err(error) = ensure_parallel_path_matches(
11559                        sequence,
11560                        parallel_group_path.as_deref(),
11561                        &self.parallel_group_path,
11562                    ) {
11563                        return Poll::Ready(Err(error));
11564                    }
11565                    if delay_seconds != requested_delay {
11566                        return Poll::Ready(Err(Error::NonDeterministicReplay(
11567                            ReplayFailure::new(
11568                                "timer_delay_mismatch",
11569                                Some(sequence),
11570                                Some(format!("timer:{delay_seconds}s")),
11571                                Some(format!("timer:{requested_delay}s")),
11572                                "recorded timer delay differs from the current workflow command",
11573                            ),
11574                        )));
11575                    }
11576                    state.command_cursor += 1;
11577                    if fired {
11578                        return Poll::Ready(Ok(()));
11579                    }
11580                    state.matched_recorded_pending = true;
11581                    self.scheduled = true;
11582                    self.matched_pending = true;
11583                    return Poll::Pending;
11584                }
11585                other => return Poll::Ready(Err(command_mismatch(&other, "timer"))),
11586            }
11587        }
11588
11589        if !self.scheduled {
11590            let mut command = serde_json::Map::from_iter([
11591                ("type".to_string(), json!("start_timer")),
11592                ("delay_seconds".to_string(), json!(requested_delay)),
11593            ]);
11594            apply_parallel_group_path(&mut command, &self.parallel_group_path);
11595            state.commands.push(Value::Object(command));
11596            self.scheduled = true;
11597        }
11598
11599        Poll::Pending
11600    }
11601}
11602
11603/// Future returned by [`WorkflowContext::wait_condition`].
11604pub struct ConditionWaitCall {
11605    ctx: WorkflowContext,
11606    options: ConditionWaitOptions,
11607    predicate: Box<dyn Fn() -> Result<bool> + Send + 'static>,
11608    occurrence_id: Option<String>,
11609    opened_wait: bool,
11610    parallel_group_path: Vec<ParallelGroupMetadata>,
11611}
11612
11613impl Future for ConditionWaitCall {
11614    type Output = Result<ConditionWaitResult>;
11615
11616    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11617        if self.opened_wait {
11618            return Poll::Pending;
11619        }
11620
11621        let options = match self.options.validate() {
11622            Ok(options) => options,
11623            Err(error) => return Poll::Ready(Err(Error::InvalidConditionWaitOptions(error))),
11624        };
11625        let ctx = self.ctx.clone();
11626        let occurrence_id = match self.occurrence_id.as_ref() {
11627            Some(occurrence_id) => occurrence_id.clone(),
11628            None => {
11629                let mut state = match ctx.state.lock() {
11630                    Ok(state) => state,
11631                    Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11632                };
11633                let ordinal = state.condition_wait_occurrence_counter;
11634                state.condition_wait_occurrence_counter = match ordinal.checked_add(1) {
11635                    Some(next) => next,
11636                    None => {
11637                        return Poll::Ready(Err(Error::WorkerLoop(
11638                            "condition wait occurrence counter overflowed".to_string(),
11639                        )))
11640                    }
11641                };
11642                let occurrence_id = format!("{CONDITION_WAIT_OCCURRENCE_PREFIX}{ordinal}");
11643                drop(state);
11644                self.occurrence_id = Some(occurrence_id.clone());
11645                occurrence_id
11646            }
11647        };
11648
11649        let recorded_result = {
11650            let mut state = match ctx.state.lock() {
11651                Ok(state) => state,
11652                Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11653            };
11654            let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() else {
11655                drop(state);
11656                return self.poll_new_condition(options);
11657            };
11658            if !matches!(recorded, RecordedCommand::ConditionWait { .. }) {
11659                return Poll::Ready(Err(command_mismatch(&recorded, "condition wait")));
11660            }
11661
11662            let mut cursor = state.command_cursor;
11663            let mut result = None;
11664            loop {
11665                let Some(RecordedCommand::ConditionWait {
11666                    sequence,
11667                    occurrence_id: recorded_occurrence_id,
11668                    condition_key,
11669                    predicate_identity,
11670                    timeout_seconds,
11671                    result: recorded_result,
11672                    parallel_group_path,
11673                    ..
11674                }) = state.recorded_commands.get(cursor)
11675                else {
11676                    break;
11677                };
11678
11679                if cursor > state.command_cursor && recorded_occurrence_id != &occurrence_id {
11680                    break;
11681                }
11682                if let Err(error) = ensure_parallel_path_matches(
11683                    *sequence,
11684                    parallel_group_path.as_deref(),
11685                    &self.parallel_group_path,
11686                ) {
11687                    return Poll::Ready(Err(error));
11688                }
11689                if let Err(error) = validate_recorded_condition_wait(
11690                    *sequence,
11691                    recorded_occurrence_id,
11692                    condition_key.as_deref(),
11693                    predicate_identity,
11694                    *timeout_seconds,
11695                    &occurrence_id,
11696                    &options,
11697                ) {
11698                    return Poll::Ready(Err(error));
11699                }
11700                if result == Some(ConditionWaitResult::TimedOut) {
11701                    return Poll::Ready(Err(Error::NonDeterministicReplay(ReplayFailure::new(
11702                        "condition_wait_reopened_after_timeout",
11703                        Some(*sequence),
11704                        Some("timed-out condition is terminal".to_string()),
11705                        Some("another physical wait-open".to_string()),
11706                        "condition history reopened one logical wait after its durable timeout",
11707                    ))));
11708                }
11709                result = *recorded_result;
11710                cursor += 1;
11711            }
11712            state.command_cursor = cursor;
11713            result
11714        };
11715
11716        if let Some(result) = recorded_result {
11717            return Poll::Ready(Ok(result));
11718        }
11719
11720        self.poll_open_condition(options)
11721    }
11722}
11723
11724impl ConditionWaitCall {
11725    fn poll_new_condition(
11726        self: Pin<&mut Self>,
11727        options: ValidatedConditionWaitOptions,
11728    ) -> Poll<Result<ConditionWaitResult>> {
11729        self.poll_open_condition(options)
11730    }
11731
11732    fn poll_open_condition(
11733        mut self: Pin<&mut Self>,
11734        options: ValidatedConditionWaitOptions,
11735    ) -> Poll<Result<ConditionWaitResult>> {
11736        let selection_member = self
11737            .parallel_group_path
11738            .first()
11739            .is_some_and(|entry| entry.parallel_group_mode.as_deref() == Some("select"));
11740        match (self.predicate)() {
11741            Ok(true) if !selection_member => {
11742                return Poll::Ready(Ok(ConditionWaitResult::Satisfied))
11743            }
11744            Ok(_) => {}
11745            Err(error) => return Poll::Ready(Err(error)),
11746        }
11747        if options.timeout_seconds == Some(0) && !selection_member {
11748            return Poll::Ready(Ok(ConditionWaitResult::TimedOut));
11749        }
11750
11751        let ctx = self.ctx.clone();
11752        let mut state = match ctx.state.lock() {
11753            Ok(state) => state,
11754            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11755        };
11756        let mut command = serde_json::Map::from_iter([
11757            ("type".to_string(), json!("open_condition_wait")),
11758            (
11759                "condition_wait_occurrence_id".to_string(),
11760                json!(self.occurrence_id.as_deref().unwrap_or_default()),
11761            ),
11762            ("condition_key".to_string(), json!(options.condition_key)),
11763            (
11764                "condition_definition_fingerprint".to_string(),
11765                json!(options.predicate_identity),
11766            ),
11767        ]);
11768        if let Some(timeout_seconds) = options.timeout_seconds {
11769            command.insert("timeout_seconds".to_string(), json!(timeout_seconds));
11770        }
11771        apply_parallel_group_path(&mut command, &self.parallel_group_path);
11772        state.commands.push(Value::Object(command));
11773        drop(state);
11774        self.opened_wait = true;
11775        Poll::Pending
11776    }
11777}
11778
11779fn validate_recorded_condition_wait(
11780    sequence: u64,
11781    recorded_occurrence_id: &str,
11782    recorded_key: Option<&str>,
11783    recorded_predicate_identity: &str,
11784    recorded_timeout_seconds: Option<u64>,
11785    current_occurrence_id: &str,
11786    current: &ValidatedConditionWaitOptions,
11787) -> Result<()> {
11788    if recorded_occurrence_id != current_occurrence_id {
11789        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
11790            "condition_wait_occurrence_mismatch",
11791            Some(sequence),
11792            Some(recorded_occurrence_id.to_string()),
11793            Some(current_occurrence_id.to_string()),
11794            "recorded condition occurrence differs from the current authored wait position",
11795        )));
11796    }
11797    if recorded_key != Some(current.condition_key.as_str()) {
11798        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
11799            "condition_wait_key_mismatch",
11800            Some(sequence),
11801            recorded_key.map(str::to_string),
11802            Some(current.condition_key.clone()),
11803            "recorded condition identity differs from the current workflow wait",
11804        )));
11805    }
11806    if recorded_predicate_identity != current.predicate_identity {
11807        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
11808            "condition_wait_predicate_mismatch",
11809            Some(sequence),
11810            Some(recorded_predicate_identity.to_string()),
11811            Some(current.predicate_identity.clone()),
11812            "recorded condition predicate behavior differs from current workflow code",
11813        )));
11814    }
11815    if recorded_timeout_seconds != current.timeout_seconds {
11816        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
11817            "condition_wait_timeout_mismatch",
11818            Some(sequence),
11819            recorded_timeout_seconds.map(|seconds| format!("{seconds}s")),
11820            current.timeout_seconds.map(|seconds| format!("{seconds}s")),
11821            "recorded condition timeout differs from the current workflow wait",
11822        )));
11823    }
11824    Ok(())
11825}
11826
11827/// Future returned by [`WorkflowContext::start_child_workflow`].
11828pub struct ChildWorkflowCall {
11829    ctx: WorkflowContext,
11830    workflow_type: String,
11831    options: ChildWorkflowOptions,
11832    args: Option<Result<AvroValue>>,
11833    scheduled: bool,
11834    matched_pending: bool,
11835    parallel_group_path: Vec<ParallelGroupMetadata>,
11836}
11837
11838impl ChildWorkflowCall {
11839    fn poll_avro_value(
11840        mut self: Pin<&mut Self>,
11841        _cx: &mut TaskContext<'_>,
11842    ) -> Poll<Result<ChildWorkflowAvroResult>> {
11843        if self.matched_pending {
11844            return Poll::Pending;
11845        }
11846
11847        let ctx = self.ctx.clone();
11848        let mut state = match ctx.state.lock() {
11849            Ok(state) => state,
11850            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11851        };
11852
11853        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
11854            let sequence = recorded.sequence();
11855            match recorded {
11856                RecordedCommand::ChildWorkflow {
11857                    workflow_type,
11858                    outcome,
11859                    parallel_group_path,
11860                    ..
11861                } => {
11862                    if let Err(error) = ensure_parallel_path_matches(
11863                        sequence,
11864                        parallel_group_path.as_deref(),
11865                        &self.parallel_group_path,
11866                    ) {
11867                        return Poll::Ready(Err(error));
11868                    }
11869                    if let Some(recorded_type) = workflow_type {
11870                        if recorded_type != self.workflow_type {
11871                            return Poll::Ready(Err(Error::NonDeterministicReplay(
11872                                ReplayFailure::new(
11873                                    "recorded_command_detail_mismatch",
11874                                    Some(sequence),
11875                                    Some(format!("child workflow:{recorded_type}")),
11876                                    Some(format!("child workflow:{}", self.workflow_type)),
11877                                    "recorded child workflow type differs from the current workflow command",
11878                                ),
11879                            )));
11880                        }
11881                    }
11882                    state.command_cursor += 1;
11883                    if let Some(outcome) = outcome {
11884                        return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed));
11885                    }
11886                    state.matched_recorded_pending = true;
11887                    self.scheduled = true;
11888                    self.matched_pending = true;
11889                    return Poll::Pending;
11890                }
11891                other => {
11892                    return Poll::Ready(Err(command_mismatch(
11893                        &other,
11894                        format!("child workflow:{}", self.workflow_type),
11895                    )));
11896                }
11897            }
11898        }
11899
11900        if !self.scheduled {
11901            if self.options.task_queue.trim().is_empty() {
11902                return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
11903                    "task_queue must not be empty".to_string(),
11904                )));
11905            }
11906            for (name, value) in [
11907                (
11908                    "execution_timeout_seconds",
11909                    self.options.execution_timeout_seconds,
11910                ),
11911                ("run_timeout_seconds", self.options.run_timeout_seconds),
11912            ] {
11913                if value == Some(0) {
11914                    return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(format!(
11915                        "{name} must be at least 1"
11916                    ))));
11917                }
11918            }
11919
11920            let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
11921                Ok(args) => args,
11922                Err(error) => return Poll::Ready(Err(error)),
11923            };
11924            let arguments = match encode_typed_envelope(
11925                &normalize_avro_arguments(args),
11926                &state.payload_codec,
11927            ) {
11928                Ok(arguments) => arguments,
11929                Err(error) => return Poll::Ready(Err(error)),
11930            };
11931            let mut command = json!({
11932                "type": "start_child_workflow",
11933                "workflow_type": self.workflow_type,
11934                "queue": self.options.task_queue,
11935                "parent_close_policy": self.options.parent_close_policy.as_str(),
11936                "arguments": arguments,
11937            });
11938            let object = command
11939                .as_object_mut()
11940                .expect("child workflow command is always an object");
11941            if let Some(policy) = &self.options.retry_policy {
11942                let mut retry_policy = serde_json::Map::new();
11943                if let Some(max_attempts) = policy.max_attempts {
11944                    if max_attempts == 0 {
11945                        return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
11946                            "retry_policy.max_attempts must be at least 1".to_string(),
11947                        )));
11948                    }
11949                    retry_policy.insert("max_attempts".to_string(), json!(max_attempts));
11950                }
11951                if !policy.backoff_seconds.is_empty() {
11952                    retry_policy
11953                        .insert("backoff_seconds".to_string(), json!(policy.backoff_seconds));
11954                }
11955                if !policy.non_retryable_error_types.is_empty() {
11956                    retry_policy.insert(
11957                        "non_retryable_error_types".to_string(),
11958                        json!(policy.non_retryable_error_types),
11959                    );
11960                }
11961                if retry_policy.is_empty() {
11962                    return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
11963                        "retry_policy must configure at least one field".to_string(),
11964                    )));
11965                }
11966                object.insert("retry_policy".to_string(), Value::Object(retry_policy));
11967            }
11968            if let Some(seconds) = self.options.execution_timeout_seconds {
11969                object.insert("execution_timeout_seconds".to_string(), json!(seconds));
11970            }
11971            if let Some(seconds) = self.options.run_timeout_seconds {
11972                object.insert("run_timeout_seconds".to_string(), json!(seconds));
11973            }
11974            apply_parallel_group_path(object, &self.parallel_group_path);
11975            state.commands.push(command);
11976            self.scheduled = true;
11977        }
11978
11979        Poll::Pending
11980    }
11981}
11982
11983impl Future for ChildWorkflowCall {
11984    type Output = Result<ChildWorkflowResult>;
11985
11986    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11987        match self.poll_avro_value(cx) {
11988            Poll::Ready(Ok(result)) => match result.result.into_json() {
11989                Ok(projected) => Poll::Ready(Ok(ChildWorkflowResult {
11990                    parent: result.parent,
11991                    child: result.child,
11992                    child_workflow_type: result.child_workflow_type,
11993                    result: projected,
11994                })),
11995                Err(error) => Poll::Ready(Err(error)),
11996            },
11997            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
11998            Poll::Pending => Poll::Pending,
11999        }
12000    }
12001}
12002
12003fn command_mismatch(recorded: &RecordedCommand, actual: impl Into<String>) -> Error {
12004    Error::NonDeterministicReplay(ReplayFailure::new(
12005        "recorded_command_mismatch",
12006        Some(recorded.sequence()),
12007        Some(recorded.shape().to_string()),
12008        Some(actual.into()),
12009        "current workflow command does not match the recorded durable command sequence",
12010    ))
12011}
12012
12013pub struct SignalCall {
12014    ctx: WorkflowContext,
12015    signal_name: String,
12016    runtime_reserved_allowed: bool,
12017    opened_wait: bool,
12018    matched_pending: bool,
12019    parallel_group_path: Vec<ParallelGroupMetadata>,
12020}
12021
12022impl SignalCall {
12023    fn poll_avro_value(
12024        mut self: Pin<&mut Self>,
12025        _cx: &mut TaskContext<'_>,
12026    ) -> Poll<Result<Vec<AvroValue>>> {
12027        if self.matched_pending {
12028            return Poll::Pending;
12029        }
12030        if !self.runtime_reserved_allowed {
12031            if let Err(error) = validate_user_signal_name(&self.signal_name) {
12032                return Poll::Ready(Err(error));
12033            }
12034        }
12035
12036        let ctx = self.ctx.clone();
12037        let mut state = match ctx.state.lock() {
12038            Ok(state) => state,
12039            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12040        };
12041
12042        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
12043            match recorded {
12044                RecordedCommand::SignalWait {
12045                    sequence,
12046                    signal_name,
12047                    value,
12048                    parallel_group_path,
12049                } => {
12050                    if let Err(error) = ensure_parallel_path_matches(
12051                        sequence,
12052                        parallel_group_path.as_deref(),
12053                        &self.parallel_group_path,
12054                    ) {
12055                        return Poll::Ready(Err(error));
12056                    }
12057                    if signal_name != self.signal_name {
12058                        return Poll::Ready(Err(Error::NonDeterministicReplay(
12059                            ReplayFailure::new(
12060                                "recorded_command_detail_mismatch",
12061                                Some(sequence),
12062                                Some(format!("signal wait:{signal_name}")),
12063                                Some(format!("signal wait:{}", self.signal_name)),
12064                                "recorded signal name differs from the current workflow command",
12065                            ),
12066                        )));
12067                    }
12068
12069                    state.command_cursor += 1;
12070                    if let Some(value) = value {
12071                        return Poll::Ready(Ok(value));
12072                    }
12073                    if state
12074                        .resume_signal
12075                        .as_ref()
12076                        .is_some_and(|signal| signal.signal_name == self.signal_name)
12077                    {
12078                        let signal = state
12079                            .resume_signal
12080                            .take()
12081                            .expect("matching resume signal is present");
12082                        return Poll::Ready(Ok(signal.arguments));
12083                    }
12084
12085                    state.matched_recorded_pending = true;
12086                    self.opened_wait = true;
12087                    self.matched_pending = true;
12088                    return Poll::Pending;
12089                }
12090                other => {
12091                    return Poll::Ready(Err(command_mismatch(
12092                        &other,
12093                        format!("signal wait:{}", self.signal_name),
12094                    )));
12095                }
12096            }
12097        }
12098
12099        if state
12100            .resume_signal
12101            .as_ref()
12102            .is_some_and(|signal| signal.signal_name == self.signal_name)
12103        {
12104            let signal = state
12105                .resume_signal
12106                .take()
12107                .expect("matching resume signal is present");
12108            return Poll::Ready(Ok(signal.arguments));
12109        }
12110
12111        if !self.opened_wait {
12112            let mut command = serde_json::Map::from_iter([
12113                ("type".to_string(), json!("open_signal_wait")),
12114                ("signal_name".to_string(), json!(self.signal_name)),
12115            ]);
12116            apply_parallel_group_path(&mut command, &self.parallel_group_path);
12117            state.commands.push(Value::Object(command));
12118            self.opened_wait = true;
12119        }
12120
12121        Poll::Pending
12122    }
12123}
12124
12125impl Future for SignalCall {
12126    type Output = Result<Vec<Value>>;
12127
12128    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12129        match self.poll_avro_value(cx) {
12130            Poll::Ready(Ok(values)) => Poll::Ready(
12131                values
12132                    .into_iter()
12133                    .map(AvroValue::into_json)
12134                    .collect::<Result<Vec<_>>>(),
12135            ),
12136            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
12137            Poll::Pending => Poll::Pending,
12138        }
12139    }
12140}
12141
12142#[derive(Clone, Debug)]
12143pub struct ActivityContext {
12144    client: Client,
12145    pub task_id: String,
12146    pub activity_attempt_id: String,
12147    pub lease_owner: String,
12148    pub activity_type: String,
12149    pub attempt_number: u64,
12150    pub task_queue: String,
12151    pub worker_id: String,
12152}
12153
12154impl ActivityContext {
12155    pub async fn heartbeat<T: Serialize>(&self, details: T) -> Result<ActivityHeartbeatResponse> {
12156        self.client
12157            .heartbeat_activity_task(
12158                &self.task_id,
12159                &self.activity_attempt_id,
12160                &self.lease_owner,
12161                details,
12162            )
12163            .await
12164    }
12165}
12166
12167fn decode_task_avro_arguments(value: Option<&Value>, codec: &str) -> Result<AvroValue> {
12168    validate_payload_codec(codec)?;
12169    match value {
12170        Some(value) => Ok(normalize_avro_arguments(decode_wire_avro_value(
12171            value, codec,
12172        )?)),
12173        None => Ok(AvroValue::Array(Vec::new())),
12174    }
12175}
12176
12177fn decode_resume_signal(task: &WorkflowTask) -> Result<Option<ResumeSignal>> {
12178    let Some(signal_name) = task
12179        .signal_name
12180        .as_deref()
12181        .filter(|value| !value.is_empty())
12182    else {
12183        return Ok(None);
12184    };
12185    let decoded = decode_task_avro_arguments(task.signal_arguments.as_ref(), &task.payload_codec)?;
12186    let AvroValue::Array(arguments) = decoded else {
12187        unreachable!("normalize_avro_arguments always returns an array");
12188    };
12189
12190    Ok(Some(ResumeSignal {
12191        signal_name: signal_name.to_string(),
12192        arguments,
12193    }))
12194}
12195
12196fn validate_workflow_task_payloads(task: &WorkflowTask) -> Result<()> {
12197    validate_payload_codec(&task.payload_codec)?;
12198    validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)?;
12199    validate_optional_inbound_payload(task.signal_arguments.as_ref(), &task.payload_codec)?;
12200    for event in &task.history_events {
12201        validate_history_event_payloads(event, &task.payload_codec)?;
12202    }
12203    Ok(())
12204}
12205
12206fn validate_activity_task_payloads(task: &ActivityTask) -> Result<()> {
12207    validate_payload_codec(&task.payload_codec)?;
12208    validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)
12209}
12210
12211fn validate_query_task_payloads(task: &QueryTask) -> Result<()> {
12212    validate_payload_codec(&task.payload_codec)?;
12213    validate_optional_inbound_payload(task.workflow_arguments.as_ref(), &task.payload_codec)?;
12214    validate_optional_inbound_payload(task.query_arguments.as_ref(), &task.payload_codec)?;
12215    for event in &task.history_events {
12216        validate_history_event_payloads(event, &task.payload_codec)?;
12217    }
12218
12219    let Some(export) = task.history_export.as_ref() else {
12220        return Ok(());
12221    };
12222    let export_codec = match export.get("payloads") {
12223        Some(payloads) => declared_payload_codec(payloads, "codec")?,
12224        None => None,
12225    }
12226    .unwrap_or(&task.payload_codec);
12227    validate_payload_codec(export_codec)?;
12228
12229    if let Some(events) = export.get("history_events").and_then(Value::as_array) {
12230        for event in events {
12231            let event_type = event
12232                .get("event_type")
12233                .or_else(|| event.get("type"))
12234                .and_then(Value::as_str)
12235                .unwrap_or_default();
12236            if let Some(payload) = event.get("payload") {
12237                validate_history_payloads(event_type, payload, export_codec)?;
12238            }
12239        }
12240    }
12241    for signal in export
12242        .get("signals")
12243        .and_then(Value::as_array)
12244        .into_iter()
12245        .flatten()
12246    {
12247        let codec = declared_payload_codec(signal, "payload_codec")?.unwrap_or(export_codec);
12248        validate_payload_codec(codec)?;
12249        validate_optional_inbound_payload(signal.get("arguments"), codec)?;
12250    }
12251    for activity in export
12252        .get("activities")
12253        .and_then(Value::as_array)
12254        .into_iter()
12255        .flatten()
12256    {
12257        let codec = declared_payload_codec(activity, "payload_codec")?.unwrap_or(export_codec);
12258        validate_payload_codec(codec)?;
12259        validate_optional_inbound_payload(activity.get("arguments"), codec)?;
12260        validate_optional_inbound_payload(activity.get("result"), codec)?;
12261    }
12262    Ok(())
12263}
12264
12265fn validate_history_event_payloads(event: &HistoryEvent, fallback_codec: &str) -> Result<()> {
12266    validate_history_payloads(&event.event_type, &event.payload, fallback_codec)
12267}
12268
12269fn validate_history_payloads(
12270    event_type: &str,
12271    payload: &Value,
12272    fallback_codec: &str,
12273) -> Result<()> {
12274    let codec = declared_payload_codec(payload, "payload_codec")?.unwrap_or(fallback_codec);
12275    validate_payload_codec(codec)?;
12276    for field in history_payload_fields(event_type) {
12277        validate_optional_inbound_payload(payload.get(*field), codec)?;
12278    }
12279    Ok(())
12280}
12281
12282const SIGNAL_HISTORY_PAYLOAD_FIELDS: &[&str] = &["value", "input", "arguments"];
12283
12284fn history_payload_fields(event_type: &str) -> &'static [&'static str] {
12285    match event_type {
12286        "ActivityCompleted" => &["result"],
12287        "SignalReceived" | "SignalApplied" => SIGNAL_HISTORY_PAYLOAD_FIELDS,
12288        "UpdateAccepted" | "UpdateRejected" | "UpdateApplied" => &["arguments"],
12289        "UpdateCompleted" | "SideEffectRecorded" => &["result"],
12290        "ChildRunCompleted" => &["result", "output"],
12291        "WorkflowCompleted" => &["output"],
12292        "ServiceCallStarted"
12293        | "ServiceCallCompleted"
12294        | "ServiceCallFailed"
12295        | "ServiceCallCancelled" => &["request_payload", "response_payload"],
12296        _ => &[],
12297    }
12298}
12299
12300fn signal_history_payload(payload: &Value) -> Option<&Value> {
12301    SIGNAL_HISTORY_PAYLOAD_FIELDS
12302        .iter()
12303        .find_map(|field| payload.get(*field))
12304}
12305
12306fn declared_payload_codec<'a>(value: &'a Value, field: &str) -> Result<Option<&'a str>> {
12307    match value.get(field) {
12308        None => Ok(None),
12309        Some(Value::String(codec)) => Ok(Some(codec)),
12310        Some(_) => Err(invalid_payload_envelope()),
12311    }
12312}
12313
12314fn validate_optional_inbound_payload(value: Option<&Value>, codec: &str) -> Result<()> {
12315    validate_payload_codec(codec)?;
12316    if let Some(value) = value.filter(|value| !value.is_null()) {
12317        decode_wire_avro_value(value, codec)?;
12318    }
12319    Ok(())
12320}
12321
12322fn recorded_parallel_group_entry(payload: &Value, sequence: u64) -> Result<ParallelGroupMetadata> {
12323    let group_id = payload_string(payload, "parallel_group_id").ok_or_else(|| {
12324        invalid_recorded_history(
12325            "parallel_group_metadata_invalid",
12326            sequence,
12327            "non-empty parallel_group_id",
12328            &payload.to_string(),
12329            "parallel-group history is missing its stable identity",
12330        )
12331    })?;
12332    let kind = payload_string(payload, "parallel_group_kind").ok_or_else(|| {
12333        invalid_recorded_history(
12334            "parallel_group_metadata_invalid",
12335            sequence,
12336            "activity, child, timer, signal, condition, or mixed group kind",
12337            &payload.to_string(),
12338            "parallel-group history is missing its group kind",
12339        )
12340    })?;
12341    if !matches!(
12342        kind.as_str(),
12343        "activity" | "child" | "timer" | "signal" | "condition" | "mixed"
12344    ) {
12345        return Err(invalid_recorded_history(
12346            "parallel_group_metadata_invalid",
12347            sequence,
12348            "activity, child, timer, signal, condition, or mixed group kind",
12349            &kind,
12350            "parallel-group history contains an unsupported group kind",
12351        ));
12352    }
12353    let base_sequence = payload
12354        .get("parallel_group_base_sequence")
12355        .and_then(value_as_u64)
12356        .filter(|value| *value > 0)
12357        .ok_or_else(|| {
12358            invalid_recorded_history(
12359                "parallel_group_metadata_invalid",
12360                sequence,
12361                "positive parallel_group_base_sequence",
12362                &payload.to_string(),
12363                "parallel-group history contains an invalid base sequence",
12364            )
12365        })?;
12366    let size = payload
12367        .get("parallel_group_size")
12368        .and_then(value_as_u64)
12369        .and_then(|value| usize::try_from(value).ok())
12370        .filter(|value| (1..=MAX_PARALLEL_OPERATIONS).contains(value))
12371        .ok_or_else(|| {
12372            invalid_recorded_history(
12373                "parallel_group_metadata_invalid",
12374                sequence,
12375                "bounded positive parallel_group_size",
12376                &payload.to_string(),
12377                "parallel-group history contains an invalid group size",
12378            )
12379        })?;
12380    let index = payload
12381        .get("parallel_group_index")
12382        .and_then(value_as_u64)
12383        .and_then(|value| usize::try_from(value).ok())
12384        .filter(|value| *value < size)
12385        .ok_or_else(|| {
12386            invalid_recorded_history(
12387                "parallel_group_metadata_invalid",
12388                sequence,
12389                "parallel_group_index within group bounds",
12390                &payload.to_string(),
12391                "parallel-group history contains an invalid member index",
12392            )
12393        })?;
12394    if base_sequence.checked_add(u64::try_from(index).unwrap_or(u64::MAX)) != Some(sequence) {
12395        return Err(invalid_recorded_history(
12396            "parallel_group_metadata_invalid",
12397            sequence,
12398            "base sequence plus member index equals workflow sequence",
12399            &payload.to_string(),
12400            "parallel-group path does not preserve durable workflow position",
12401        ));
12402    }
12403    let mode = payload
12404        .get("parallel_group_mode")
12405        .and_then(Value::as_str)
12406        .unwrap_or("all");
12407    if !matches!(mode, "all" | "select") {
12408        return Err(invalid_recorded_history(
12409            "parallel_group_metadata_invalid",
12410            sequence,
12411            "parallel group mode all or select",
12412            mode,
12413            "parallel-group history contains an unsupported group mode",
12414        ));
12415    }
12416    let expected_id = if mode == "select" {
12417        format!("select-calls:{base_sequence}:{size}")
12418    } else {
12419        format!("{}:{base_sequence}:{size}", parallel_group_prefix(&kind))
12420    };
12421    if group_id != expected_id {
12422        return Err(invalid_recorded_history(
12423            "parallel_group_metadata_invalid",
12424            sequence,
12425            &expected_id,
12426            &group_id,
12427            "parallel-group history contains an incompatible stable group ID",
12428        ));
12429    }
12430    let selection_member_key = if mode == "select" {
12431        Some(selection_key_from_value(
12432            payload.get("selection_member_key"),
12433            sequence,
12434        )?)
12435    } else {
12436        None
12437    };
12438    let selection_member_index = if mode == "select" {
12439        Some(required_parallel_usize(
12440            payload,
12441            "selection_member_index",
12442            sequence,
12443        )?)
12444    } else {
12445        None
12446    };
12447    let selection_member_base_sequence = if mode == "select" {
12448        Some(
12449            payload
12450                .get("selection_member_base_sequence")
12451                .and_then(value_as_u64)
12452                .filter(|value| *value >= base_sequence)
12453                .ok_or_else(|| {
12454                    invalid_recorded_history(
12455                        "parallel_group_metadata_invalid",
12456                        sequence,
12457                        "selection member base within its group",
12458                        &payload.to_string(),
12459                        "selection history contains an invalid member base sequence",
12460                    )
12461                })?,
12462        )
12463    } else {
12464        None
12465    };
12466    let selection_member_size = if mode == "select" {
12467        let member_size = required_parallel_usize(payload, "selection_member_size", sequence)?;
12468        if member_size == 0 {
12469            return Err(invalid_recorded_history(
12470                "parallel_group_metadata_invalid",
12471                sequence,
12472                "positive selection member size",
12473                &payload.to_string(),
12474                "selection history contains an invalid member size",
12475            ));
12476        }
12477        Some(member_size)
12478    } else {
12479        None
12480    };
12481    let selection_member_kind = if mode == "select" {
12482        let kind = payload_string(payload, "selection_member_kind").ok_or_else(|| {
12483            invalid_recorded_history(
12484                "parallel_group_metadata_invalid",
12485                sequence,
12486                "selection member operation kind",
12487                &payload.to_string(),
12488                "selection history is missing its authored member kind",
12489            )
12490        })?;
12491        if !matches!(
12492            kind.as_str(),
12493            "activity" | "child" | "timer" | "signal" | "condition" | "group"
12494        ) {
12495            return Err(invalid_recorded_history(
12496                "parallel_group_metadata_invalid",
12497                sequence,
12498                "activity, child, timer, signal, condition, or group selection member kind",
12499                &kind,
12500                "selection history contains an unsupported member kind",
12501            ));
12502        }
12503        Some(kind)
12504    } else {
12505        None
12506    };
12507    if let (Some(member_base), Some(member_size)) =
12508        (selection_member_base_sequence, selection_member_size)
12509    {
12510        let member_end = member_base
12511            .checked_add(u64::try_from(member_size).unwrap_or(u64::MAX))
12512            .ok_or_else(|| {
12513                invalid_recorded_history(
12514                    "parallel_group_metadata_invalid",
12515                    sequence,
12516                    "bounded selection member range",
12517                    &payload.to_string(),
12518                    "selection member range overflowed",
12519                )
12520            })?;
12521        let group_end = base_sequence
12522            .checked_add(u64::try_from(size).unwrap_or(u64::MAX))
12523            .unwrap_or(u64::MAX);
12524        if sequence < member_base || sequence >= member_end || member_end > group_end {
12525            return Err(invalid_recorded_history(
12526                "parallel_group_metadata_invalid",
12527                sequence,
12528                "workflow sequence within one bounded selection member",
12529                &payload.to_string(),
12530                "selection member range does not contain its durable leaf",
12531            ));
12532        }
12533    }
12534    Ok(ParallelGroupMetadata {
12535        parallel_group_id: group_id,
12536        parallel_group_kind: kind,
12537        parallel_group_base_sequence: base_sequence,
12538        parallel_group_size: size,
12539        parallel_group_index: index,
12540        parallel_group_mode: (mode == "select").then(|| "select".to_string()),
12541        selection_member_key,
12542        selection_member_index,
12543        selection_member_base_sequence,
12544        selection_member_size,
12545        selection_member_kind,
12546    })
12547}
12548
12549fn required_parallel_usize(payload: &Value, field: &str, sequence: u64) -> Result<usize> {
12550    payload
12551        .get(field)
12552        .and_then(value_as_u64)
12553        .and_then(|value| usize::try_from(value).ok())
12554        .ok_or_else(|| {
12555            invalid_recorded_history(
12556                "parallel_group_metadata_invalid",
12557                sequence,
12558                &format!("non-negative integer {field}"),
12559                &payload.to_string(),
12560                "selection history contains invalid member metadata",
12561            )
12562        })
12563}
12564
12565fn selection_key_from_value(value: Option<&Value>, sequence: u64) -> Result<SelectionKey> {
12566    match value {
12567        Some(Value::String(value)) if !value.is_empty() => Ok(SelectionKey::Name(value.clone())),
12568        Some(value) => value_as_u64(value)
12569            .and_then(|value| usize::try_from(value).ok())
12570            .map(SelectionKey::Index)
12571            .ok_or_else(|| {
12572                invalid_recorded_history(
12573                    "selection_member_key_invalid",
12574                    sequence,
12575                    "non-empty string or non-negative integer member key",
12576                    &value.to_string(),
12577                    "selection history contains an invalid member key",
12578                )
12579            }),
12580        None => Err(invalid_recorded_history(
12581            "selection_member_key_missing",
12582            sequence,
12583            "selection_member_key",
12584            "<missing>",
12585            "selection history is missing its stable member key",
12586        )),
12587    }
12588}
12589
12590fn recorded_parallel_group_path(
12591    events: &[&HistoryEvent],
12592    sequence: u64,
12593) -> Result<Option<Vec<ParallelGroupMetadata>>> {
12594    let mut recorded: Option<Vec<ParallelGroupMetadata>> = None;
12595    for event in events {
12596        let payload = &event.payload;
12597        let has_metadata = payload.get("parallel_group_path").is_some()
12598            || payload.get("parallel_group_id").is_some()
12599            || payload.get("parallel_group_kind").is_some()
12600            || payload.get("parallel_group_base_sequence").is_some()
12601            || payload.get("parallel_group_size").is_some()
12602            || payload.get("parallel_group_index").is_some()
12603            || payload.get("parallel_group_mode").is_some()
12604            || payload.get("selection_member_key").is_some();
12605        if !has_metadata {
12606            continue;
12607        }
12608
12609        let top_level = recorded_parallel_group_entry(payload, sequence)?;
12610        let path = match payload.get("parallel_group_path") {
12611            None => vec![top_level.clone()],
12612            Some(Value::Array(entries)) if !entries.is_empty() => entries
12613                .iter()
12614                .map(|entry| recorded_parallel_group_entry(entry, sequence))
12615                .collect::<Result<Vec<_>>>()?,
12616            Some(value) => {
12617                return Err(invalid_recorded_history(
12618                    "parallel_group_metadata_invalid",
12619                    sequence,
12620                    "non-empty parallel_group_path list",
12621                    &value.to_string(),
12622                    "parallel-group history contains an invalid group path",
12623                ));
12624            }
12625        };
12626        if path.last() != Some(&top_level) {
12627            return Err(invalid_recorded_history(
12628                "parallel_group_metadata_invalid",
12629                sequence,
12630                &serde_json::to_string(&path.last()).unwrap_or_default(),
12631                &serde_json::to_string(&top_level).unwrap_or_default(),
12632                "parallel-group top-level fields do not match the innermost path entry",
12633            ));
12634        }
12635        if recorded.as_ref().is_some_and(|existing| existing != &path) {
12636            return Err(invalid_recorded_history(
12637                "parallel_group_history_conflict",
12638                sequence,
12639                &serde_json::to_string(&recorded.as_ref()).unwrap_or_default(),
12640                &serde_json::to_string(&path).unwrap_or_default(),
12641                "parallel-group metadata changed between scheduling and resolution history",
12642            ));
12643        }
12644        recorded = Some(path);
12645    }
12646    Ok(recorded)
12647}
12648
12649fn recorded_commands(
12650    events: &[HistoryEvent],
12651    fallback_codec: &str,
12652    parent: WorkflowIdentity,
12653) -> Result<Vec<RecordedCommand>> {
12654    let mut events_by_sequence: BTreeMap<u64, Vec<&HistoryEvent>> = BTreeMap::new();
12655    let mut last_new_sequence = None;
12656
12657    for event in events {
12658        let is_activity = matches!(
12659            event.event_type.as_str(),
12660            "ActivityScheduled"
12661                | "ActivityStarted"
12662                | "ActivityHeartbeatRecorded"
12663                | "ActivityRetryScheduled"
12664                | "ActivityCompleted"
12665                | "ActivityFailed"
12666                | "ActivityCancelled"
12667                | "ActivityTimedOut"
12668        );
12669        let is_workflow_timer = matches!(
12670            event.event_type.as_str(),
12671            "TimerScheduled" | "TimerCancelled" | "TimerFired"
12672        ) && !is_internal_timer_event(event);
12673        let is_child_workflow = matches!(
12674            event.event_type.as_str(),
12675            "ChildWorkflowScheduled"
12676                | "ChildRunCompleted"
12677                | "ChildRunFailed"
12678                | "ChildRunCancelled"
12679                | "ChildRunTerminated"
12680        );
12681        let is_signal_wait = is_recorded_signal_wait_event(event);
12682        let is_condition_wait = is_recorded_condition_wait_event(event);
12683        let is_search_attributes = event.event_type == "SearchAttributesUpserted";
12684        let is_side_effect = event.event_type == "SideEffectRecorded";
12685        let is_version_marker = event.event_type == "VersionMarkerRecorded";
12686        let is_memo = event.event_type == "MemoUpserted";
12687        if !is_activity
12688            && !is_workflow_timer
12689            && !is_child_workflow
12690            && !is_signal_wait
12691            && !is_condition_wait
12692            && !is_search_attributes
12693            && !is_side_effect
12694            && !is_version_marker
12695            && !is_memo
12696        {
12697            continue;
12698        }
12699
12700        let sequence = durable_event_sequence(event).ok_or_else(|| {
12701            Error::NonDeterministicReplay(ReplayFailure::new(
12702                "durable_command_sequence_missing",
12703                None,
12704                Some("positive workflow sequence".to_string()),
12705                Some(event.event_type.clone()),
12706                "durable command history event has no workflow sequence",
12707            ))
12708        })?;
12709        if sequence == 0 {
12710            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12711                "durable_command_sequence_invalid",
12712                Some(sequence),
12713                Some("positive workflow sequence".to_string()),
12714                Some(sequence.to_string()),
12715                "durable command history uses an invalid workflow sequence",
12716            )));
12717        }
12718        if !events_by_sequence.contains_key(&sequence) {
12719            if let Some(previous) = last_new_sequence {
12720                if sequence < previous {
12721                    return Err(invalid_recorded_history(
12722                        "durable_command_sequence_mismatch",
12723                        sequence,
12724                        &format!("workflow sequence greater than {previous}"),
12725                        &sequence.to_string(),
12726                        "durable commands are not strictly ordered by their recorded workflow sequence",
12727                    ));
12728                }
12729            }
12730            last_new_sequence = Some(sequence);
12731        }
12732        events_by_sequence.entry(sequence).or_default().push(event);
12733    }
12734
12735    let commands: Vec<RecordedCommand> = events_by_sequence
12736        .into_iter()
12737        .map(|(sequence, sequence_events)| {
12738            let activity_events: Vec<_> = sequence_events
12739                .iter()
12740                .copied()
12741                .filter(|event| event.event_type.starts_with("Activity"))
12742                .collect();
12743            let timer_events: Vec<_> = sequence_events
12744                .iter()
12745                .copied()
12746                .filter(|event| event.event_type.starts_with("Timer"))
12747                .collect();
12748            let child_events: Vec<_> = sequence_events
12749                .iter()
12750                .copied()
12751                .filter(|event| {
12752                    event.event_type == "ChildWorkflowScheduled"
12753                        || event.event_type.starts_with("ChildRun")
12754                })
12755                .collect();
12756            let signal_wait_events: Vec<_> = sequence_events
12757                .iter()
12758                .copied()
12759                .filter(|event| is_recorded_signal_wait_event(event))
12760                .collect();
12761            let condition_wait_events: Vec<_> = sequence_events
12762                .iter()
12763                .copied()
12764                .filter(|event| is_recorded_condition_wait_event(event))
12765                .collect();
12766            let search_attribute_events: Vec<_> = sequence_events
12767                .iter()
12768                .copied()
12769                .filter(|event| event.event_type == "SearchAttributesUpserted")
12770                .collect();
12771            let side_effect_events: Vec<_> = sequence_events
12772                .iter()
12773                .copied()
12774                .filter(|event| event.event_type == "SideEffectRecorded")
12775                .collect();
12776            let version_marker_events: Vec<_> = sequence_events
12777                .iter()
12778                .copied()
12779                .filter(|event| event.event_type == "VersionMarkerRecorded")
12780                .collect();
12781            let memo_events: Vec<_> = sequence_events
12782                .iter()
12783                .copied()
12784                .filter(|event| event.event_type == "MemoUpserted")
12785                .collect();
12786
12787            let command_kind_count = usize::from(!activity_events.is_empty())
12788                + usize::from(!timer_events.is_empty())
12789                + usize::from(!child_events.is_empty())
12790                + usize::from(!signal_wait_events.is_empty())
12791                + usize::from(!condition_wait_events.is_empty())
12792                + usize::from(!search_attribute_events.is_empty())
12793                + usize::from(!side_effect_events.is_empty())
12794                + usize::from(!version_marker_events.is_empty())
12795                + usize::from(!memo_events.is_empty());
12796            if command_kind_count > 1 {
12797                let actual = [
12798                    (!activity_events.is_empty()).then_some("activity"),
12799                    (!timer_events.is_empty()).then_some("timer"),
12800                    (!child_events.is_empty()).then_some("child workflow"),
12801                    (!signal_wait_events.is_empty()).then_some("signal wait"),
12802                    (!condition_wait_events.is_empty()).then_some("condition wait"),
12803                    (!search_attribute_events.is_empty()).then_some("search-attribute update"),
12804                    (!side_effect_events.is_empty()).then_some("side effect"),
12805                    (!version_marker_events.is_empty()).then_some("version marker"),
12806                    (!memo_events.is_empty()).then_some("memo upsert"),
12807                ]
12808                .into_iter()
12809                .flatten()
12810                .collect::<Vec<_>>()
12811                .join(" and ");
12812                return Err(invalid_recorded_history(
12813                    "durable_command_sequence_collision",
12814                    sequence,
12815                    "one durable command kind",
12816                    &actual,
12817                    "one workflow sequence records more than one durable command kind",
12818                ));
12819            }
12820
12821            if !activity_events.is_empty() {
12822                let parallel_group_path =
12823                    recorded_parallel_group_path(&activity_events, sequence)?;
12824                let scheduled_count = activity_events
12825                    .iter()
12826                    .filter(|event| event.event_type == "ActivityScheduled")
12827                    .count();
12828                if scheduled_count > 1 {
12829                    return Err(invalid_recorded_history(
12830                        "duplicate_activity_schedule",
12831                        sequence,
12832                        "at most one ActivityScheduled event",
12833                        "multiple ActivityScheduled events",
12834                        "activity history schedules more than one command at one workflow sequence",
12835                    ));
12836                }
12837                let activity_type = activity_events.iter().find_map(|event| {
12838                    event
12839                        .payload
12840                        .get("activity_type")
12841                        .or_else(|| event.payload.get("activity_name"))
12842                        .and_then(Value::as_str)
12843                        .map(str::to_string)
12844                });
12845                if activity_events.iter().filter_map(|event| {
12846                    event
12847                        .payload
12848                        .get("activity_type")
12849                        .or_else(|| event.payload.get("activity_name"))
12850                        .and_then(Value::as_str)
12851                }).any(|candidate| Some(candidate) != activity_type.as_deref()) {
12852                    return Err(invalid_recorded_history(
12853                        "activity_identity_mismatch",
12854                        sequence,
12855                        activity_type.as_deref().unwrap_or("one activity identity"),
12856                        "conflicting activity identities",
12857                        "activity lifecycle events at one workflow sequence disagree on identity",
12858                    ));
12859                }
12860                let terminal: Vec<_> = activity_events
12861                    .iter()
12862                    .copied()
12863                    .filter(|event| {
12864                        matches!(
12865                            event.event_type.as_str(),
12866                            "ActivityCompleted"
12867                                | "ActivityFailed"
12868                                | "ActivityCancelled"
12869                                | "ActivityTimedOut"
12870                        )
12871                    })
12872                    .collect();
12873                let duplicate_delivery = terminal.first().is_some_and(|first| {
12874                    terminal.iter().all(|event| {
12875                        event.event_type == first.event_type && event.payload == first.payload
12876                    })
12877                });
12878                if terminal.len() > 1 && !duplicate_delivery {
12879                    return Err(invalid_recorded_history(
12880                        "duplicate_activity_terminal_event",
12881                        sequence,
12882                        "at most one terminal activity event",
12883                        "multiple terminal activity events",
12884                        "activity history settles one command more than once",
12885                    ));
12886                }
12887                let outcome = terminal
12888                    .first()
12889                    .map(|event| activity_outcome(event, fallback_codec, activity_type.clone()))
12890                    .transpose()?;
12891                let options = activity_events
12892                    .iter()
12893                    .find(|event| event.event_type == "ActivityScheduled")
12894                    .and_then(|event| event.payload.get("activity"))
12895                    .and_then(Value::as_object)
12896                    .map(|activity| RecordedActivityOptions {
12897                        task_queue: recorded_optional_string(activity, "queue"),
12898                        execution_mode: recorded_optional_string(activity, "execution_mode"),
12899                        retry_policy: recorded_activity_retry_snapshot(
12900                            activity.get("retry_policy"),
12901                        ),
12902                    });
12903                return Ok(RecordedCommand::Activity {
12904                    sequence,
12905                    activity_type,
12906                    options,
12907                    outcome,
12908                    parallel_group_path,
12909                });
12910            }
12911
12912            if !child_events.is_empty() {
12913                let parallel_group_path = recorded_parallel_group_path(&child_events, sequence)?;
12914                let scheduled: Vec<_> = child_events
12915                    .iter()
12916                    .copied()
12917                    .filter(|event| event.event_type == "ChildWorkflowScheduled")
12918                    .collect();
12919                if scheduled.len() != 1 {
12920                    return Err(invalid_recorded_history(
12921                        "child_workflow_schedule_missing_or_duplicate",
12922                        sequence,
12923                        "one ChildWorkflowScheduled event",
12924                        &format!("{} ChildWorkflowScheduled events", scheduled.len()),
12925                        "child workflow replay requires exactly one recorded schedule event",
12926                    ));
12927                }
12928                let workflow_type = child_events.iter().find_map(|event| {
12929                    event
12930                        .payload
12931                        .get("child_workflow_type")
12932                        .or_else(|| event.payload.get("workflow_type"))
12933                        .and_then(Value::as_str)
12934                        .filter(|value| !value.is_empty())
12935                        .map(str::to_string)
12936                });
12937                if child_events
12938                    .iter()
12939                    .filter_map(|event| {
12940                        event
12941                            .payload
12942                            .get("child_workflow_type")
12943                            .or_else(|| event.payload.get("workflow_type"))
12944                            .and_then(Value::as_str)
12945                    })
12946                    .any(|candidate| Some(candidate) != workflow_type.as_deref())
12947                {
12948                    return Err(invalid_recorded_history(
12949                        "child_workflow_identity_mismatch",
12950                        sequence,
12951                        workflow_type
12952                            .as_deref()
12953                            .unwrap_or("one child workflow type"),
12954                        "conflicting child workflow types",
12955                        "child workflow lifecycle events at one sequence disagree on type",
12956                    ));
12957                }
12958                let mut outcomes = child_workflow_outcomes(
12959                    &child_events.iter().map(|event| (*event).clone()).collect::<Vec<_>>(),
12960                    fallback_codec,
12961                    parent.clone(),
12962                )?;
12963                let terminal_events = child_events
12964                    .iter()
12965                    .copied()
12966                    .filter(|event| event.event_type.starts_with("ChildRun"))
12967                    .collect::<Vec<_>>();
12968                let duplicate_delivery = terminal_events.first().is_some_and(|first| {
12969                    terminal_events.iter().all(|event| {
12970                        event.event_type == first.event_type && event.payload == first.payload
12971                    })
12972                });
12973                if outcomes.len() > 1 && !duplicate_delivery {
12974                    return Err(invalid_recorded_history(
12975                        "duplicate_child_workflow_terminal_event",
12976                        sequence,
12977                        "at most one terminal child event",
12978                        "multiple terminal child events",
12979                        "child workflow history settles one command more than once",
12980                    ));
12981                }
12982                return Ok(RecordedCommand::ChildWorkflow {
12983                    sequence,
12984                    workflow_type,
12985                    outcome: outcomes.pop(),
12986                    parallel_group_path,
12987                });
12988            }
12989
12990            if !signal_wait_events.is_empty() {
12991                let opened: Vec<_> = signal_wait_events
12992                    .iter()
12993                    .copied()
12994                    .filter(|event| event.event_type == "SignalWaitOpened")
12995                    .collect();
12996                if opened.len() != 1 {
12997                    return Err(invalid_recorded_history(
12998                        "signal_wait_open_missing_or_duplicate",
12999                        sequence,
13000                        "one SignalWaitOpened event",
13001                        &format!("{} SignalWaitOpened events", opened.len()),
13002                        "signal replay requires exactly one canonical wait-open event",
13003                    ));
13004                }
13005
13006                let applied: Vec<_> = signal_wait_events
13007                    .iter()
13008                    .copied()
13009                    .filter(|event| event.event_type == "SignalApplied")
13010                    .collect();
13011                if applied.len() > 1 {
13012                    return Err(invalid_recorded_history(
13013                        "duplicate_signal_wait_apply",
13014                        sequence,
13015                        "at most one SignalApplied event",
13016                        "multiple SignalApplied events",
13017                        "signal history applies one durable wait more than once",
13018                    ));
13019                }
13020
13021                let signal_names = signal_wait_events
13022                    .iter()
13023                    .map(|event| required_signal_wait_name(event, sequence))
13024                    .collect::<Result<Vec<_>>>()?;
13025                let signal_name = signal_names
13026                    .first()
13027                    .expect("signal wait events are not empty")
13028                    .clone();
13029                if signal_names.iter().any(|candidate| candidate != &signal_name) {
13030                    return Err(invalid_recorded_history(
13031                        "signal_wait_identity_mismatch",
13032                        sequence,
13033                        &signal_name,
13034                        "conflicting signal names",
13035                        "signal wait lifecycle events at one workflow sequence disagree on identity",
13036                    ));
13037                }
13038                let value = applied
13039                    .first()
13040                    .map(|event| decode_signal_event_arguments(event, fallback_codec))
13041                    .transpose()?;
13042                return Ok(RecordedCommand::SignalWait {
13043                    sequence,
13044                    signal_name,
13045                    value,
13046                    parallel_group_path: recorded_parallel_group_path(
13047                        &signal_wait_events,
13048                        sequence,
13049                    )?,
13050                });
13051            }
13052
13053            if !condition_wait_events.is_empty() {
13054                return recorded_condition_wait(
13055                    sequence,
13056                    &condition_wait_events,
13057                    events,
13058                );
13059            }
13060
13061            if !search_attribute_events.is_empty() {
13062                if search_attribute_events.len() != 1 {
13063                    return Err(invalid_recorded_history(
13064                        "duplicate_search_attribute_update",
13065                        sequence,
13066                        "one SearchAttributesUpserted event",
13067                        &format!(
13068                            "{} SearchAttributesUpserted events",
13069                            search_attribute_events.len()
13070                        ),
13071                        "search-attribute history records one workflow command more than once",
13072                    ));
13073                }
13074                let payload = &search_attribute_events[0].payload;
13075                let attributes = payload
13076                    .get("attributes")
13077                    .filter(|value| value.as_object().is_some_and(|values| !values.is_empty()))
13078                    .cloned()
13079                    .ok_or_else(|| {
13080                        invalid_recorded_history(
13081                            "search_attribute_update_missing",
13082                            sequence,
13083                            "non-empty attributes object",
13084                            "missing or invalid attributes",
13085                            "search-attribute history is missing its recorded mutation",
13086                        )
13087                    })?;
13088                let attribute_types =
13089                    recorded_search_attribute_types(payload, &attributes, sequence)?;
13090                return Ok(RecordedCommand::SearchAttributes {
13091                    sequence,
13092                    attributes,
13093                    attribute_types,
13094                });
13095            }
13096
13097            if !side_effect_events.is_empty() {
13098                if side_effect_events.len() != 1 {
13099                    return Err(invalid_recorded_history(
13100                        "duplicate_side_effect_record",
13101                        sequence,
13102                        "one SideEffectRecorded event",
13103                        &format!("{} SideEffectRecorded events", side_effect_events.len()),
13104                        "side-effect history records one workflow command more than once",
13105                    ));
13106                }
13107                let event = side_effect_events[0];
13108                let result = event.payload.get("result").ok_or_else(|| {
13109                    invalid_recorded_history(
13110                        "side_effect_result_missing",
13111                        sequence,
13112                        "recorded result payload",
13113                        "missing result",
13114                        "side-effect history is missing its recorded value",
13115                    )
13116                })?;
13117                let has_published_envelope = result.as_str().is_some()
13118                    || result.as_object().is_some_and(|envelope| {
13119                        envelope.get("codec").and_then(Value::as_str).is_some()
13120                            && envelope.get("blob").and_then(Value::as_str).is_some()
13121                    });
13122                if !has_published_envelope {
13123                    return Err(invalid_recorded_history(
13124                        "side_effect_payload_malformed",
13125                        sequence,
13126                        "payload blob or {codec, blob} envelope",
13127                        &result.to_string(),
13128                        "side-effect history result does not use a published payload envelope",
13129                    ));
13130                }
13131                let codec = event
13132                    .payload
13133                    .get("payload_codec")
13134                    .and_then(Value::as_str)
13135                    .unwrap_or(fallback_codec);
13136                let value = decode_wire_avro_value(result, codec).map_err(|error| {
13137                    if error.to_string().contains("unsupported_payload_codec") {
13138                        return error;
13139                    }
13140
13141                    invalid_recorded_history(
13142                        "side_effect_payload_incompatible",
13143                        sequence,
13144                        &format!("valid {codec} payload envelope"),
13145                        &error.to_string(),
13146                        "side-effect history payload cannot be decoded with its recorded codec",
13147                    )
13148                })?;
13149                return Ok(RecordedCommand::SideEffect { sequence, value });
13150            }
13151
13152            if !version_marker_events.is_empty() {
13153                if version_marker_events.len() != 1 {
13154                    return Err(invalid_recorded_history(
13155                        "duplicate_version_marker_record",
13156                        sequence,
13157                        "one VersionMarkerRecorded event",
13158                        &format!("{} VersionMarkerRecorded events", version_marker_events.len()),
13159                        "version-marker history records one workflow command more than once",
13160                    ));
13161                }
13162                let payload = &version_marker_events[0].payload;
13163                let change_id = payload
13164                    .get("change_id")
13165                    .and_then(Value::as_str)
13166                    .filter(|value| !value.is_empty())
13167                    .map(str::to_string)
13168                    .ok_or_else(|| {
13169                        invalid_recorded_history(
13170                            "version_marker_field_missing",
13171                            sequence,
13172                            "non-empty change_id",
13173                            "missing or invalid change_id",
13174                            "version-marker history is missing its stable change ID",
13175                        )
13176                    })?;
13177                let version = required_version_i32(payload, "version", sequence)?;
13178                let min_supported = required_version_i32(payload, "min_supported", sequence)?;
13179                let max_supported = required_version_i32(payload, "max_supported", sequence)?;
13180                if min_supported > max_supported || version < min_supported || version > max_supported {
13181                    return Err(invalid_recorded_history(
13182                        "version_marker_history_range_invalid",
13183                        sequence,
13184                        "min_supported <= version <= max_supported",
13185                        &format!("{min_supported} <= {version} <= {max_supported}"),
13186                        "recorded version marker contains an internally incompatible range",
13187                    ));
13188                }
13189                return Ok(RecordedCommand::VersionMarker {
13190                    sequence,
13191                    change_id,
13192                    version,
13193                });
13194            }
13195
13196            if !memo_events.is_empty() {
13197                if memo_events.len() != 1 {
13198                    return Err(invalid_recorded_history(
13199                        "duplicate_memo_upsert_record",
13200                        sequence,
13201                        "one MemoUpserted event",
13202                        &format!("{} MemoUpserted events", memo_events.len()),
13203                        "memo history records one workflow update more than once",
13204                    ));
13205                }
13206                let payload = &memo_events[0].payload;
13207                let entries = payload.get("entries").cloned().ok_or_else(|| {
13208                    invalid_recorded_history(
13209                        "memo_entries_missing",
13210                        sequence,
13211                        "memo entries object",
13212                        "missing entries",
13213                        "MemoUpserted history is missing replay identity entries",
13214                    )
13215                })?;
13216                let entries = decode_memo_history_map(&entries, true).map_err(|error| {
13217                    invalid_recorded_history(
13218                        "memo_entries_invalid",
13219                        sequence,
13220                        "valid canonical memo entries",
13221                        &error.to_string(),
13222                        "MemoUpserted history contains invalid replay identity entries",
13223                    )
13224                })?;
13225                let merged = payload.get("merged").cloned().ok_or_else(|| {
13226                    invalid_recorded_history(
13227                        "memo_merged_projection_missing",
13228                        sequence,
13229                        "merged memo projection",
13230                        "missing merged",
13231                        "MemoUpserted history is missing its merged projection",
13232                    )
13233                })?;
13234                decode_memo_history_map(&merged, false).map_err(|error| {
13235                    invalid_recorded_history(
13236                        "memo_merged_projection_invalid",
13237                        sequence,
13238                        "valid merged memo projection",
13239                        &error.to_string(),
13240                        "MemoUpserted history contains an invalid merged projection",
13241                    )
13242                })?;
13243
13244                return Ok(RecordedCommand::Memo { sequence, entries });
13245            }
13246            let scheduled: Vec<_> = timer_events
13247                .iter()
13248                .copied()
13249                .filter(|event| event.event_type == "TimerScheduled")
13250                .collect();
13251            let fired: Vec<_> = timer_events
13252                .iter()
13253                .copied()
13254                .filter(|event| event.event_type == "TimerFired")
13255                .collect();
13256            if scheduled.len() != 1 {
13257                return Err(invalid_recorded_history(
13258                    "timer_schedule_missing_or_duplicate",
13259                    sequence,
13260                    "one TimerScheduled event",
13261                    &format!("{} TimerScheduled events", scheduled.len()),
13262                    "timer replay requires exactly one recorded schedule event",
13263                ));
13264            }
13265            if fired.len() > 1 {
13266                return Err(invalid_recorded_history(
13267                    "duplicate_timer_fire",
13268                    sequence,
13269                    "at most one TimerFired event",
13270                    "multiple TimerFired events",
13271                    "timer history contains more than one fire event for a workflow sequence",
13272                ));
13273            }
13274
13275            let scheduled = scheduled[0];
13276            let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
13277            let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
13278            if let Some(fired) = fired.first() {
13279                let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
13280                if fired_timer_id != timer_id {
13281                    return Err(invalid_recorded_history(
13282                        "timer_identity_mismatch",
13283                        sequence,
13284                        &timer_id,
13285                        &fired_timer_id,
13286                        "TimerFired does not correspond to the recorded TimerScheduled event",
13287                    ));
13288                }
13289                let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
13290                if fired_delay != delay_seconds {
13291                    return Err(invalid_recorded_history(
13292                        "timer_history_delay_mismatch",
13293                        sequence,
13294                        &delay_seconds.to_string(),
13295                        &fired_delay.to_string(),
13296                        "TimerScheduled and TimerFired record different delays",
13297                    ));
13298                }
13299            }
13300
13301            Ok(RecordedCommand::Timer {
13302                sequence,
13303                delay_seconds,
13304                fired: !fired.is_empty(),
13305                parallel_group_path: recorded_parallel_group_path(&timer_events, sequence)?,
13306            })
13307        })
13308        .collect::<Result<_>>()?;
13309
13310    let mut marker_sequences = HashMap::new();
13311    for command in &commands {
13312        if let RecordedCommand::VersionMarker {
13313            sequence,
13314            change_id,
13315            ..
13316        } = command
13317        {
13318            if let Some(first_sequence) = marker_sequences.insert(change_id.clone(), *sequence) {
13319                return Err(invalid_recorded_history(
13320                    "duplicate_version_marker",
13321                    *sequence,
13322                    &format!("one marker for change ID {change_id:?}"),
13323                    &format!("markers at sequences {first_sequence} and {sequence}"),
13324                    "workflow history contains duplicate markers for one stable change ID",
13325                ));
13326            }
13327        }
13328    }
13329
13330    Ok(commands)
13331}
13332
13333fn required_version_i32(payload: &Value, field: &str, sequence: u64) -> Result<i32> {
13334    payload
13335        .get(field)
13336        .and_then(Value::as_i64)
13337        .and_then(|value| i32::try_from(value).ok())
13338        .ok_or_else(|| {
13339            invalid_recorded_history(
13340                "version_marker_field_missing",
13341                sequence,
13342                &format!("integer {field}"),
13343                "missing or out-of-range integer",
13344                "version-marker history is missing a required integer field",
13345            )
13346        })
13347}
13348
13349fn durable_event_sequence(event: &HistoryEvent) -> Option<u64> {
13350    event
13351        .payload
13352        .get("sequence")
13353        .or_else(|| event.payload.get("workflow_sequence"))
13354        .or_else(|| event.raw.get("sequence"))
13355        .or_else(|| event.raw.get("workflow_sequence"))
13356        .and_then(value_as_u64)
13357}
13358
13359fn is_internal_timer_event(event: &HistoryEvent) -> bool {
13360    matches!(
13361        event
13362            .payload
13363            .get("timer_kind")
13364            .or_else(|| event.raw.get("timer_kind"))
13365            .and_then(Value::as_str),
13366        Some("condition_timeout" | "signal_timeout")
13367    )
13368}
13369
13370fn is_recorded_condition_wait_event(event: &HistoryEvent) -> bool {
13371    matches!(
13372        event.event_type.as_str(),
13373        "ConditionWaitOpened" | "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
13374    )
13375}
13376
13377fn recorded_condition_wait(
13378    sequence: u64,
13379    condition_events: &[&HistoryEvent],
13380    all_events: &[HistoryEvent],
13381) -> Result<RecordedCommand> {
13382    let opened = condition_events
13383        .iter()
13384        .copied()
13385        .filter(|event| event.event_type == "ConditionWaitOpened")
13386        .collect::<Vec<_>>();
13387    if opened.len() != 1 {
13388        return Err(invalid_recorded_history(
13389            "condition_wait_open_missing_or_duplicate",
13390            sequence,
13391            "one ConditionWaitOpened event",
13392            &format!("{} ConditionWaitOpened events", opened.len()),
13393            "condition replay requires exactly one canonical wait-open event",
13394        ));
13395    }
13396    let terminal = condition_events
13397        .iter()
13398        .copied()
13399        .filter(|event| {
13400            matches!(
13401                event.event_type.as_str(),
13402                "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
13403            )
13404        })
13405        .collect::<Vec<_>>();
13406    if terminal.len() > 1 {
13407        return Err(invalid_recorded_history(
13408            "duplicate_condition_wait_terminal_event",
13409            sequence,
13410            "at most one condition terminal event",
13411            "multiple condition terminal events",
13412            "condition history settles one durable wait more than once",
13413        ));
13414    }
13415
13416    let opened = opened[0];
13417    let condition_wait_id = required_condition_wait_id(opened, sequence)?;
13418    let occurrence_id = required_condition_wait_occurrence_id(opened, sequence)?;
13419    for event in condition_events
13420        .iter()
13421        .copied()
13422        .filter(|event| !std::ptr::eq(*event, opened))
13423    {
13424        let event_wait_id = required_condition_wait_id(event, sequence)?;
13425        if event_wait_id != condition_wait_id {
13426            return Err(invalid_recorded_history(
13427                "condition_wait_id_mismatch",
13428                sequence,
13429                &condition_wait_id,
13430                &event_wait_id,
13431                "condition lifecycle events at one sequence disagree on wait identity",
13432            ));
13433        }
13434        let event_occurrence_id = required_condition_wait_occurrence_id(event, sequence)?;
13435        if event_occurrence_id != occurrence_id {
13436            return Err(invalid_recorded_history(
13437                "condition_wait_occurrence_history_mismatch",
13438                sequence,
13439                &occurrence_id,
13440                &event_occurrence_id,
13441                "condition lifecycle events at one sequence disagree on authored occurrence identity",
13442            ));
13443        }
13444    }
13445
13446    let condition_key = optional_non_empty_history_string(opened, "condition_key");
13447    let predicate_identity = opened
13448        .payload
13449        .get("condition_definition_fingerprint")
13450        .and_then(Value::as_str)
13451        .filter(|value| !value.is_empty())
13452        .map(str::to_string)
13453        .ok_or_else(|| {
13454            invalid_recorded_history(
13455                "condition_wait_predicate_fingerprint_missing",
13456                sequence,
13457                "non-empty condition_definition_fingerprint",
13458                &opened.event_type,
13459                "canonical condition history is missing its predicate identity",
13460            )
13461        })?;
13462    let timeout_seconds = optional_history_u64(opened, "timeout_seconds", sequence)?;
13463    for event in condition_events
13464        .iter()
13465        .copied()
13466        .filter(|event| !std::ptr::eq(*event, opened))
13467    {
13468        for (field, opened_value) in [
13469            ("condition_key", condition_key.as_deref()),
13470            (
13471                "condition_definition_fingerprint",
13472                Some(predicate_identity.as_str()),
13473            ),
13474        ] {
13475            if let Some(value) = optional_non_empty_history_string(event, field) {
13476                if opened_value.is_some_and(|opened_value| opened_value != value) {
13477                    return Err(invalid_recorded_history(
13478                        "condition_wait_definition_history_mismatch",
13479                        sequence,
13480                        opened_value.unwrap_or_default(),
13481                        &value,
13482                        "condition lifecycle events disagree on the recorded definition",
13483                    ));
13484                }
13485            }
13486        }
13487        if let Some(event_timeout) = optional_history_u64(event, "timeout_seconds", sequence)? {
13488            if timeout_seconds.is_some_and(|opened_timeout| opened_timeout != event_timeout) {
13489                return Err(invalid_recorded_history(
13490                    "condition_wait_definition_history_mismatch",
13491                    sequence,
13492                    &format!("{}s", timeout_seconds.unwrap_or_default()),
13493                    &format!("{event_timeout}s"),
13494                    "condition lifecycle events disagree on the recorded timeout",
13495                ));
13496            }
13497        }
13498    }
13499
13500    let timeout_timer_events = all_events
13501        .iter()
13502        .filter(|event| {
13503            matches!(
13504                event.event_type.as_str(),
13505                "TimerScheduled" | "TimerCancelled" | "TimerFired"
13506            ) && event.payload.get("timer_kind").and_then(Value::as_str)
13507                == Some("condition_timeout")
13508                && event
13509                    .payload
13510                    .get("condition_wait_id")
13511                    .and_then(Value::as_str)
13512                    == Some(condition_wait_id.as_str())
13513        })
13514        .collect::<Vec<_>>();
13515    let scheduled = timeout_timer_events
13516        .iter()
13517        .copied()
13518        .filter(|event| event.event_type == "TimerScheduled")
13519        .collect::<Vec<_>>();
13520    let fired = timeout_timer_events
13521        .iter()
13522        .copied()
13523        .filter(|event| event.event_type == "TimerFired")
13524        .collect::<Vec<_>>();
13525    if scheduled.len() > 1 || fired.len() > 1 || (!fired.is_empty() && scheduled.len() != 1) {
13526        return Err(invalid_recorded_history(
13527            "condition_wait_timeout_history_invalid",
13528            sequence,
13529            "one timeout schedule and at most one fire",
13530            &format!("{} schedules and {} fires", scheduled.len(), fired.len()),
13531            "condition timeout history has a missing or duplicate lifecycle event",
13532        ));
13533    }
13534    if let Some(scheduled) = scheduled.first() {
13535        let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
13536        let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
13537        if timeout_seconds.is_some_and(|timeout| timeout != delay_seconds) {
13538            return Err(invalid_recorded_history(
13539                "condition_wait_timeout_delay_mismatch",
13540                sequence,
13541                &format!("{}s", timeout_seconds.unwrap_or_default()),
13542                &format!("{delay_seconds}s"),
13543                "condition timeout timer differs from the wait definition",
13544            ));
13545        }
13546        if let Some(fired) = fired.first() {
13547            let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
13548            let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
13549            if fired_timer_id != timer_id || fired_delay != delay_seconds {
13550                return Err(invalid_recorded_history(
13551                    "condition_wait_timeout_identity_mismatch",
13552                    sequence,
13553                    &format!("{timer_id}:{delay_seconds}s"),
13554                    &format!("{fired_timer_id}:{fired_delay}s"),
13555                    "condition timeout fire does not match its durable schedule",
13556                ));
13557            }
13558        }
13559    }
13560
13561    let result = terminal.first().map(|event| {
13562        if event.event_type == "ConditionWaitTimedOut" {
13563            ConditionWaitResult::TimedOut
13564        } else {
13565            ConditionWaitResult::Satisfied
13566        }
13567    });
13568    let result = if !fired.is_empty() {
13569        if result == Some(ConditionWaitResult::Satisfied) {
13570            return Err(invalid_recorded_history(
13571                "condition_wait_terminal_conflict",
13572                sequence,
13573                "one satisfied or timed-out outcome",
13574                "satisfied event and fired timeout",
13575                "condition history records conflicting terminal outcomes",
13576            ));
13577        }
13578        Some(ConditionWaitResult::TimedOut)
13579    } else {
13580        result
13581    };
13582
13583    Ok(RecordedCommand::ConditionWait {
13584        sequence,
13585        occurrence_id,
13586        condition_key,
13587        predicate_identity,
13588        timeout_seconds,
13589        result,
13590        parallel_group_path: recorded_parallel_group_path(condition_events, sequence)?,
13591    })
13592}
13593
13594fn required_condition_wait_occurrence_id(event: &HistoryEvent, sequence: u64) -> Result<String> {
13595    event
13596        .payload
13597        .get("condition_wait_occurrence_id")
13598        .and_then(Value::as_str)
13599        .filter(|value| !value.is_empty())
13600        .map(str::to_string)
13601        .ok_or_else(|| {
13602            invalid_recorded_history(
13603                "condition_wait_occurrence_id_missing",
13604                sequence,
13605                "non-empty condition_wait_occurrence_id",
13606                &event.event_type,
13607                "condition history is missing authored occurrence identity",
13608            )
13609        })
13610}
13611
13612fn required_condition_wait_id(event: &HistoryEvent, sequence: u64) -> Result<String> {
13613    event
13614        .payload
13615        .get("condition_wait_id")
13616        .and_then(Value::as_str)
13617        .filter(|value| !value.is_empty())
13618        .map(str::to_string)
13619        .ok_or_else(|| {
13620            invalid_recorded_history(
13621                "condition_wait_id_missing",
13622                sequence,
13623                "non-empty condition_wait_id",
13624                &event.event_type,
13625                "canonical condition history is missing its durable wait identity",
13626            )
13627        })
13628}
13629
13630fn optional_non_empty_history_string(event: &HistoryEvent, field: &str) -> Option<String> {
13631    event
13632        .payload
13633        .get(field)
13634        .and_then(Value::as_str)
13635        .filter(|value| !value.is_empty())
13636        .map(str::to_string)
13637}
13638
13639fn optional_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<Option<u64>> {
13640    match event.payload.get(field) {
13641        None | Some(Value::Null) => Ok(None),
13642        Some(value) => value_as_u64(value).map(Some).ok_or_else(|| {
13643            invalid_recorded_history(
13644                "condition_wait_definition_invalid",
13645                sequence,
13646                &format!("non-negative integer {field}"),
13647                &value.to_string(),
13648                "condition history contains an invalid numeric definition field",
13649            )
13650        }),
13651    }
13652}
13653
13654fn required_signal_wait_name(event: &HistoryEvent, sequence: u64) -> Result<String> {
13655    event
13656        .payload
13657        .get("signal_name")
13658        .or_else(|| event.raw.get("signal_name"))
13659        .and_then(Value::as_str)
13660        .filter(|value| !value.is_empty())
13661        .map(str::to_string)
13662        .ok_or_else(|| {
13663            invalid_recorded_history(
13664                "signal_wait_name_missing",
13665                sequence,
13666                "non-empty signal_name",
13667                &event.event_type,
13668                "canonical signal-wait history is missing its signal identity",
13669            )
13670        })
13671}
13672
13673fn is_recorded_signal_wait_event(event: &HistoryEvent) -> bool {
13674    matches!(
13675        event.event_type.as_str(),
13676        "SignalWaitOpened" | "SignalApplied"
13677    )
13678}
13679
13680fn required_history_string(event: &HistoryEvent, field: &str, sequence: u64) -> Result<String> {
13681    event
13682        .payload
13683        .get(field)
13684        .and_then(Value::as_str)
13685        .filter(|value| !value.is_empty())
13686        .map(str::to_string)
13687        .ok_or_else(|| {
13688            invalid_recorded_history(
13689                "timer_history_field_missing",
13690                sequence,
13691                field,
13692                &event.event_type,
13693                "timer history is missing a required identity field",
13694            )
13695        })
13696}
13697
13698fn required_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<u64> {
13699    event
13700        .payload
13701        .get(field)
13702        .and_then(value_as_u64)
13703        .ok_or_else(|| {
13704            invalid_recorded_history(
13705                "timer_history_field_missing",
13706                sequence,
13707                field,
13708                &event.event_type,
13709                "timer history is missing a required numeric field",
13710            )
13711        })
13712}
13713
13714fn recorded_search_attribute_types(
13715    payload: &Value,
13716    attributes: &Value,
13717    sequence: u64,
13718) -> Result<RecordedSnapshotValue<BTreeMap<String, String>>> {
13719    let Some(raw_types) = payload.get("attribute_types") else {
13720        // This is the explicit compatibility rule for histories recorded
13721        // before typed identity was persisted. Values still constrain replay;
13722        // the unknown type snapshot does not assert a typed match.
13723        return Ok(RecordedSnapshotValue::Unknown);
13724    };
13725    let Some(raw_types) = raw_types.as_object() else {
13726        return Err(invalid_recorded_history(
13727            "search_attribute_types_malformed",
13728            sequence,
13729            "canonical attribute type map",
13730            &raw_types.to_string(),
13731            "search-attribute history contains malformed type identity",
13732        ));
13733    };
13734    let attribute_keys = attributes
13735        .as_object()
13736        .expect("recorded search attributes were validated as an object");
13737    let mut types = BTreeMap::new();
13738    for (key, value) in raw_types {
13739        let Some(attribute_type) = value.as_str() else {
13740            return Err(invalid_recorded_history(
13741                "search_attribute_types_malformed",
13742                sequence,
13743                "canonical string type name",
13744                &value.to_string(),
13745                "search-attribute history contains a non-string type identity",
13746            ));
13747        };
13748        if !attribute_keys.contains_key(key)
13749            || !matches!(
13750                attribute_type,
13751                "string" | "keyword" | "keyword_list" | "int" | "float" | "bool" | "datetime"
13752            )
13753        {
13754            return Err(invalid_recorded_history(
13755                "search_attribute_types_malformed",
13756                sequence,
13757                "canonical types for keys present in attributes",
13758                &format!("{key}:{attribute_type}"),
13759                "search-attribute history contains unsupported or orphaned type identity",
13760            ));
13761        }
13762        types.insert(key.clone(), attribute_type.to_string());
13763    }
13764    Ok(RecordedSnapshotValue::Known(types))
13765}
13766
13767fn invalid_recorded_history(
13768    reason: &str,
13769    sequence: u64,
13770    expected: &str,
13771    actual: &str,
13772    message: &str,
13773) -> Error {
13774    Error::NonDeterministicReplay(ReplayFailure::new(
13775        reason,
13776        Some(sequence),
13777        Some(expected.to_string()),
13778        Some(actual.to_string()),
13779        message,
13780    ))
13781}
13782
13783type ActivityOutcome = std::result::Result<AvroValue, ActivityFailure>;
13784
13785fn activity_outcome(
13786    event: &HistoryEvent,
13787    fallback_codec: &str,
13788    recorded_activity_type: Option<String>,
13789) -> Result<ActivityOutcome> {
13790    if event.event_type == "ActivityCompleted" {
13791        let codec = event
13792            .payload
13793            .get("payload_codec")
13794            .and_then(Value::as_str)
13795            .unwrap_or(fallback_codec);
13796        return Ok(Ok(decode_wire_avro_value(
13797            event.payload.get("result").unwrap_or(&Value::Null),
13798            codec,
13799        )?));
13800    }
13801
13802    let payload = &event.payload;
13803    let (kind, fallback_reason, fallback_message) = match event.event_type.as_str() {
13804        "ActivityFailed" => (ActivityFailureKind::Failed, "activity", "activity failed"),
13805        "ActivityCancelled" => (
13806            ActivityFailureKind::Cancelled,
13807            "cancelled",
13808            "activity was cancelled",
13809        ),
13810        "ActivityTimedOut" => (
13811            ActivityFailureKind::TimedOut,
13812            "timeout",
13813            "activity timed out",
13814        ),
13815        _ => unreachable!("activity_outcome is called only for terminal activity events"),
13816    };
13817    let exception = payload
13818        .get("exception")
13819        .filter(|value| !value.is_null())
13820        .cloned();
13821    let failure_category = payload_string(payload, "failure_category");
13822    let timeout_kind = payload_string(payload, "timeout_kind");
13823    let reason = payload_string(payload, "reason").unwrap_or_else(|| match kind {
13824        ActivityFailureKind::Failed => failure_category
13825            .clone()
13826            .unwrap_or_else(|| fallback_reason.to_string()),
13827        ActivityFailureKind::Cancelled => fallback_reason.to_string(),
13828        ActivityFailureKind::TimedOut => timeout_kind
13829            .clone()
13830            .unwrap_or_else(|| fallback_reason.to_string()),
13831    });
13832    let message = payload_string(payload, "message")
13833        .or_else(|| {
13834            exception
13835                .as_ref()
13836                .and_then(|value| payload_string(value, "message"))
13837        })
13838        .unwrap_or_else(|| fallback_message.to_string());
13839
13840    Ok(Err(ActivityFailure {
13841        kind,
13842        reason,
13843        message,
13844        activity_execution_id: payload_string(payload, "activity_execution_id"),
13845        activity_attempt_id: payload_string(payload, "activity_attempt_id"),
13846        activity_type: payload_string(payload, "activity_type")
13847            .or_else(|| payload_string(payload, "activity_name"))
13848            .or(recorded_activity_type),
13849        activity_class: payload_string(payload, "activity_class"),
13850        attempt_number: payload.get("attempt_number").and_then(value_as_u64),
13851        failure_id: payload_string(payload, "failure_id"),
13852        failure_category,
13853        timeout_kind,
13854        non_retryable: payload
13855            .get("non_retryable")
13856            .and_then(Value::as_bool)
13857            .unwrap_or(false),
13858        exception_type: payload_string(payload, "exception_type").or_else(|| {
13859            exception
13860                .as_ref()
13861                .and_then(|value| payload_string(value, "type"))
13862        }),
13863        exception_class: payload_string(payload, "exception_class").or_else(|| {
13864            exception
13865                .as_ref()
13866                .and_then(|value| payload_string(value, "class"))
13867        }),
13868        code: payload
13869            .get("code")
13870            .filter(|value| !value.is_null())
13871            .cloned(),
13872        exception,
13873    }))
13874}
13875
13876type ChildWorkflowOutcome = std::result::Result<ChildWorkflowAvroResult, ChildWorkflowFailure>;
13877
13878fn child_workflow_outcomes(
13879    events: &[HistoryEvent],
13880    fallback_codec: &str,
13881    parent: WorkflowIdentity,
13882) -> Result<Vec<ChildWorkflowOutcome>> {
13883    let mut outcomes = Vec::new();
13884
13885    for event in events {
13886        let kind = match event.event_type.as_str() {
13887            "ChildRunCompleted" => None,
13888            "ChildRunFailed" => Some((
13889                ChildWorkflowFailureKind::Failed,
13890                "child_workflow",
13891                "child workflow failed",
13892            )),
13893            "ChildRunCancelled" => Some((
13894                ChildWorkflowFailureKind::Cancelled,
13895                "cancelled",
13896                "child workflow was cancelled",
13897            )),
13898            "ChildRunTerminated" => Some((
13899                ChildWorkflowFailureKind::Terminated,
13900                "terminated",
13901                "child workflow was terminated",
13902            )),
13903            _ => continue,
13904        };
13905        let payload = &event.payload;
13906        let child_workflow_id = payload_string(payload, "child_workflow_instance_id");
13907        let child_workflow_run_id = payload_string(payload, "child_workflow_run_id");
13908        let child_workflow_type = payload_string(payload, "child_workflow_type");
13909
13910        if let Some((kind, reason, fallback_message)) = kind {
13911            let exception = payload
13912                .get("exception")
13913                .filter(|value| !value.is_null())
13914                .cloned();
13915            let message = payload_string(payload, "message")
13916                .or_else(|| {
13917                    exception
13918                        .as_ref()
13919                        .and_then(|value| payload_string(value, "message"))
13920                })
13921                .unwrap_or_else(|| fallback_message.to_string());
13922            let exception_type = payload_string(payload, "exception_type").or_else(|| {
13923                exception
13924                    .as_ref()
13925                    .and_then(|value| payload_string(value, "type"))
13926            });
13927            let exception_class = payload_string(payload, "exception_class").or_else(|| {
13928                exception
13929                    .as_ref()
13930                    .and_then(|value| payload_string(value, "class"))
13931            });
13932            outcomes.push(Err(ChildWorkflowFailure {
13933                kind,
13934                reason: reason.to_string(),
13935                message,
13936                parent_workflow_id: parent.workflow_id.clone(),
13937                parent_workflow_run_id: parent.run_id.clone(),
13938                child_workflow_id,
13939                child_workflow_run_id,
13940                child_workflow_type,
13941                failure_id: payload_string(payload, "failure_id"),
13942                failure_category: payload_string(payload, "failure_category"),
13943                exception_type,
13944                exception_class,
13945                non_retryable: payload
13946                    .get("non_retryable")
13947                    .and_then(Value::as_bool)
13948                    .unwrap_or(false),
13949                code: payload
13950                    .get("code")
13951                    .filter(|value| !value.is_null())
13952                    .cloned(),
13953                exception,
13954            }));
13955            continue;
13956        }
13957
13958        let codec = payload
13959            .get("payload_codec")
13960            .and_then(Value::as_str)
13961            .unwrap_or(fallback_codec);
13962        let result = payload
13963            .get("result")
13964            .or_else(|| payload.get("output"))
13965            .unwrap_or(&Value::Null);
13966        outcomes.push(Ok(ChildWorkflowAvroResult {
13967            parent: parent.clone(),
13968            child: WorkflowIdentity {
13969                workflow_id: child_workflow_id,
13970                run_id: child_workflow_run_id,
13971            },
13972            child_workflow_type,
13973            result: decode_wire_avro_value(result, codec)?,
13974        }));
13975    }
13976
13977    Ok(outcomes)
13978}
13979
13980fn payload_string(payload: &Value, key: &str) -> Option<String> {
13981    payload
13982        .get(key)
13983        .and_then(Value::as_str)
13984        .filter(|value| !value.is_empty())
13985        .map(str::to_string)
13986}
13987
13988fn workflow_failure_command(error: &Error) -> Value {
13989    let (exception_type, exception_class, properties) = match error {
13990        Error::ActivityFailed(failure) => (
13991            match failure.kind {
13992                ActivityFailureKind::Failed => "ActivityFailed",
13993                ActivityFailureKind::Cancelled => "ActivityCancelled",
13994                ActivityFailureKind::TimedOut => "ActivityTimedOut",
13995            },
13996            "durable_workflow::ActivityFailure",
13997            json!({
13998                "reason": failure.reason,
13999                "activity_execution_id": failure.activity_execution_id,
14000                "activity_attempt_id": failure.activity_attempt_id,
14001                "activity_type": failure.activity_type,
14002                "activity_class": failure.activity_class,
14003                "attempt_number": failure.attempt_number,
14004                "failure_id": failure.failure_id,
14005                "failure_category": failure.failure_category,
14006                "timeout_kind": failure.timeout_kind,
14007                "activity_non_retryable": failure.non_retryable,
14008                "activity_exception_type": failure.exception_type,
14009                "activity_exception_class": failure.exception_class,
14010                "activity_code": failure.code,
14011                "activity_exception": failure.exception,
14012            }),
14013        ),
14014        Error::ChildWorkflowFailed(failure) => (
14015            match failure.kind {
14016                ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
14017                ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
14018                ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
14019            },
14020            "durable_workflow::ChildWorkflowFailure",
14021            json!({
14022                "reason": failure.reason,
14023                "parent_workflow_id": failure.parent_workflow_id,
14024                "parent_workflow_run_id": failure.parent_workflow_run_id,
14025                "child_workflow_id": failure.child_workflow_id,
14026                "child_workflow_run_id": failure.child_workflow_run_id,
14027                "child_workflow_type": failure.child_workflow_type,
14028                "failure_id": failure.failure_id,
14029                "failure_category": failure.failure_category,
14030                "child_exception_type": failure.exception_type,
14031                "child_exception_class": failure.exception_class,
14032                "child_non_retryable": failure.non_retryable,
14033                "child_code": failure.code,
14034                "child_exception": failure.exception,
14035            }),
14036        ),
14037        Error::ParallelFailed(failure) => (
14038            "ParallelFailed",
14039            "durable_workflow::ParallelFailure",
14040            json!({
14041                "parallel_group_id": failure.group_id,
14042                "parallel_member_path": failure.member_path,
14043                "parallel_group_path": failure.group_path,
14044                "completed_members": failure.completed.iter().map(|completion| &completion.member_path).collect::<Vec<_>>(),
14045                "cause_type": workflow_error_type(&failure.cause),
14046                "cause_message": failure.cause.to_string(),
14047            }),
14048        ),
14049        Error::SagaCompensationFailed(failure) => (
14050            "SagaCompensationFailed",
14051            "durable_workflow::SagaCompensationFailure",
14052            json!({
14053                "initiating_failure_type": workflow_error_type(&failure.initiating_failure),
14054                "initiating_failure_message": failure.initiating_failure.to_string(),
14055                "compensation_activity_type": failure.compensation_activity_type,
14056                "compensation_registration_order": failure.compensation_registration_order,
14057                "compensation_failure_type": workflow_error_type(&failure.compensation_failure),
14058                "compensation_failure_message": failure.compensation_failure.to_string(),
14059            }),
14060        ),
14061        Error::WorkflowCancellationRequested(_) => (
14062            "WorkflowCancellationRequested",
14063            "durable_workflow::WorkflowCancellationRequested",
14064            json!({"reason": "cancelled"}),
14065        ),
14066        Error::NonDeterministicReplay(_) => (
14067            "NonDeterministicReplay",
14068            "durable_workflow::Error",
14069            Value::Null,
14070        ),
14071        _ => ("RustWorkflowError", "durable_workflow::Error", Value::Null),
14072    };
14073    let non_retryable = match error {
14074        Error::ActivityFailed(failure) => failure.non_retryable,
14075        Error::ChildWorkflowFailed(failure) => failure.non_retryable,
14076        Error::ParallelFailed(failure) => workflow_error_non_retryable(&failure.cause),
14077        Error::SagaCompensationFailed(failure) => {
14078            workflow_error_non_retryable(&failure.compensation_failure)
14079        }
14080        Error::WorkflowCancellationRequested(_) => true,
14081        Error::NonDeterministicReplay(_) => true,
14082        _ => false,
14083    };
14084
14085    json!({
14086        "type": "fail_workflow",
14087        "message": error.to_string(),
14088        "exception_type": exception_type,
14089        "exception_class": exception_class,
14090        "non_retryable": non_retryable,
14091        "exception": {
14092            "type": exception_type,
14093            "class": exception_class,
14094            "message": error.to_string(),
14095            "properties": properties,
14096        }
14097    })
14098}
14099
14100fn workflow_error_type(error: &Error) -> &'static str {
14101    match error {
14102        Error::ActivityFailed(failure) => match failure.kind {
14103            ActivityFailureKind::Failed => "ActivityFailed",
14104            ActivityFailureKind::Cancelled => "ActivityCancelled",
14105            ActivityFailureKind::TimedOut => "ActivityTimedOut",
14106        },
14107        Error::ChildWorkflowFailed(failure) => match failure.kind {
14108            ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
14109            ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
14110            ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
14111        },
14112        Error::ParallelFailed(_) => "ParallelFailed",
14113        Error::SagaCompensationFailed(_) => "SagaCompensationFailed",
14114        Error::WorkflowCancellationRequested(_) => "WorkflowCancellationRequested",
14115        Error::NonDeterministicReplay(_) => "NonDeterministicReplay",
14116        _ => "RustWorkflowError",
14117    }
14118}
14119
14120fn workflow_error_non_retryable(error: &Error) -> bool {
14121    match error {
14122        Error::ActivityFailed(failure) => failure.non_retryable,
14123        Error::ChildWorkflowFailed(failure) => failure.non_retryable,
14124        Error::ParallelFailed(failure) => workflow_error_non_retryable(&failure.cause),
14125        Error::SagaCompensationFailed(failure) => {
14126            workflow_error_non_retryable(&failure.compensation_failure)
14127        }
14128        Error::WorkflowCancellationRequested(_) | Error::NonDeterministicReplay(_) => true,
14129        _ => false,
14130    }
14131}
14132
14133fn workflow_task_integrity_error(error: &Error) -> bool {
14134    matches!(
14135        error,
14136        Error::NonDeterministicReplay(_)
14137            | Error::Protocol(_)
14138            | Error::MissingWorkflowCommandIdentity
14139            | Error::WorkflowStatePoisoned
14140    )
14141}
14142
14143fn decode_signal_event_arguments(
14144    event: &HistoryEvent,
14145    fallback_codec: &str,
14146) -> Result<Vec<AvroValue>> {
14147    let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
14148    validate_payload_codec(codec)?;
14149    let raw = signal_history_payload(&event.payload);
14150    let decoded = match raw.filter(|value| !value.is_null()) {
14151        Some(value) => decode_wire_avro_value(value, codec)?,
14152        None => AvroValue::Array(Vec::new()),
14153    };
14154    let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
14155        unreachable!("normalize_avro_arguments always returns an array");
14156    };
14157    Ok(arguments)
14158}
14159
14160fn decode_update_event_arguments(
14161    event: &HistoryEvent,
14162    fallback_codec: &str,
14163) -> Result<Vec<AvroValue>> {
14164    let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
14165    validate_payload_codec(codec)?;
14166    let decoded = match event
14167        .payload
14168        .get("arguments")
14169        .filter(|value| !value.is_null())
14170    {
14171        Some(value) => decode_wire_avro_value(value, codec)?,
14172        None => AvroValue::Array(Vec::new()),
14173    };
14174    let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
14175        unreachable!("normalize_avro_arguments always returns an array");
14176    };
14177    Ok(arguments)
14178}
14179
14180fn hydrate_query_history_from_export(task: &mut QueryTask) -> Result<()> {
14181    let Some(export_events) = task
14182        .history_export
14183        .as_ref()
14184        .and_then(|export| export.get("history_events"))
14185        .and_then(Value::as_array)
14186    else {
14187        return Ok(());
14188    };
14189
14190    if export_events.len() > task.history_events.len() {
14191        task.history_events = serde_json::from_value(Value::Array(export_events.clone()))?;
14192    }
14193
14194    Ok(())
14195}
14196
14197fn enrich_query_history_from_export(task: &mut QueryTask) -> Result<()> {
14198    let Some(export) = task.history_export.as_ref() else {
14199        return Ok(());
14200    };
14201    let signals = export
14202        .get("signals")
14203        .and_then(Value::as_array)
14204        .cloned()
14205        .unwrap_or_default();
14206    let activities = export
14207        .get("activities")
14208        .and_then(Value::as_array)
14209        .cloned()
14210        .unwrap_or_default();
14211    let export_codec = export
14212        .get("payloads")
14213        .and_then(|payloads| payloads.get("codec"))
14214        .and_then(Value::as_str)
14215        .unwrap_or(&task.payload_codec)
14216        .to_string();
14217    let mut signal_name_offsets: HashMap<String, usize> = HashMap::new();
14218
14219    for event in &mut task.history_events {
14220        if event.event_type == "ActivityCompleted" {
14221            let sequence = event
14222                .payload
14223                .get("sequence")
14224                .or_else(|| event.payload.get("workflow_sequence"))
14225                .and_then(value_as_u64);
14226            let Some(activity) = sequence.and_then(|sequence| {
14227                activities.iter().find(|activity| {
14228                    activity.get("sequence").and_then(value_as_u64) == Some(sequence)
14229                })
14230            }) else {
14231                continue;
14232            };
14233            let Some(payload) = event.payload.as_object_mut() else {
14234                continue;
14235            };
14236            if missing_payload(payload.get("result")) {
14237                if let Some(result) = activity
14238                    .get("result")
14239                    .filter(|value| !missing_payload(Some(value)))
14240                {
14241                    payload.insert("result".to_string(), result.clone());
14242                }
14243            }
14244            for field in ["payload_codec", "activity_type"] {
14245                if payload
14246                    .get(field)
14247                    .and_then(Value::as_str)
14248                    .unwrap_or_default()
14249                    .is_empty()
14250                {
14251                    if let Some(value) = activity.get(field) {
14252                        payload.insert(field.to_string(), value.clone());
14253                    }
14254                }
14255            }
14256            continue;
14257        }
14258
14259        if event.event_type != "SignalReceived" && event.event_type != "SignalApplied" {
14260            continue;
14261        }
14262        let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
14263        let command_id = event
14264            .payload
14265            .get("workflow_command_id")
14266            .or_else(|| event.raw.get("workflow_command_id"))
14267            .and_then(Value::as_str);
14268        let signal_name = event
14269            .payload
14270            .get("signal_name")
14271            .and_then(Value::as_str)
14272            .unwrap_or_default()
14273            .to_string();
14274        let matched = signals
14275            .iter()
14276            .find(|signal| {
14277                signal_id.is_some() && signal.get("id").and_then(Value::as_str) == signal_id
14278            })
14279            .or_else(|| {
14280                signals.iter().find(|signal| {
14281                    command_id.is_some()
14282                        && signal.get("command_id").and_then(Value::as_str) == command_id
14283                })
14284            })
14285            .or_else(|| {
14286                let offset = signal_name_offsets.entry(signal_name.clone()).or_default();
14287                let signal = signals
14288                    .iter()
14289                    .filter(|signal| {
14290                        signal.get("name").and_then(Value::as_str) == Some(signal_name.as_str())
14291                    })
14292                    .nth(*offset);
14293                if signal.is_some() {
14294                    *offset += 1;
14295                }
14296                signal
14297            });
14298        let Some(signal) = matched else {
14299            continue;
14300        };
14301        let signal_codec = signal
14302            .get("payload_codec")
14303            .and_then(Value::as_str)
14304            .unwrap_or(&export_codec);
14305        let Some(payload) = event.payload.as_object_mut() else {
14306            continue;
14307        };
14308        if missing_payload(payload.get("arguments")) {
14309            if let Some(arguments) = signal
14310                .get("arguments")
14311                .filter(|value| !missing_payload(Some(value)))
14312            {
14313                let envelope = match arguments {
14314                    Value::String(blob) => json!({"codec": signal_codec, "blob": blob}),
14315                    other => other.clone(),
14316                };
14317                payload.insert("arguments".to_string(), envelope);
14318            }
14319        }
14320        if payload
14321            .get("payload_codec")
14322            .and_then(Value::as_str)
14323            .unwrap_or_default()
14324            .is_empty()
14325        {
14326            payload.insert("payload_codec".to_string(), json!(signal_codec));
14327        }
14328    }
14329
14330    Ok(())
14331}
14332
14333fn missing_payload(value: Option<&Value>) -> bool {
14334    match value {
14335        None | Some(Value::Null) => true,
14336        Some(Value::String(value)) => value.is_empty(),
14337        Some(_) => false,
14338    }
14339}
14340
14341fn query_signal_events(task: &QueryTask) -> Result<Vec<QuerySignal>> {
14342    let export_signals = task
14343        .history_export
14344        .as_ref()
14345        .and_then(|export| export.get("signals"))
14346        .and_then(Value::as_array)
14347        .cloned()
14348        .unwrap_or_default();
14349    let export_codec = task
14350        .history_export
14351        .as_ref()
14352        .and_then(|export| export.get("payloads"))
14353        .and_then(|payloads| payloads.get("codec"))
14354        .and_then(Value::as_str)
14355        .unwrap_or(&task.payload_codec);
14356    let mut name_offsets: HashMap<String, usize> = HashMap::new();
14357    let mut signals = Vec::new();
14358
14359    for event in &task.history_events {
14360        if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
14361            continue;
14362        }
14363
14364        let name = event
14365            .payload
14366            .get("signal_name")
14367            .and_then(Value::as_str)
14368            .unwrap_or_default();
14369        if name.is_empty() {
14370            continue;
14371        }
14372        let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
14373        let command_id = event
14374            .payload
14375            .get("workflow_command_id")
14376            .or_else(|| event.raw.get("workflow_command_id"))
14377            .and_then(Value::as_str);
14378        let matched_export = export_signals
14379            .iter()
14380            .find(|candidate| {
14381                signal_id.is_some() && candidate.get("id").and_then(Value::as_str) == signal_id
14382            })
14383            .or_else(|| {
14384                export_signals.iter().find(|candidate| {
14385                    command_id.is_some()
14386                        && candidate.get("command_id").and_then(Value::as_str) == command_id
14387                })
14388            })
14389            .or_else(|| {
14390                let offset = name_offsets.entry(name.to_string()).or_default();
14391                let candidate = export_signals
14392                    .iter()
14393                    .filter(|candidate| candidate.get("name").and_then(Value::as_str) == Some(name))
14394                    .nth(*offset);
14395                if candidate.is_some() {
14396                    *offset += 1;
14397                }
14398                candidate
14399            });
14400        let codec = event
14401            .payload
14402            .get("payload_codec")
14403            .and_then(Value::as_str)
14404            .or_else(|| {
14405                matched_export
14406                    .and_then(|signal| signal.get("payload_codec"))
14407                    .and_then(Value::as_str)
14408            })
14409            .unwrap_or(export_codec);
14410        let raw_arguments = signal_history_payload(&event.payload)
14411            .filter(|value| !value.is_null())
14412            .or_else(|| matched_export.and_then(|signal| signal.get("arguments")));
14413        let (arguments, avro_arguments) = decode_query_signal_arguments(raw_arguments, codec)?;
14414        let workflow_sequence = event
14415            .payload
14416            .get("workflow_sequence")
14417            .and_then(value_as_u64)
14418            .or_else(|| {
14419                matched_export
14420                    .and_then(|signal| signal.get("workflow_sequence"))
14421                    .and_then(value_as_u64)
14422            });
14423
14424        signals.push(QuerySignal {
14425            id: signal_id.map(str::to_string).or_else(|| {
14426                matched_export
14427                    .and_then(|signal| signal.get("id"))
14428                    .and_then(Value::as_str)
14429                    .map(str::to_string)
14430            }),
14431            name: name.to_string(),
14432            arguments,
14433            avro_arguments,
14434            workflow_sequence,
14435        });
14436    }
14437
14438    if signals.is_empty() {
14439        for signal in export_signals {
14440            if signal.get("status").and_then(Value::as_str) == Some("rejected") {
14441                continue;
14442            }
14443            let Some(name) = signal.get("name").and_then(Value::as_str) else {
14444                continue;
14445            };
14446            let codec = signal
14447                .get("payload_codec")
14448                .and_then(Value::as_str)
14449                .unwrap_or(export_codec);
14450            let (arguments, avro_arguments) =
14451                decode_query_signal_arguments(signal.get("arguments"), codec)?;
14452            signals.push(QuerySignal {
14453                id: signal.get("id").and_then(Value::as_str).map(str::to_string),
14454                name: name.to_string(),
14455                arguments,
14456                avro_arguments,
14457                workflow_sequence: signal.get("workflow_sequence").and_then(value_as_u64),
14458            });
14459        }
14460        signals.sort_by_key(|signal| signal.workflow_sequence.unwrap_or(u64::MAX));
14461    }
14462
14463    Ok(signals)
14464}
14465
14466fn decode_query_signal_arguments(
14467    raw: Option<&Value>,
14468    codec: &str,
14469) -> Result<(Vec<Value>, Vec<AvroValue>)> {
14470    validate_payload_codec(codec)?;
14471    let decoded = match raw.filter(|value| !value.is_null()) {
14472        Some(value) => decode_wire_avro_value(value, codec)?,
14473        None => AvroValue::Array(Vec::new()),
14474    };
14475    let AvroValue::Array(avro_arguments) = normalize_avro_arguments(decoded) else {
14476        unreachable!("normalize_avro_arguments always returns an array");
14477    };
14478    let arguments = avro_arguments
14479        .iter()
14480        .cloned()
14481        .map(AvroValue::into_json)
14482        .collect::<Result<Vec<_>>>()?;
14483    Ok((arguments, avro_arguments))
14484}
14485
14486fn value_as_u64(value: &Value) -> Option<u64> {
14487    value
14488        .as_u64()
14489        .or_else(|| value.as_str().and_then(|value| value.parse().ok()))
14490}
14491
14492#[cfg(test)]
14493mod tests {
14494    use super::*;
14495    use std::{
14496        fs,
14497        io::{Read, Write},
14498        net::{SocketAddr, TcpListener, TcpStream},
14499        process::Command as ProcessCommand,
14500        sync::atomic::AtomicUsize,
14501        thread,
14502    };
14503
14504    #[derive(Clone, Copy, Debug)]
14505    enum InvalidTaskPayloadCodec {
14506        Missing,
14507        Null,
14508        NonString,
14509    }
14510
14511    impl InvalidTaskPayloadCodec {
14512        fn label(self) -> &'static str {
14513            match self {
14514                Self::Missing => "missing",
14515                Self::Null => "null",
14516                Self::NonString => "non-string",
14517            }
14518        }
14519
14520        fn apply(self, task: &mut Value) {
14521            let task = task.as_object_mut().expect("task fixture object");
14522            match self {
14523                Self::Missing => {
14524                    task.remove("payload_codec");
14525                }
14526                Self::Null => {
14527                    task.insert("payload_codec".to_string(), Value::Null);
14528                }
14529                Self::NonString => {
14530                    task.insert("payload_codec".to_string(), json!(42));
14531                }
14532            }
14533        }
14534    }
14535
14536    fn fixture_envelope(value: Value) -> Value {
14537        encode_value_envelope(&value, DEFAULT_CODEC).expect("encode Avro test fixture")
14538    }
14539
14540    fn fixture_blob(value: Value) -> String {
14541        encode_payload(&value, DEFAULT_CODEC)
14542            .expect("encode Avro test fixture")
14543            .blob
14544    }
14545
14546    #[test]
14547    fn client_builder_rejects_the_sdk_owned_api_suffix() {
14548        for base_url in [
14549            "http://127.0.0.1:8080/api",
14550            "http://localhost:8080/api/",
14551            "https://runtime.example.test/namespaces/orders/api",
14552        ] {
14553            let error = Client::builder(base_url)
14554                .build()
14555                .expect_err("SDK-owned /api suffix must be rejected during build");
14556
14557            assert!(matches!(error, Error::InvalidBaseUrl), "{base_url}");
14558            assert!(
14559                error.to_string().contains("SDK appends /api automatically"),
14560                "the validation error must explain how to fix the endpoint"
14561            );
14562        }
14563    }
14564
14565    #[test]
14566    fn client_builder_preserves_self_hosted_and_managed_runtime_prefixes() {
14567        for (base_url, expected) in [
14568            ("http://127.0.0.1:8080", "http://127.0.0.1:8080"),
14569            (
14570                "http://localhost:8080/durable-workflow/",
14571                "http://localhost:8080/durable-workflow",
14572            ),
14573            (
14574                "https://runtime.example.test/namespaces/orders",
14575                "https://runtime.example.test/namespaces/orders",
14576            ),
14577            (
14578                "https://runtime.example.test/gateway/api/namespaces/orders",
14579                "https://runtime.example.test/gateway/api/namespaces/orders",
14580            ),
14581            (
14582                "https://api.example.test/runtime/orders/",
14583                "https://api.example.test/runtime/orders",
14584            ),
14585        ] {
14586            let client = Client::builder(base_url)
14587                .build()
14588                .expect("Server and Cloud runtime base URL must remain valid");
14589
14590            assert_eq!(client.base_url, expected);
14591        }
14592    }
14593
14594    #[test]
14595    fn workflow_completion_uses_the_additive_command_protocol_floor() {
14596        assert_eq!(
14597            workflow_completion_protocol_version(&[json!({"type": "complete_workflow"})]),
14598            WORKER_PROTOCOL_VERSION
14599        );
14600        assert_eq!(
14601            workflow_completion_protocol_version(&[json!({
14602                "type": "upsert_search_attributes",
14603                "attributes": {"OrderStatus": "waiting"},
14604            })]),
14605            SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION
14606        );
14607        assert_eq!(
14608            workflow_completion_protocol_version(&[json!({
14609                "type": "upsert_search_attributes",
14610                "attributes": {"OrderStatus": "waiting"},
14611                "attribute_types": {"OrderStatus": "keyword"},
14612            })]),
14613            TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION
14614        );
14615        assert_eq!(
14616            workflow_completion_protocol_version(&[
14617                json!({"type": "upsert_memo", "entries": {"status": "waiting"}}),
14618                json!({"type": "open_condition_wait", "condition_key": "ready"}),
14619            ]),
14620            MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION
14621        );
14622        assert_eq!(
14623            workflow_completion_protocol_version(&[
14624                json!({"type": "upsert_search_attributes", "attributes": {"State": "waiting"}}),
14625                json!({"type": "open_condition_wait", "condition_key": "ready"}),
14626            ]),
14627            CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION
14628        );
14629        assert_eq!(
14630            workflow_completion_protocol_version(&[json!({
14631                "type": "open_condition_wait",
14632                "condition_wait_occurrence_id": "rust:condition-wait:0",
14633                "condition_key": "ready",
14634            })]),
14635            CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
14636        );
14637        assert_eq!(
14638            workflow_completion_protocol_version_with_message_streams(
14639                &[json!({"type": "upsert_memo", "entries": {"status": "waiting"}})],
14640                true,
14641            ),
14642            MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION
14643        );
14644        assert_eq!(
14645            workflow_completion_protocol_version_with_message_streams(
14646                &[json!({
14647                    "type": "open_condition_wait",
14648                    "condition_wait_occurrence_id": "rust:condition-wait:0",
14649                    "condition_key": "ready",
14650                })],
14651                true,
14652            ),
14653            CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
14654        );
14655    }
14656
14657    #[test]
14658    fn portable_worker_affinity_manifest_explicitly_refuses_unimplemented_features() {
14659        let manifest = portable_worker_affinity_capability_manifest();
14660
14661        for capability in ["local_activities", "worker_sessions", "sticky_execution"] {
14662            assert_eq!(manifest[capability]["supported"], json!(false));
14663            assert_eq!(
14664                manifest[capability]["minimum_protocol_version"],
14665                json!(PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION)
14666            );
14667            assert!(manifest[capability]["reason"]
14668                .as_str()
14669                .is_some_and(|reason| !reason.is_empty()));
14670        }
14671    }
14672
14673    fn typed_fidelity_probe() -> AvroValue {
14674        AvroValue::Map(BTreeMap::from([
14675            ("bytes".to_string(), AvroValue::Bytes(vec![0, 0xff])),
14676            ("empty".to_string(), AvroValue::Map(BTreeMap::new())),
14677            (
14678                "numeric".to_string(),
14679                AvroValue::Map(BTreeMap::from([
14680                    ("0".to_string(), AvroValue::String("zero".to_string())),
14681                    ("1".to_string(), AvroValue::String("one".to_string())),
14682                ])),
14683            ),
14684            (
14685                "nested".to_string(),
14686                AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([(
14687                    "enabled".to_string(),
14688                    AvroValue::Boolean(true),
14689                )]))]),
14690            ),
14691            (
14692                "projection_collisions".to_string(),
14693                AvroValue::Array(projection_collision_probe()),
14694            ),
14695        ]))
14696    }
14697
14698    fn projection_collision_probe() -> Vec<AvroValue> {
14699        vec![
14700            AvroValue::Map(BTreeMap::from([
14701                ("$type".to_string(), AvroValue::String("bytes".to_string())),
14702                (
14703                    "base64".to_string(),
14704                    AvroValue::String("ordinary user text".to_string()),
14705                ),
14706            ])),
14707            AvroValue::Map(BTreeMap::from([
14708                ("$type".to_string(), AvroValue::String("map".to_string())),
14709                (
14710                    "entries".to_string(),
14711                    AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([
14712                        ("key".to_string(), AvroValue::String("ordinary".to_string())),
14713                        (
14714                            "value".to_string(),
14715                            AvroValue::String("user map".to_string()),
14716                        ),
14717                    ]))]),
14718                ),
14719            ])),
14720        ]
14721    }
14722
14723    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
14724    struct TypedContract {
14725        nested: TypedNested,
14726        mode: TypedMode,
14727        optional: Option<String>,
14728        absent: Option<String>,
14729        items: Vec<i64>,
14730        labels: BTreeMap<String, String>,
14731        bytes: serde_bytes::ByteBuf,
14732        signed: i64,
14733        finite: f64,
14734    }
14735
14736    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
14737    struct TypedNested {
14738        enabled: bool,
14739    }
14740
14741    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
14742    enum TypedMode {
14743        Detailed { label: String },
14744    }
14745
14746    fn typed_contract() -> TypedContract {
14747        TypedContract {
14748            nested: TypedNested { enabled: true },
14749            mode: TypedMode::Detailed {
14750                label: "compiler-checked".to_string(),
14751            },
14752            optional: Some("present".to_string()),
14753            absent: None,
14754            items: vec![i64::MIN, 0, i64::MAX],
14755            labels: BTreeMap::from([
14756                ("language".to_string(), "rust".to_string()),
14757                ("wire".to_string(), "avro".to_string()),
14758            ]),
14759            bytes: serde_bytes::ByteBuf::from(vec![0, 0xff, 7]),
14760            signed: -9_223_372_036_854_775_000,
14761            finite: 12.5,
14762        }
14763    }
14764
14765    #[derive(Clone, Debug, Default, PartialEq)]
14766    struct ReplayCounterState {
14767        loaded: Option<String>,
14768        count: i64,
14769        finished: bool,
14770    }
14771
14772    fn replay_counter_worker() -> Worker {
14773        let client = Client::new("http://127.0.0.1:8080").expect("client");
14774        let mut worker = Worker::new(client, "rust-workers");
14775        worker.register_replayed_workflow(
14776            "replay-counter",
14777            ReplayCounterState::default,
14778            |ctx, _input, state| async move {
14779                let loaded = ctx.activity("load-counter", json!([])).await?;
14780                state.update(|current| {
14781                    current.loaded = loaded.as_str().map(str::to_string);
14782                })?;
14783                for _ in 0..2 {
14784                    let signal = ctx.wait_signal("increment").await?;
14785                    let amount = signal.first().and_then(Value::as_i64).unwrap_or_default();
14786                    state.update(|current| current.count += amount)?;
14787                }
14788                state.update(|current| current.finished = true)?;
14789                state.read(|current| Ok(json!(current.count)))?
14790            },
14791        );
14792        worker.register_replayed_query::<ReplayCounterState, _, _>(
14793            "replay-counter",
14794            "current",
14795            |_ctx, state, _args| async move {
14796                Ok(json!({
14797                    "loaded": state.loaded,
14798                    "count": state.count,
14799                    "finished": state.finished,
14800                }))
14801            },
14802        );
14803        worker.register_replayed_query::<ReplayCounterState, _, _>(
14804            "replay-counter",
14805            "detached-mutation",
14806            |_ctx, state, _args| async move {
14807                let mut detached = (*state).clone();
14808                detached.count = 999;
14809                Ok(json!(detached.count))
14810            },
14811        );
14812        worker.register_replayed_query::<ReplayCounterState, _, _>(
14813            "replay-counter",
14814            "failed-mutation",
14815            |_ctx, state, _args| async move {
14816                let mut detached = (*state).clone();
14817                detached.count = 999;
14818                Err(Error::WorkerLoop("query refused".to_string()))
14819            },
14820        );
14821        worker
14822    }
14823
14824    fn replay_counter_query(
14825        query_name: &str,
14826        history_events: Value,
14827        run_status: &str,
14828    ) -> QueryTask {
14829        let arguments = fixture_envelope(json!([]));
14830        serde_json::from_value(json!({
14831            "query_task_id": format!("query-{query_name}"),
14832            "workflow_type": "replay-counter",
14833            "query_name": query_name,
14834            "payload_codec": DEFAULT_CODEC,
14835            "workflow_arguments": arguments.clone(),
14836            "query_arguments": arguments,
14837            "history_events": history_events,
14838            "run_status": run_status,
14839        }))
14840        .expect("query task")
14841    }
14842
14843    fn workflow_context(history: Vec<HistoryEvent>) -> WorkflowContext {
14844        workflow_context_with_codec(history, DEFAULT_CODEC)
14845    }
14846
14847    fn workflow_context_with_codec(
14848        history: Vec<HistoryEvent>,
14849        payload_codec: &str,
14850    ) -> WorkflowContext {
14851        WorkflowContext {
14852            state: Arc::new(Mutex::new(
14853                WorkflowState::new_with_identity(
14854                    history,
14855                    None,
14856                    None,
14857                    "rust-workers".to_string(),
14858                    payload_codec.to_string(),
14859                    None,
14860                )
14861                .expect("valid workflow history"),
14862            )),
14863        }
14864    }
14865
14866    fn history_event(event_type: &str, payload: Value) -> HistoryEvent {
14867        HistoryEvent {
14868            event_type: event_type.to_string(),
14869            payload,
14870            raw: HashMap::new(),
14871        }
14872    }
14873
14874    fn parallel_path_entry(
14875        kind: &str,
14876        base: u64,
14877        size: usize,
14878        index: usize,
14879    ) -> ParallelGroupMetadata {
14880        parallel_group_entry(base, size, index, kind)
14881    }
14882
14883    fn parallel_history_event(
14884        event_type: &str,
14885        sequence: u64,
14886        identity_field: &str,
14887        identity: &str,
14888        path: Vec<ParallelGroupMetadata>,
14889        result: Option<Value>,
14890    ) -> HistoryEvent {
14891        let mut payload = serde_json::Map::from_iter([
14892            ("sequence".to_string(), json!(sequence)),
14893            (identity_field.to_string(), json!(identity)),
14894        ]);
14895        let inner = path.last().expect("parallel history path");
14896        apply_parallel_group_path(&mut payload, std::slice::from_ref(inner));
14897        payload.insert("parallel_group_path".to_string(), json!(path));
14898        if let Some(result) = result {
14899            let field = if event_type == "ChildRunCompleted" {
14900                "result"
14901            } else {
14902                "result"
14903            };
14904            payload.insert(field.to_string(), fixture_envelope(result));
14905            payload.insert("payload_codec".to_string(), json!(DEFAULT_CODEC));
14906        }
14907        history_event(event_type, Value::Object(payload))
14908    }
14909
14910    fn nested_parallel_operations() -> Vec<ParallelOperation> {
14911        vec![
14912            ParallelOperation::activity("first", json!([])),
14913            ParallelOperation::group(vec![
14914                ParallelOperation::child_workflow(
14915                    "second",
14916                    ChildWorkflowOptions::new("child-workers"),
14917                    json!([]),
14918                ),
14919                ParallelOperation::activity("third", json!([])),
14920            ]),
14921        ]
14922    }
14923
14924    fn nested_parallel_paths() -> [Vec<ParallelGroupMetadata>; 3] {
14925        let outer = [
14926            parallel_path_entry("mixed", 1, 3, 0),
14927            parallel_path_entry("mixed", 1, 3, 1),
14928            parallel_path_entry("mixed", 1, 3, 2),
14929        ];
14930        [
14931            vec![outer[0].clone()],
14932            vec![outer[1].clone(), parallel_path_entry("mixed", 2, 2, 0)],
14933            vec![outer[2].clone(), parallel_path_entry("mixed", 2, 2, 1)],
14934        ]
14935    }
14936
14937    #[test]
14938    fn parallel_schedules_every_nested_mixed_leaf_with_stable_metadata() {
14939        let ctx = workflow_context(Vec::new());
14940        let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
14941        let mut task_context = TaskContext::from_waker(noop_waker_ref());
14942
14943        assert!(matches!(
14944            call.as_mut().poll(&mut task_context),
14945            Poll::Pending
14946        ));
14947        let commands = ctx.take_commands().expect("parallel commands");
14948        assert_eq!(
14949            commands
14950                .iter()
14951                .map(|command| command["type"].as_str().unwrap_or_default())
14952                .collect::<Vec<_>>(),
14953            [
14954                "schedule_activity",
14955                "start_child_workflow",
14956                "schedule_activity"
14957            ]
14958        );
14959        let paths = nested_parallel_paths();
14960        for (command, path) in commands.iter().zip(paths) {
14961            assert_eq!(command["parallel_group_path"], json!(path));
14962            assert_eq!(
14963                command["parallel_group_id"],
14964                json!(path.last().expect("inner group").parallel_group_id)
14965            );
14966        }
14967    }
14968
14969    fn completed_nested_parallel_history() -> Vec<HistoryEvent> {
14970        let paths = nested_parallel_paths();
14971        let third = parallel_history_event(
14972            "ActivityCompleted",
14973            3,
14974            "activity_type",
14975            "third",
14976            paths[2].clone(),
14977            Some(json!("three")),
14978        );
14979        vec![
14980            parallel_history_event(
14981                "ActivityCompleted",
14982                1,
14983                "activity_type",
14984                "first",
14985                paths[0].clone(),
14986                Some(json!("one")),
14987            ),
14988            parallel_history_event(
14989                "ChildWorkflowScheduled",
14990                2,
14991                "child_workflow_type",
14992                "second",
14993                paths[1].clone(),
14994                None,
14995            ),
14996            parallel_history_event(
14997                "ChildRunCompleted",
14998                2,
14999                "child_workflow_type",
15000                "second",
15001                paths[1].clone(),
15002                Some(json!("two")),
15003            ),
15004            third.clone(),
15005            third,
15006        ]
15007    }
15008
15009    #[test]
15010    fn parallel_replay_rebuilds_input_order_and_tolerates_duplicate_delivery() {
15011        for _restart_or_completed_replay in 0..2 {
15012            let ctx = workflow_context(completed_nested_parallel_history());
15013            let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15014            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15015            let Poll::Ready(Ok(results)) = call.as_mut().poll(&mut task_context) else {
15016                panic!("completed nested parallel history must replay");
15017            };
15018            assert_eq!(
15019                results,
15020                vec![
15021                    ParallelResult::Activity(json!("one")),
15022                    ParallelResult::Group(vec![
15023                        ParallelResult::ChildWorkflow(ChildWorkflowResult {
15024                            parent: WorkflowIdentity {
15025                                workflow_id: None,
15026                                run_id: None,
15027                            },
15028                            child: WorkflowIdentity {
15029                                workflow_id: None,
15030                                run_id: None,
15031                            },
15032                            child_workflow_type: Some("second".to_string()),
15033                            result: json!("two"),
15034                        }),
15035                        ParallelResult::Activity(json!("three")),
15036                    ]),
15037                ]
15038            );
15039            assert!(ctx.take_commands().expect("commands").is_empty());
15040            ctx.ensure_history_consumed().expect("history consumed");
15041        }
15042    }
15043
15044    #[test]
15045    fn parallel_failure_keeps_typed_cause_path_and_late_completions() {
15046        let paths = nested_parallel_paths();
15047        let history = vec![
15048            parallel_history_event(
15049                "ActivityCompleted",
15050                1,
15051                "activity_type",
15052                "first",
15053                paths[0].clone(),
15054                Some(json!("one")),
15055            ),
15056            parallel_history_event(
15057                "ChildWorkflowScheduled",
15058                2,
15059                "child_workflow_type",
15060                "second",
15061                paths[1].clone(),
15062                None,
15063            ),
15064            parallel_history_event(
15065                "ChildRunFailed",
15066                2,
15067                "child_workflow_type",
15068                "second",
15069                paths[1].clone(),
15070                None,
15071            ),
15072            parallel_history_event(
15073                "ActivityCompleted",
15074                3,
15075                "activity_type",
15076                "third",
15077                paths[2].clone(),
15078                Some(json!("late")),
15079            ),
15080        ];
15081        let ctx = workflow_context(history);
15082        let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15083        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15084        let outcome = call.as_mut().poll(&mut task_context);
15085        let Poll::Ready(Err(Error::ParallelFailed(failure))) = outcome else {
15086            panic!("one failed child must return a typed partial failure: {outcome:?}");
15087        };
15088        assert_eq!(failure.member_path, [1, 0]);
15089        assert_eq!(failure.group_id, "parallel-calls:1:3");
15090        assert!(matches!(*failure.cause, Error::ChildWorkflowFailed(_)));
15091        assert_eq!(
15092            failure
15093                .completed
15094                .iter()
15095                .map(|completion| completion.member_path.clone())
15096                .collect::<Vec<_>>(),
15097            [vec![0], vec![1, 1]]
15098        );
15099    }
15100
15101    #[test]
15102    fn pending_parallel_history_restarts_without_rescheduling_any_leaf() {
15103        let paths = nested_parallel_paths();
15104        let history = vec![
15105            parallel_history_event(
15106                "ActivityScheduled",
15107                1,
15108                "activity_type",
15109                "first",
15110                paths[0].clone(),
15111                None,
15112            ),
15113            parallel_history_event(
15114                "ChildWorkflowScheduled",
15115                2,
15116                "child_workflow_type",
15117                "second",
15118                paths[1].clone(),
15119                None,
15120            ),
15121            parallel_history_event(
15122                "ActivityScheduled",
15123                3,
15124                "activity_type",
15125                "third",
15126                paths[2].clone(),
15127                None,
15128            ),
15129        ];
15130        for _restart in 0..2 {
15131            let ctx = workflow_context(history.clone());
15132            let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15133            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15134            let outcome = call.as_mut().poll(&mut task_context);
15135            assert!(matches!(outcome, Poll::Pending), "{outcome:?}");
15136            assert!(ctx.take_commands().expect("commands").is_empty());
15137        }
15138    }
15139
15140    fn selection_path(index: usize, key: &str) -> Vec<ParallelGroupMetadata> {
15141        vec![selection_group_entry(
15142            1,
15143            2,
15144            index,
15145            "activity",
15146            &SelectionMemberMetadata {
15147                key: SelectionKey::Name(key.to_string()),
15148                index,
15149                base_sequence: index as u64 + 1,
15150                size: 1,
15151                kind: "activity".to_string(),
15152            },
15153        )]
15154    }
15155
15156    fn selection_activity_event(
15157        event_type: &str,
15158        index: usize,
15159        key: &str,
15160        result: Option<Value>,
15161    ) -> HistoryEvent {
15162        let sequence = index as u64 + 1;
15163        let mut event = parallel_history_event(
15164            event_type,
15165            sequence,
15166            "activity_type",
15167            &format!("{key}-activity"),
15168            selection_path(index, key),
15169            result,
15170        );
15171        event.payload["activity_execution_id"] = json!(format!("activity-{key}"));
15172        event.raw.insert(
15173            "id".to_string(),
15174            json!(if event_type == "ActivityCompleted" {
15175                format!("event-{key}")
15176            } else {
15177                format!("{event_type}-{key}")
15178            }),
15179        );
15180        event
15181    }
15182
15183    fn selection_winner_marker() -> HistoryEvent {
15184        history_event(
15185            "SelectionResolved",
15186            json!({
15187                "selection_group_id": "select-calls:1:2",
15188                "selection_group_base_sequence": 1,
15189                "selection_group_size": 2,
15190                "member_key": "fast",
15191                "member_index": 1,
15192                "member_base_sequence": 2,
15193                "member_size": 1,
15194                "operation_kind": "activity",
15195                "operation_identity": "activity-fast",
15196                "outcome": "completed",
15197                "resolution_event_id": "event-fast",
15198                "resolution_event_type": "ActivityCompleted",
15199            }),
15200        )
15201    }
15202
15203    fn keyed_activity_selection(ctx: &WorkflowContext) -> SelectCall {
15204        ctx.select_keyed(vec![
15205            (
15206                "slow",
15207                ParallelOperation::activity_with_options(
15208                    "slow-activity",
15209                    ActivityOptions::new().task_queue("default"),
15210                    json!([]),
15211                ),
15212            ),
15213            (
15214                "fast",
15215                ParallelOperation::activity_with_options(
15216                    "fast-activity",
15217                    ActivityOptions::new().task_queue("default"),
15218                    json!([]),
15219                ),
15220            ),
15221        ])
15222    }
15223
15224    fn assert_persisted_selection_replay(history: Vec<HistoryEvent>) {
15225        let ctx = workflow_context(history);
15226        let mut call = Box::pin(keyed_activity_selection(&ctx));
15227        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15228        let selected = match call.as_mut().poll(&mut task_context) {
15229            Poll::Ready(Ok(selected)) => selected,
15230            Poll::Ready(Err(error)) => panic!("persisted selection winner must replay: {error:?}"),
15231            Poll::Pending => panic!("persisted selection winner must replay without pending"),
15232        };
15233        assert_eq!(selected.key, SelectionKey::Name("fast".to_string()));
15234        assert_eq!(
15235            selected.value,
15236            Some(ParallelResult::Activity(json!("winner-value")))
15237        );
15238        let slow = selected
15239            .handle(&SelectionKey::Name("slow".to_string()))
15240            .expect("slow handle")
15241            .clone();
15242        let mut await_slow = Box::pin(slow.await_result());
15243        assert!(matches!(
15244            await_slow.as_mut().poll(&mut task_context),
15245            Poll::Ready(Ok(ParallelResult::Activity(value))) if value == json!("loser-value")
15246        ));
15247        assert!(ctx.take_commands().expect("commands").is_empty());
15248    }
15249
15250    const SELECTION_COLD_REPLAY_HISTORY: &str = "DURABLE_WORKFLOW_SELECTION_COLD_REPLAY_HISTORY";
15251
15252    fn canonical_selection_history() -> Vec<HistoryEvent> {
15253        const FIXTURE: &[u8] =
15254            include_bytes!("../tests/fixtures/durable_selection_runtime_history.json");
15255        assert_eq!(
15256            format!("{:x}", Sha256::digest(FIXTURE)),
15257            "51fd8b9c16e978dcef536a5c727b9fdc0ae724d9afc17d9a7837d219f41ee3ba",
15258        );
15259        let fixture: Value = serde_json::from_slice(FIXTURE).expect("canonical selection fixture");
15260
15261        serde_json::from_value(fixture["history"].clone()).expect("canonical selection history")
15262    }
15263
15264    #[test]
15265    fn selection_fresh_process_entrypoint() {
15266        let Ok(path) = std::env::var(SELECTION_COLD_REPLAY_HISTORY) else {
15267            return;
15268        };
15269        let persisted = fs::read(path).expect("persisted selection history");
15270        assert_eq!(
15271            format!("{:x}", Sha256::digest(&persisted)),
15272            "51fd8b9c16e978dcef536a5c727b9fdc0ae724d9afc17d9a7837d219f41ee3ba",
15273        );
15274        let fixture: Value =
15275            serde_json::from_slice(&persisted).expect("valid persisted selection fixture");
15276        let history: Vec<HistoryEvent> = serde_json::from_value(fixture["history"].clone())
15277            .expect("valid persisted selection history");
15278
15279        assert_persisted_selection_replay(history);
15280    }
15281
15282    #[test]
15283    fn selection_starts_every_member_with_stable_keys_and_group_identity() {
15284        let ctx = workflow_context(Vec::new());
15285        let mut call = Box::pin(keyed_activity_selection(&ctx));
15286        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15287
15288        assert!(matches!(
15289            call.as_mut().poll(&mut task_context),
15290            Poll::Pending
15291        ));
15292        let commands = ctx.take_commands().expect("selection commands");
15293        assert_eq!(commands.len(), 2);
15294        assert_eq!(commands[0]["selection_member_key"], json!("slow"));
15295        assert_eq!(commands[1]["selection_member_key"], json!("fast"));
15296        assert!(commands.iter().all(|command| {
15297            command["parallel_group_id"] == json!("select-calls:1:2")
15298                && command["parallel_group_mode"] == json!("select")
15299        }));
15300    }
15301
15302    #[test]
15303    fn selection_key_domain_rejects_empty_authoring_and_malformed_history() {
15304        let ctx = workflow_context(Vec::new());
15305        let mut invalid = Box::pin(ctx.select_keyed(vec![(
15306            "",
15307            ParallelOperation::activity("invalid", json!([])),
15308        )]));
15309        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15310        assert!(matches!(
15311            invalid.as_mut().poll(&mut task_context),
15312            Poll::Ready(Err(Error::InvalidParallelGroup(ParallelGroupError {
15313                reason: "selection_key_invalid",
15314                ..
15315            })))
15316        ));
15317
15318        for invalid_key in [json!(""), json!(-1)] {
15319            let mut event = selection_activity_event("ActivityScheduled", 0, "slow", None);
15320            event.payload["selection_member_key"] = invalid_key.clone();
15321            event.payload["parallel_group_path"][0]["selection_member_key"] = invalid_key;
15322            assert!(matches!(
15323                WorkflowState::new_with_identity(
15324                    vec![event],
15325                    None,
15326                    None,
15327                    "rust-workers".to_string(),
15328                    DEFAULT_CODEC.to_string(),
15329                    None,
15330                ),
15331                Err(Error::NonDeterministicReplay(_))
15332            ));
15333        }
15334    }
15335
15336    #[test]
15337    fn selection_preserves_valid_named_and_numeric_keys() {
15338        let ctx = workflow_context(Vec::new());
15339        let mut selection = Box::pin(ctx.select_keyed(vec![
15340            (
15341                SelectionKey::Index(0),
15342                ParallelOperation::activity("numeric", json!([])),
15343            ),
15344            (
15345                SelectionKey::Name("named".to_string()),
15346                ParallelOperation::timer(Duration::from_secs(1)),
15347            ),
15348        ]));
15349        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15350
15351        assert!(matches!(
15352            selection.as_mut().poll(&mut task_context),
15353            Poll::Pending
15354        ));
15355        let commands = ctx.take_commands().expect("selection commands");
15356        assert_eq!(commands[0]["selection_member_key"], json!(0));
15357        assert_eq!(commands[1]["selection_member_key"], json!("named"));
15358    }
15359
15360    #[test]
15361    fn selection_replays_persisted_winner_and_loser_can_be_awaited_later() {
15362        let history = canonical_selection_history();
15363        assert_persisted_selection_replay(history.clone());
15364
15365        let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
15366            .join("tests/fixtures/durable_selection_runtime_history.json");
15367        let output =
15368            ProcessCommand::new(std::env::current_exe().expect("current Rust test binary"))
15369                .args([
15370                    "--exact",
15371                    "tests::selection_fresh_process_entrypoint",
15372                    "--nocapture",
15373                ])
15374                .env(SELECTION_COLD_REPLAY_HISTORY, &path)
15375                .output()
15376                .expect("run fresh selection replay process");
15377
15378        assert!(
15379            output.status.success(),
15380            "fresh selection replay failed:\nstdout:\n{}\nstderr:\n{}",
15381            String::from_utf8_lossy(&output.stdout),
15382            String::from_utf8_lossy(&output.stderr),
15383        );
15384    }
15385
15386    #[test]
15387    fn selection_waits_durably_when_terminal_members_precede_the_winner_marker() {
15388        let mut history = canonical_selection_history();
15389        history.retain(|event| event.event_type != "SelectionResolved");
15390        let ctx = workflow_context(history);
15391        let mut selection = Box::pin(keyed_activity_selection(&ctx));
15392        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15393
15394        assert!(matches!(
15395            selection.as_mut().poll(&mut task_context),
15396            Poll::Pending
15397        ));
15398        assert!(ctx.take_commands().expect("commands").is_empty());
15399        assert!(
15400            ctx.matched_recorded_pending()
15401                .expect("selection pending state"),
15402            "terminal member history must keep the workflow durably pending until SelectionResolved commits"
15403        );
15404    }
15405
15406    #[test]
15407    fn selection_terminal_condition_history_waits_durably_for_its_winner_marker() {
15408        for (terminal_event, predicate_satisfied, timeout_seconds) in [
15409            ("ConditionWaitSatisfied", true, None),
15410            ("ConditionWaitTimedOut", false, Some(0)),
15411        ] {
15412            let member = SelectionMemberMetadata {
15413                key: SelectionKey::Name("condition".to_string()),
15414                index: 0,
15415                base_sequence: 1,
15416                size: 1,
15417                kind: "condition".to_string(),
15418            };
15419            let path = vec![selection_group_entry(1, 1, 0, "condition", &member)];
15420            let mut payload = json!({
15421                "sequence": 1,
15422                "condition_wait_id": "condition-1",
15423                "condition_wait_occurrence_id": "rust:condition-wait:0",
15424                "condition_key": "ready",
15425                "condition_definition_fingerprint": "sha256:ready-v1",
15426                "parallel_group_path": path,
15427            });
15428            payload
15429                .as_object_mut()
15430                .expect("condition history payload")
15431                .extend(
15432                    serde_json::to_value(&path[0])
15433                        .expect("condition selection metadata")
15434                        .as_object()
15435                        .expect("condition selection metadata object")
15436                        .clone(),
15437                );
15438            if let Some(timeout_seconds) = timeout_seconds {
15439                payload["timeout_seconds"] = json!(timeout_seconds);
15440            }
15441            let history = vec![
15442                history_event("ConditionWaitOpened", payload.clone()),
15443                history_event(terminal_event, payload),
15444            ];
15445            let ctx = workflow_context(history);
15446            let mut options = ConditionWaitOptions::new("ready", "sha256:ready-v1");
15447            if timeout_seconds.is_some() {
15448                options = options.timeout(Duration::ZERO);
15449            }
15450            let mut selection = Box::pin(ctx.select_keyed(vec![(
15451                "condition",
15452                ParallelOperation::condition(options, move || Ok(predicate_satisfied)),
15453            )]));
15454            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15455
15456            assert!(matches!(
15457                selection.as_mut().poll(&mut task_context),
15458                Poll::Pending
15459            ));
15460            assert!(ctx.take_commands().expect("commands").is_empty());
15461            assert!(
15462                ctx.matched_recorded_pending()
15463                    .expect("condition selection pending state"),
15464                "{terminal_event} must keep the workflow durably pending until SelectionResolved commits"
15465            );
15466        }
15467    }
15468
15469    #[test]
15470    fn selection_immediate_condition_members_open_a_durable_wait() {
15471        for predicate_satisfied in [true, false] {
15472            let ctx = workflow_context(Vec::new());
15473            let mut selection = Box::pin(ctx.select_keyed(vec![(
15474                "condition",
15475                ParallelOperation::condition(
15476                    ConditionWaitOptions::new("ready", "sha256:ready-v1").timeout(Duration::ZERO),
15477                    move || Ok(predicate_satisfied),
15478                ),
15479            )]));
15480            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15481
15482            assert!(matches!(
15483                selection.as_mut().poll(&mut task_context),
15484                Poll::Pending
15485            ));
15486            let commands = ctx.take_commands().expect("condition selection command");
15487            assert_eq!(commands.len(), 1);
15488            assert_eq!(commands[0]["type"], json!("open_condition_wait"));
15489            assert_eq!(commands[0]["timeout_seconds"], json!(0));
15490            assert_eq!(
15491                commands[0]["parallel_group_path"][0]["parallel_group_mode"],
15492                json!("select")
15493            );
15494        }
15495    }
15496
15497    #[test]
15498    fn selection_loser_cancellation_is_explicit_and_idempotent() {
15499        let history = vec![
15500            selection_activity_event("ActivityScheduled", 0, "slow", None),
15501            selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
15502            selection_winner_marker(),
15503        ];
15504        let ctx = workflow_context(history.clone());
15505        let mut call = Box::pin(keyed_activity_selection(&ctx));
15506        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15507        let Poll::Ready(Ok(selected)) = call.as_mut().poll(&mut task_context) else {
15508            panic!("winner must replay");
15509        };
15510        let slow = selected
15511            .handle(&SelectionKey::Name("slow".to_string()))
15512            .expect("slow handle")
15513            .clone();
15514        let mut cancel = Box::pin(slow.cancel());
15515        assert!(matches!(
15516            cancel.as_mut().poll(&mut task_context),
15517            Poll::Pending
15518        ));
15519        assert!(matches!(
15520            cancel.as_mut().poll(&mut task_context),
15521            Poll::Pending
15522        ));
15523        let commands = ctx.take_commands().expect("cancel command");
15524        assert_eq!(commands.len(), 1);
15525        assert_eq!(commands[0]["type"], json!("cancel_selection_operation"));
15526        assert_eq!(commands[0]["member_key"], json!("slow"));
15527
15528        let mut cancelled_history = history;
15529        cancelled_history.push(history_event(
15530            "SelectionOperationCancelled",
15531            json!({
15532                "selection_group_id": "select-calls:1:2",
15533                "member_key": "slow",
15534                "member_index": 0,
15535                "member_base_sequence": 1,
15536                "member_size": 1,
15537                "operation_kind": "activity",
15538                "operation_identity": "activity-slow",
15539                "cancelled_at": "2026-08-27T00:00:00Z",
15540            }),
15541        ));
15542        let replayed = workflow_context(cancelled_history);
15543        let mut call = Box::pin(keyed_activity_selection(&replayed));
15544        let Poll::Ready(Ok(selected)) = call.as_mut().poll(&mut task_context) else {
15545            panic!("winner must replay after cancellation");
15546        };
15547        let slow = selected
15548            .handle(&SelectionKey::Name("slow".to_string()))
15549            .expect("slow handle")
15550            .clone();
15551        let mut cancel = Box::pin(slow.cancel());
15552        assert!(matches!(
15553            cancel.as_mut().poll(&mut task_context),
15554            Poll::Ready(Ok(()))
15555        ));
15556        assert!(replayed.take_commands().expect("commands").is_empty());
15557    }
15558
15559    #[test]
15560    fn selection_cancellation_marker_is_bound_to_every_authored_handle_field() {
15561        let base_history = vec![
15562            selection_activity_event("ActivityScheduled", 0, "slow", None),
15563            selection_activity_event("ActivityScheduled", 1, "fast", None),
15564            selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
15565            selection_winner_marker(),
15566        ];
15567        for (field, corrupt) in [
15568            ("member_key", json!("fast")),
15569            ("member_index", json!(1)),
15570            ("member_base_sequence", json!(3)),
15571            ("member_size", json!(2)),
15572            ("operation_kind", json!("timer")),
15573            ("operation_identity", json!("forged")),
15574        ] {
15575            let mut cancellation = json!({
15576                "selection_group_id": "select-calls:1:2",
15577                "member_key": "slow",
15578                "member_index": 0,
15579                "member_base_sequence": 1,
15580                "member_size": 1,
15581                "operation_kind": "activity",
15582                "operation_identity": "activity-slow",
15583            });
15584            cancellation[field] = corrupt;
15585            let mut history = base_history.clone();
15586            history.push(history_event("SelectionOperationCancelled", cancellation));
15587            let ctx = workflow_context(history);
15588            let mut selection = Box::pin(keyed_activity_selection(&ctx));
15589            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15590
15591            assert!(matches!(
15592                selection.as_mut().poll(&mut task_context),
15593                Poll::Ready(Err(Error::NonDeterministicReplay(_)))
15594            ));
15595        }
15596    }
15597
15598    #[test]
15599    fn selection_child_identity_prefers_the_durable_run_id() {
15600        let ctx = workflow_context(vec![history_event(
15601            "ChildWorkflowScheduled",
15602            json!({
15603                "sequence": 1,
15604                "child_workflow_type": "child",
15605                "child_workflow_instance_id": "child-instance",
15606                "child_workflow_run_id": "child-run",
15607            }),
15608        )]);
15609        let state = ctx.state.lock().expect("workflow state");
15610
15611        assert_eq!(
15612            selection_operation_identity(&state, "child", 1, 1),
15613            "child-run"
15614        );
15615    }
15616
15617    #[test]
15618    fn selection_activity_identity_requires_canonical_execution_id() {
15619        let slow = selection_activity_event("ActivityScheduled", 0, "slow", None);
15620        let mut fast_open = selection_activity_event("ActivityScheduled", 1, "fast", None);
15621        let mut fast_completed =
15622            selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner")));
15623        for event in [&mut fast_open, &mut fast_completed] {
15624            event
15625                .payload
15626                .as_object_mut()
15627                .expect("activity payload")
15628                .remove("activity_execution_id");
15629            event.payload["activity_id"] = json!("forged-activity-id");
15630        }
15631        let mut marker = selection_winner_marker();
15632        marker.payload["operation_identity"] = json!("forged-activity-id");
15633        let ctx = workflow_context(vec![slow, fast_open, fast_completed, marker]);
15634        let mut selection = Box::pin(keyed_activity_selection(&ctx));
15635        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15636
15637        assert!(matches!(
15638            selection.as_mut().poll(&mut task_context),
15639            Poll::Ready(Err(Error::NonDeterministicReplay(_)))
15640        ));
15641    }
15642
15643    #[test]
15644    fn selection_completion_before_cancellation_remains_awaitable() {
15645        let history = vec![
15646            selection_activity_event("ActivityScheduled", 0, "slow", None),
15647            selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
15648            selection_winner_marker(),
15649            selection_activity_event(
15650                "ActivityCompleted",
15651                0,
15652                "slow",
15653                Some(json!("completed-first")),
15654            ),
15655        ];
15656        let ctx = workflow_context(history);
15657        let mut selection = Box::pin(keyed_activity_selection(&ctx));
15658        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15659        let Poll::Ready(Ok(selected)) = selection.as_mut().poll(&mut task_context) else {
15660            panic!("winner must replay");
15661        };
15662        let slow = selected
15663            .handle(&SelectionKey::Name("slow".to_string()))
15664            .expect("slow handle")
15665            .clone();
15666        let mut cancel = Box::pin(slow.cancel());
15667        assert!(matches!(
15668            cancel.as_mut().poll(&mut task_context),
15669            Poll::Ready(Ok(()))
15670        ));
15671        let mut await_slow = Box::pin(slow.await_result());
15672        assert!(matches!(
15673            await_slow.as_mut().poll(&mut task_context),
15674            Poll::Ready(Ok(ParallelResult::Activity(value))) if value == json!("completed-first")
15675        ));
15676        let commands = ctx.take_commands().expect("commands");
15677        assert!(commands.is_empty());
15678    }
15679
15680    #[test]
15681    fn selection_nested_later_failure_before_cancel_remains_the_awaited_failure() {
15682        let nested_member = SelectionMemberMetadata {
15683            key: SelectionKey::Name("nested".to_string()),
15684            index: 0,
15685            base_sequence: 1,
15686            size: 2,
15687            kind: "group".to_string(),
15688        };
15689        let deadline_member = SelectionMemberMetadata {
15690            key: SelectionKey::Name("deadline".to_string()),
15691            index: 1,
15692            base_sequence: 3,
15693            size: 1,
15694            kind: "timer".to_string(),
15695        };
15696        let nested_paths = [
15697            vec![
15698                selection_group_entry(1, 3, 0, "mixed", &nested_member),
15699                parallel_group_entry(1, 2, 0, "activity"),
15700            ],
15701            vec![
15702                selection_group_entry(1, 3, 1, "mixed", &nested_member),
15703                parallel_group_entry(1, 2, 1, "activity"),
15704            ],
15705        ];
15706        let deadline_path = vec![selection_group_entry(1, 3, 2, "mixed", &deadline_member)];
15707        let mut timer_fired = parallel_history_event(
15708            "TimerFired",
15709            3,
15710            "timer_id",
15711            "timer-3",
15712            deadline_path.clone(),
15713            None,
15714        );
15715        timer_fired.payload["delay_seconds"] = json!(0);
15716        timer_fired
15717            .raw
15718            .insert("id".to_string(), json!("timer-fired"));
15719        let mut timer_scheduled = parallel_history_event(
15720            "TimerScheduled",
15721            3,
15722            "timer_id",
15723            "timer-3",
15724            deadline_path,
15725            None,
15726        );
15727        timer_scheduled.payload["delay_seconds"] = json!(0);
15728        let history = vec![
15729            parallel_history_event(
15730                "ActivityScheduled",
15731                1,
15732                "activity_type",
15733                "nested-first",
15734                nested_paths[0].clone(),
15735                None,
15736            ),
15737            parallel_history_event(
15738                "ActivityScheduled",
15739                2,
15740                "activity_type",
15741                "nested-second",
15742                nested_paths[1].clone(),
15743                None,
15744            ),
15745            timer_scheduled,
15746            timer_fired,
15747            history_event(
15748                "SelectionResolved",
15749                json!({
15750                    "selection_group_id": "select-calls:1:3",
15751                    "selection_group_base_sequence": 1,
15752                    "selection_group_size": 3,
15753                    "member_key": "deadline",
15754                    "member_index": 1,
15755                    "member_base_sequence": 3,
15756                    "member_size": 1,
15757                    "operation_kind": "timer",
15758                    "operation_identity": "timer-3",
15759                    "outcome": "completed",
15760                    "resolution_event_id": "timer-fired",
15761                    "resolution_event_type": "TimerFired",
15762                }),
15763            ),
15764            parallel_history_event(
15765                "ActivityFailed",
15766                2,
15767                "activity_type",
15768                "nested-second",
15769                nested_paths[1].clone(),
15770                None,
15771            ),
15772        ];
15773        let ctx = workflow_context(history);
15774        let mut selection = Box::pin(ctx.select_keyed(vec![
15775            (
15776                "nested",
15777                ParallelOperation::group(vec![
15778                    ParallelOperation::activity("nested-first", json!([])),
15779                    ParallelOperation::activity("nested-second", json!([])),
15780                ]),
15781            ),
15782            ("deadline", ParallelOperation::timer(Duration::ZERO)),
15783        ]));
15784        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15785        let Poll::Ready(Ok(selected)) = selection.as_mut().poll(&mut task_context) else {
15786            panic!("deadline winner must replay");
15787        };
15788        let nested = selected
15789            .handle(&SelectionKey::Name("nested".to_string()))
15790            .expect("nested handle")
15791            .clone();
15792        let mut cancel = Box::pin(nested.cancel());
15793        assert!(matches!(
15794            cancel.as_mut().poll(&mut task_context),
15795            Poll::Ready(Ok(()))
15796        ));
15797        let mut await_nested = Box::pin(nested.await_result());
15798
15799        assert!(matches!(
15800            await_nested.as_mut().poll(&mut task_context),
15801            Poll::Ready(Err(Error::ActivityFailed(_)))
15802        ));
15803        assert!(ctx.take_commands().expect("commands").is_empty());
15804    }
15805
15806    #[test]
15807    fn selection_supports_child_timer_signal_condition_and_nested_groups() {
15808        let ctx = workflow_context(Vec::new());
15809        let mut call = Box::pin(ctx.select(vec![
15810            ParallelOperation::child_workflow(
15811                "child",
15812                ChildWorkflowOptions::new("children"),
15813                json!([]),
15814            ),
15815            ParallelOperation::timer(Duration::from_secs(30)),
15816            ParallelOperation::signal("approval"),
15817            ParallelOperation::condition(
15818                ConditionWaitOptions::new("ready", "sha256:ready"),
15819                || Ok(false),
15820            ),
15821            ParallelOperation::group(vec![
15822                ParallelOperation::activity("nested-one", json!([])),
15823                ParallelOperation::activity("nested-two", json!([])),
15824            ]),
15825        ]));
15826        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15827        assert!(matches!(
15828            call.as_mut().poll(&mut task_context),
15829            Poll::Pending
15830        ));
15831        let commands = ctx.take_commands().expect("selection commands");
15832        assert_eq!(
15833            commands
15834                .iter()
15835                .map(|command| command["type"].as_str().unwrap_or_default())
15836                .collect::<Vec<_>>(),
15837            [
15838                "start_child_workflow",
15839                "start_timer",
15840                "open_signal_wait",
15841                "open_condition_wait",
15842                "schedule_activity",
15843                "schedule_activity",
15844            ]
15845        );
15846        assert!(commands.iter().all(|command| {
15847            command["parallel_group_path"][0]["parallel_group_mode"] == json!("select")
15848        }));
15849        assert_eq!(
15850            commands[4]["parallel_group_path"].as_array().map(Vec::len),
15851            Some(2)
15852        );
15853        assert_eq!(
15854            commands[4]["parallel_group_path"][0]["selection_member_kind"],
15855            json!("group")
15856        );
15857        assert_eq!(
15858            commands[5]["parallel_group_path"][0]["selection_member_kind"],
15859            json!("group")
15860        );
15861
15862        let one_leaf_ctx = workflow_context(Vec::new());
15863        let mut one_leaf = Box::pin(one_leaf_ctx.select(vec![ParallelOperation::group(vec![
15864            ParallelOperation::activity("nested-only", json!([])),
15865        ])]));
15866        assert!(matches!(
15867            one_leaf.as_mut().poll(&mut task_context),
15868            Poll::Pending
15869        ));
15870        let one_leaf_commands = one_leaf_ctx.take_commands().expect("one-leaf commands");
15871        assert_eq!(one_leaf_commands.len(), 1);
15872        assert_eq!(
15873            one_leaf_commands[0]["parallel_group_path"][0]["selection_member_kind"],
15874            json!("group")
15875        );
15876        assert_eq!(
15877            one_leaf_commands[0]["parallel_group_path"][0]["selection_member_size"],
15878            json!(1)
15879        );
15880    }
15881
15882    async fn trip_saga(ctx: WorkflowContext) -> Result<Value> {
15883        let mut saga = ctx.saga();
15884        let outcome = async {
15885            let flight = ctx.activity("trip.reserve-flight", json!([])).await?;
15886            saga.add_compensation("trip.cancel-flight", json!([flight]))?;
15887            let hotel = ctx.activity("trip.reserve-hotel", json!([])).await?;
15888            saga.add_compensation("trip.cancel-hotel", json!([hotel]))?;
15889            ctx.activity("trip.charge", json!([])).await?;
15890            Ok(json!({"status": "booked"}))
15891        }
15892        .await;
15893        saga.finish(outcome).await
15894    }
15895
15896    fn saga_activity(
15897        event_type: &str,
15898        sequence: u64,
15899        activity_type: &str,
15900        result: Option<Value>,
15901    ) -> HistoryEvent {
15902        let mut payload = json!({
15903            "sequence": sequence,
15904            "activity_type": activity_type,
15905            "message": format!("{activity_type} failed"),
15906            "exception_type": "PlannedFailure",
15907            "non_retryable": true,
15908        });
15909        if let Some(result) = result {
15910            payload["result"] = fixture_envelope(result);
15911        }
15912        history_event(event_type, payload)
15913    }
15914
15915    #[test]
15916    fn saga_replays_reverse_compensation_across_restart_and_duplicate_delivery() {
15917        let completed_hotel_compensation = saga_activity(
15918            "ActivityCompleted",
15919            4,
15920            "trip.cancel-hotel",
15921            Some(Value::Null),
15922        );
15923        let history = vec![
15924            saga_activity(
15925                "ActivityCompleted",
15926                1,
15927                "trip.reserve-flight",
15928                Some(json!("flight-1")),
15929            ),
15930            saga_activity(
15931                "ActivityCompleted",
15932                2,
15933                "trip.reserve-hotel",
15934                Some(json!("hotel-1")),
15935            ),
15936            saga_activity("ActivityFailed", 3, "trip.charge", None),
15937            completed_hotel_compensation.clone(),
15938            completed_hotel_compensation,
15939        ];
15940
15941        for _restart in 0..2 {
15942            let ctx = workflow_context(history.clone());
15943            let mut future = Box::pin(trip_saga(ctx.clone()));
15944            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15945            assert!(matches!(
15946                future.as_mut().poll(&mut task_context),
15947                Poll::Pending
15948            ));
15949            let commands = ctx.take_commands().expect("compensation command");
15950            assert_eq!(commands.len(), 1);
15951            assert_eq!(commands[0]["activity_type"], "trip.cancel-flight");
15952        }
15953    }
15954
15955    #[test]
15956    fn saga_compensation_failure_preserves_both_typed_failures() {
15957        let history = vec![
15958            saga_activity(
15959                "ActivityCompleted",
15960                1,
15961                "trip.reserve-flight",
15962                Some(json!("flight-1")),
15963            ),
15964            saga_activity(
15965                "ActivityCompleted",
15966                2,
15967                "trip.reserve-hotel",
15968                Some(json!("hotel-1")),
15969            ),
15970            saga_activity("ActivityFailed", 3, "trip.charge", None),
15971            saga_activity("ActivityFailed", 4, "trip.cancel-hotel", None),
15972        ];
15973        let ctx = workflow_context(history);
15974        let mut future = Box::pin(trip_saga(ctx));
15975        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15976        let Poll::Ready(Err(Error::SagaCompensationFailed(failure))) =
15977            future.as_mut().poll(&mut task_context)
15978        else {
15979            panic!("compensation failure must remain structured");
15980        };
15981        assert!(matches!(
15982            *failure.initiating_failure,
15983            Error::ActivityFailed(_)
15984        ));
15985        assert!(matches!(
15986            *failure.compensation_failure,
15987            Error::ActivityFailed(_)
15988        ));
15989        assert_eq!(failure.compensation_activity_type, "trip.cancel-hotel");
15990        assert_eq!(failure.compensation_registration_order, 2);
15991    }
15992
15993    #[test]
15994    fn saga_compensates_cooperative_cancellation() {
15995        let ctx = workflow_context(vec![saga_activity(
15996            "ActivityCompleted",
15997            1,
15998            "trip.reserve-flight",
15999            Some(json!("flight-1")),
16000        )]);
16001        ctx.state.lock().expect("state").cancel_requested = true;
16002        let run = {
16003            let ctx = ctx.clone();
16004            async move {
16005                let mut saga = ctx.saga();
16006                let outcome = async {
16007                    let flight = ctx.activity("trip.reserve-flight", json!([])).await?;
16008                    saga.add_compensation("trip.cancel-flight", json!([flight]))?;
16009                    ctx.throw_if_cancellation_requested()?;
16010                    Ok(json!("unexpected"))
16011                }
16012                .await;
16013                saga.finish(outcome).await
16014            }
16015        };
16016        let mut future = Box::pin(run);
16017        let mut task_context = TaskContext::from_waker(noop_waker_ref());
16018        assert!(matches!(
16019            future.as_mut().poll(&mut task_context),
16020            Poll::Pending
16021        ));
16022        let commands = ctx.take_commands().expect("cancellation compensation");
16023        assert_eq!(commands[0]["activity_type"], "trip.cancel-flight");
16024    }
16025
16026    fn workflow_task(
16027        workflow_type: &str,
16028        history_events: Vec<HistoryEvent>,
16029        payload_codec: &str,
16030    ) -> WorkflowTask {
16031        WorkflowTask {
16032            task_id: format!("wft-{workflow_type}"),
16033            workflow_command_id: None,
16034            workflow_id: Some(format!("wf-{workflow_type}")),
16035            run_id: Some(format!("run-{workflow_type}")),
16036            workflow_type: workflow_type.to_string(),
16037            cancel_requested: false,
16038            payload_codec: payload_codec.to_string(),
16039            arguments: Some(
16040                encode_value_envelope(&json!([]), payload_codec).expect("workflow arguments"),
16041            ),
16042            total_history_events: Some(history_events.len() as u64),
16043            history_size_bytes: None,
16044            continue_as_new_recommended: None,
16045            history_budget_pressure: None,
16046            history_events,
16047            next_history_page_token: None,
16048            workflow_task_attempt: 1,
16049            workflow_signal_id: None,
16050            signal_name: None,
16051            signal_arguments: None,
16052            workflow_update_id: None,
16053            update_name: None,
16054            lease_owner: Some("rust-worker".to_string()),
16055        }
16056    }
16057
16058    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
16059    struct SideEffectProbe {
16060        request_id: String,
16061        attempt: u32,
16062    }
16063
16064    #[test]
16065    fn typed_side_effect_runs_callback_once_and_replay_skips_it() {
16066        let calls = AtomicUsize::new(0);
16067        let ctx = workflow_context(Vec::new());
16068        let value = ctx
16069            .side_effect(|| {
16070                calls.fetch_add(1, Ordering::SeqCst);
16071                SideEffectProbe {
16072                    request_id: "request-42".to_string(),
16073                    attempt: 3,
16074                }
16075            })
16076            .expect("first side effect");
16077        assert_eq!(value.attempt, 3);
16078        assert_eq!(calls.load(Ordering::SeqCst), 1);
16079        let commands = ctx.take_commands().expect("commands");
16080        assert_eq!(commands.len(), 1);
16081        assert_eq!(commands[0]["type"], "record_side_effect");
16082        assert_eq!(
16083            decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
16084            serde_json::to_value(&value).expect("value")
16085        );
16086
16087        let replay = workflow_context(vec![history_event(
16088            "SideEffectRecorded",
16089            json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16090        )]);
16091        let replayed: SideEffectProbe = replay
16092            .side_effect(|| {
16093                calls.fetch_add(1, Ordering::SeqCst);
16094                panic!("committed side-effect callbacks must not run during replay")
16095            })
16096            .expect("replayed side effect");
16097        assert_eq!(replayed, value);
16098        assert_eq!(calls.load(Ordering::SeqCst), 1);
16099        assert!(replay.take_commands().expect("commands").is_empty());
16100        replay.ensure_history_consumed().expect("history consumed");
16101    }
16102
16103    #[test]
16104    fn side_effect_uses_avro_envelope_and_uuid_is_replay_stable() {
16105        let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
16106        let value = ctx
16107            .side_effect(|| SideEffectProbe {
16108                request_id: "avro-request".to_string(),
16109                attempt: 1,
16110            })
16111            .expect("Avro side effect");
16112        let uuid = ctx.uuid_v4().expect("deterministic UUID");
16113        let commands = ctx.take_commands().expect("commands");
16114        assert_eq!(commands.len(), 2);
16115        assert_eq!(commands[0]["result"]["codec"], DEFAULT_CODEC);
16116        assert_eq!(commands[1]["result"]["codec"], DEFAULT_CODEC);
16117        assert_eq!(
16118            decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
16119            serde_json::to_value(&value).expect("value")
16120        );
16121
16122        let replay = workflow_context_with_codec(
16123            vec![
16124                history_event(
16125                    "SideEffectRecorded",
16126                    json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16127                ),
16128                history_event(
16129                    "SideEffectRecorded",
16130                    json!({"sequence": 2, "result": commands[1]["result"].clone()}),
16131                ),
16132            ],
16133            DEFAULT_CODEC,
16134        );
16135        let replayed: SideEffectProbe = replay
16136            .side_effect(|| panic!("Avro callback must not run"))
16137            .expect("replayed Avro value");
16138        let replayed_uuid = replay.uuid_v4().expect("replayed UUID");
16139        assert_eq!(replayed, value);
16140        assert_eq!(replayed_uuid, uuid);
16141        assert!(replay.take_commands().expect("commands").is_empty());
16142    }
16143
16144    #[test]
16145    fn typed_side_effect_replay_preserves_bytes_and_maps() {
16146        let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
16147        let value = ctx
16148            .side_effect_avro_value(typed_fidelity_probe)
16149            .expect("typed side effect");
16150        let commands = ctx.take_commands().expect("side-effect command");
16151        assert_eq!(
16152            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
16153                .expect("recorded side effect"),
16154            value
16155        );
16156
16157        let replay = workflow_context_with_codec(
16158            vec![history_event(
16159                "SideEffectRecorded",
16160                json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16161            )],
16162            DEFAULT_CODEC,
16163        );
16164        assert_eq!(
16165            replay
16166                .side_effect_avro_value(|| panic!("replay must not invoke callback"))
16167                .expect("replayed typed side effect"),
16168            value
16169        );
16170    }
16171
16172    #[test]
16173    fn ordered_side_effects_share_the_durable_command_stream() {
16174        let first = encode_value_envelope(&json!("first"), DEFAULT_CODEC).expect("first");
16175        let second = encode_value_envelope(&json!(29), DEFAULT_CODEC).expect("second");
16176        let ctx = workflow_context(vec![
16177            history_event(
16178                "SideEffectRecorded",
16179                json!({"sequence": 1, "result": first}),
16180            ),
16181            history_event(
16182                "SideEffectRecorded",
16183                json!({"sequence": 2, "result": second}),
16184            ),
16185        ]);
16186        let first: String = ctx
16187            .side_effect(|| panic!("first callback must not run"))
16188            .expect("first replay");
16189        let second: i32 = ctx
16190            .side_effect(|| panic!("second callback must not run"))
16191            .expect("second replay");
16192        assert_eq!(first, "first");
16193        assert_eq!(second, 29);
16194        ctx.ensure_history_consumed().expect("ordered history");
16195
16196        let reordered = workflow_context(vec![history_event(
16197            "VersionMarkerRecorded",
16198            json!({
16199                "sequence": 1,
16200                "change_id": "before-side-effect",
16201                "version": 1,
16202                "min_supported": 1,
16203                "max_supported": 1,
16204            }),
16205        )]);
16206        let error = reordered
16207            .side_effect(|| "new".to_string())
16208            .expect_err("command reordering must fail");
16209        assert!(matches!(
16210            error,
16211            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16212                if reason == "recorded_command_mismatch"
16213        ));
16214    }
16215
16216    #[test]
16217    fn version_markers_replay_across_upgrades_and_do_not_duplicate() {
16218        let ctx = workflow_context(Vec::new());
16219        assert_eq!(ctx.get_version("checkout-v2", 1, 2).expect("version"), 2);
16220        assert_eq!(ctx.get_version("checkout-v2", 1, 3).expect("cached"), 2);
16221        assert!(ctx.patched("new-search").expect("patch"));
16222        ctx.deprecate_patch("new-search").expect("deprecate patch");
16223        let commands = ctx.take_commands().expect("commands");
16224        assert_eq!(commands.len(), 2);
16225        assert_eq!(commands[0]["type"], "record_version_marker");
16226        assert_eq!(commands[0]["version"], 2);
16227        assert_eq!(commands[1]["change_id"], "new-search");
16228
16229        let replay = workflow_context(vec![history_event(
16230            "VersionMarkerRecorded",
16231            json!({
16232                "sequence": 1,
16233                "change_id": "checkout-v2",
16234                "version": 2,
16235                "min_supported": 1,
16236                "max_supported": 2,
16237            }),
16238        )]);
16239        assert_eq!(replay.get_version("checkout-v2", 1, 4).expect("upgrade"), 2);
16240        assert_eq!(replay.get_version("checkout-v2", 2, 5).expect("repeat"), 2);
16241        assert!(replay.take_commands().expect("commands").is_empty());
16242        replay.ensure_history_consumed().expect("history consumed");
16243    }
16244
16245    #[test]
16246    fn version_markers_reject_incompatible_or_malformed_history() {
16247        let incompatible = workflow_context(vec![history_event(
16248            "VersionMarkerRecorded",
16249            json!({
16250                "sequence": 1,
16251                "change_id": "checkout-v2",
16252                "version": 1,
16253                "min_supported": 1,
16254                "max_supported": 2,
16255            }),
16256        )]);
16257        let error = incompatible
16258            .get_version("checkout-v2", 2, 3)
16259            .expect_err("old version is unsupported");
16260        assert!(matches!(
16261            error,
16262            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16263                if reason == "version_marker_incompatible_range"
16264        ));
16265
16266        for (history, reason) in [
16267            (
16268                vec![history_event("SideEffectRecorded", json!({"sequence": 1}))],
16269                "side_effect_result_missing",
16270            ),
16271            (
16272                vec![history_event(
16273                    "SideEffectRecorded",
16274                    json!({
16275                        "sequence": 1,
16276                        "result": {"codec": "avro", "blob": "not-base64"},
16277                    }),
16278                )],
16279                "side_effect_payload_incompatible",
16280            ),
16281            (
16282                vec![history_event(
16283                    "SideEffectRecorded",
16284                    json!({"sequence": 1, "result": {"unwrapped": true}}),
16285                )],
16286                "side_effect_payload_malformed",
16287            ),
16288            (
16289                vec![history_event(
16290                    "VersionMarkerRecorded",
16291                    json!({
16292                        "sequence": 1,
16293                        "change_id": "change",
16294                        "version": 1,
16295                        "min_supported": 2,
16296                        "max_supported": 1,
16297                    }),
16298                )],
16299                "version_marker_history_range_invalid",
16300            ),
16301        ] {
16302            let error = WorkflowState::new(
16303                history,
16304                "rust-workers".to_string(),
16305                DEFAULT_CODEC.to_string(),
16306                None,
16307            )
16308            .expect_err("malformed history must fail");
16309            assert!(matches!(
16310                error,
16311                Error::NonDeterministicReplay(ReplayFailure { reason: actual, .. })
16312                    if actual == reason
16313            ));
16314        }
16315    }
16316
16317    #[test]
16318    fn typed_search_attributes_replay_value_and_type_identity_after_restart() {
16319        let history = vec![history_event(
16320            "SearchAttributesUpserted",
16321            json!({
16322                "sequence": 1,
16323                "attributes": {"customer_tier": "gold"},
16324                "attribute_types": {"customer_tier": "keyword"},
16325                "merged": {"customer_tier": "gold"}
16326            }),
16327        )];
16328
16329        let matching = workflow_context(history.clone());
16330        matching
16331            .upsert_search_attributes(
16332                SearchAttributeUpdate::new()
16333                    .keyword("customer_tier", "gold")
16334                    .expect("keyword update"),
16335            )
16336            .expect("matching typed update must replay");
16337        matching
16338            .ensure_history_consumed()
16339            .expect("history consumed");
16340
16341        let changed_type = workflow_context(history.clone());
16342        let error = changed_type
16343            .upsert_search_attributes(
16344                SearchAttributeUpdate::new()
16345                    .string("customer_tier", "gold")
16346                    .expect("string update"),
16347            )
16348            .expect_err("same JSON value with a different declaration must be nondeterministic");
16349        let Error::NonDeterministicReplay(failure) = error else {
16350            panic!("typed identity drift must be a replay failure");
16351        };
16352        assert_eq!(failure.reason, "search_attribute_type_mismatch");
16353        assert_eq!(failure.sequence, Some(1));
16354
16355        let changed_value = workflow_context(history);
16356        let error = changed_value
16357            .upsert_search_attributes(
16358                SearchAttributeUpdate::new()
16359                    .keyword("customer_tier", "platinum")
16360                    .expect("keyword update"),
16361            )
16362            .expect_err("changed values must be nondeterministic");
16363        let Error::NonDeterministicReplay(failure) = error else {
16364            panic!("value drift must be a replay failure");
16365        };
16366        assert_eq!(failure.reason, "search_attribute_value_mismatch");
16367    }
16368
16369    #[test]
16370    fn legacy_search_attribute_history_keeps_type_identity_unknown() {
16371        let history = vec![history_event(
16372            "SearchAttributesUpserted",
16373            json!({
16374                "sequence": 1,
16375                "attributes": {"customer_tier": "gold"},
16376                "merged": {"customer_tier": "gold"}
16377            }),
16378        )];
16379
16380        for update in [
16381            SearchAttributeUpdate::new()
16382                .keyword("customer_tier", "gold")
16383                .expect("keyword update"),
16384            SearchAttributeUpdate::new()
16385                .string("customer_tier", "gold")
16386                .expect("string update"),
16387        ] {
16388            let restarted = workflow_context(history.clone());
16389            restarted
16390                .upsert_search_attributes(update)
16391                .expect("legacy history constrains values but has unknown type identity");
16392            restarted
16393                .ensure_history_consumed()
16394                .expect("history consumed");
16395        }
16396    }
16397
16398    #[test]
16399    fn search_attribute_command_emits_canonical_types() {
16400        let ctx = workflow_context(Vec::new());
16401        ctx.upsert_search_attributes(
16402            SearchAttributeUpdate::new()
16403                .keyword("customer_tier", "gold")
16404                .expect("keyword update")
16405                .int("attempts", 3)
16406                .expect("int update")
16407                .delete("obsolete")
16408                .expect("delete update"),
16409        )
16410        .expect("valid search attributes");
16411
16412        assert_eq!(
16413            ctx.take_commands().expect("commands"),
16414            vec![json!({
16415                "type": "upsert_search_attributes",
16416                "attributes": {
16417                    "attempts": 3,
16418                    "customer_tier": "gold",
16419                    "obsolete": null
16420                },
16421                "attribute_types": {
16422                    "attempts": "int",
16423                    "customer_tier": "keyword"
16424                }
16425            })]
16426        );
16427    }
16428
16429    #[test]
16430    fn duplicate_side_effects_and_version_markers_are_rejected() {
16431        let duplicate_side_effect = WorkflowState::new(
16432            vec![
16433                history_event(
16434                    "SideEffectRecorded",
16435                    json!({"sequence": 1, "result": fixture_envelope(json!(1))}),
16436                ),
16437                history_event(
16438                    "SideEffectRecorded",
16439                    json!({"sequence": 1, "result": fixture_envelope(json!(2))}),
16440                ),
16441            ],
16442            "rust-workers".to_string(),
16443            DEFAULT_CODEC.to_string(),
16444            None,
16445        )
16446        .expect_err("duplicate side effect");
16447        assert!(matches!(
16448            duplicate_side_effect,
16449            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16450                if reason == "duplicate_side_effect_record"
16451        ));
16452
16453        let marker = |sequence| {
16454            history_event(
16455                "VersionMarkerRecorded",
16456                json!({
16457                    "sequence": sequence,
16458                    "change_id": "same-change",
16459                    "version": 1,
16460                    "min_supported": 1,
16461                    "max_supported": 1,
16462                }),
16463            )
16464        };
16465        let duplicate_marker = WorkflowState::new(
16466            vec![marker(1), marker(3)],
16467            "rust-workers".to_string(),
16468            DEFAULT_CODEC.to_string(),
16469            None,
16470        )
16471        .expect_err("duplicate marker");
16472        assert!(matches!(
16473            duplicate_marker,
16474            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16475                if reason == "duplicate_version_marker"
16476        ));
16477    }
16478
16479    #[test]
16480    fn workflow_stream_authoring_derives_identity_and_replay_skips_duplicate_append() {
16481        let mut state = WorkflowState::new(
16482            Vec::new(),
16483            "rust-workers".to_string(),
16484            DEFAULT_CODEC.to_string(),
16485            None,
16486        )
16487        .expect("workflow state");
16488        state.workflow_command_identity = "command-7".to_string();
16489        let context = WorkflowContext {
16490            state: Arc::new(Mutex::new(state)),
16491        };
16492        let item =
16493            WorkflowStreamAppendItem::from_reference("s3://bucket/item.avro").item_type("receipt");
16494
16495        context
16496            .append_workflow_stream("output", &[item], Some(10))
16497            .expect("append command");
16498        context
16499            .error_workflow_stream("output", "producer failed", None)
16500            .expect("error command");
16501        let commands = context.take_commands().expect("commands");
16502
16503        assert_eq!(commands[0]["type"], "record_side_effect");
16504        assert_eq!(
16505            commands[0]["workflow_stream"]["command_identity"],
16506            "command-7"
16507        );
16508        assert_eq!(commands[0]["workflow_stream"]["command_ordinal"], 0);
16509        assert_eq!(
16510            commands[0]["workflow_stream"]["items"][0]["idempotency_key"],
16511            "dw-stream:command-7:0:0"
16512        );
16513        assert_eq!(commands[1]["workflow_stream"]["operation"], "error");
16514
16515        let recorded = history_event(
16516            "SideEffectRecorded",
16517            json!({"sequence": 1, "result": fixture_envelope(Value::Null)}),
16518        );
16519        let mut replay_state = WorkflowState::new(
16520            vec![recorded],
16521            "rust-workers".to_string(),
16522            DEFAULT_CODEC.to_string(),
16523            None,
16524        )
16525        .expect("replay state");
16526        replay_state.workflow_command_identity = "command-7".to_string();
16527        let replay_context = WorkflowContext {
16528            state: Arc::new(Mutex::new(replay_state)),
16529        };
16530        replay_context
16531            .append_workflow_stream(
16532                "output",
16533                &[WorkflowStreamAppendItem::from_reference(
16534                    "s3://bucket/item.avro",
16535                )],
16536                Some(10),
16537            )
16538            .expect("replayed append");
16539        assert!(replay_context
16540            .take_commands()
16541            .expect("replayed commands")
16542            .is_empty());
16543    }
16544
16545    #[test]
16546    fn workflow_stream_authoring_requires_server_durable_command_identity() {
16547        let context = workflow_context(Vec::new());
16548        let error = context
16549            .append_workflow_stream(
16550                "output",
16551                &[WorkflowStreamAppendItem::from_reference(
16552                    "s3://bucket/item.avro",
16553                )],
16554                None,
16555            )
16556            .expect_err("stream append without durable command identity must fail closed");
16557
16558        assert!(matches!(error, Error::MissingWorkflowCommandIdentity));
16559        assert!(context.take_commands().expect("commands").is_empty());
16560    }
16561
16562    #[test]
16563    fn cold_worker_replay_does_not_repeat_committed_side_effects_or_markers() {
16564        fn worker(calls: Arc<AtomicUsize>) -> Worker {
16565            let client = Client::new("http://127.0.0.1:8080").expect("client");
16566            let mut worker = Worker::new(client, "rust-workers");
16567            worker.register_workflow("rust.side-effect-version", move |ctx, _input| {
16568                let calls = Arc::clone(&calls);
16569                async move {
16570                    let captured = ctx.side_effect(|| {
16571                        calls.fetch_add(1, Ordering::SeqCst);
16572                        "captured-once".to_string()
16573                    })?;
16574                    let version = ctx.get_version("cold-restart", 1, 2)?;
16575                    Ok(json!({"captured": captured, "version": version}))
16576                }
16577            });
16578            worker
16579        }
16580
16581        fn task(history_events: Vec<HistoryEvent>) -> WorkflowTask {
16582            WorkflowTask {
16583                task_id: "wft-side-effect-version".to_string(),
16584                workflow_command_id: None,
16585                workflow_id: Some("wf-side-effect-version".to_string()),
16586                run_id: Some("run-side-effect-version".to_string()),
16587                workflow_type: "rust.side-effect-version".to_string(),
16588                cancel_requested: false,
16589                payload_codec: DEFAULT_CODEC.to_string(),
16590                arguments: Some(
16591                    encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("arguments"),
16592                ),
16593                history_events,
16594                total_history_events: None,
16595                history_size_bytes: None,
16596                continue_as_new_recommended: None,
16597                history_budget_pressure: None,
16598                next_history_page_token: None,
16599                workflow_task_attempt: 1,
16600                workflow_signal_id: None,
16601                signal_name: None,
16602                signal_arguments: None,
16603                workflow_update_id: None,
16604                update_name: None,
16605                lease_owner: Some("rust-worker".to_string()),
16606            }
16607        }
16608
16609        let calls = Arc::new(AtomicUsize::new(0));
16610        let initial = worker(Arc::clone(&calls))
16611            .execute_workflow_task(task(Vec::new()))
16612            .expect("initial execution");
16613        assert_eq!(
16614            initial
16615                .iter()
16616                .map(|command| &command["type"])
16617                .collect::<Vec<_>>(),
16618            vec![
16619                "record_side_effect",
16620                "record_version_marker",
16621                "complete_workflow"
16622            ]
16623        );
16624        assert_eq!(calls.load(Ordering::SeqCst), 1);
16625
16626        let restarted = worker(Arc::clone(&calls));
16627        let replayed = restarted
16628            .execute_workflow_task(task(vec![
16629                history_event(
16630                    "SideEffectRecorded",
16631                    json!({"sequence": 1, "result": initial[0]["result"].clone()}),
16632                ),
16633                history_event(
16634                    "VersionMarkerRecorded",
16635                    json!({
16636                        "sequence": 2,
16637                        "change_id": "cold-restart",
16638                        "version": 2,
16639                        "min_supported": 1,
16640                        "max_supported": 2,
16641                    }),
16642                ),
16643            ]))
16644            .expect("cold replay");
16645        assert_eq!(replayed.len(), 1);
16646        assert_eq!(replayed[0]["type"], "complete_workflow");
16647        assert_eq!(calls.load(Ordering::SeqCst), 1);
16648    }
16649
16650    #[test]
16651    fn side_effect_replay_rejects_changed_rust_value_type() {
16652        let result = encode_value_envelope(&json!({"value": 42}), DEFAULT_CODEC).expect("result");
16653        let ctx = workflow_context(vec![history_event(
16654            "SideEffectRecorded",
16655            json!({"sequence": 1, "result": result}),
16656        )]);
16657        let error = ctx
16658            .side_effect::<Vec<String>, _>(|| panic!("callback must not run"))
16659            .expect_err("changed type must fail replay");
16660        assert!(matches!(
16661            error,
16662            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16663                if reason == "side_effect_type_mismatch"
16664        ));
16665    }
16666
16667    fn completed_retry_activity_history() -> Vec<HistoryEvent> {
16668        vec![
16669            history_event(
16670                "ActivityScheduled",
16671                json!({
16672                    "sequence": 1,
16673                    "activity_type": "flaky",
16674                    "activity_execution_id": "act-1",
16675                    "activity": {
16676                        "id": "act-1",
16677                        "sequence": 1,
16678                        "type": "flaky",
16679                        "queue": "critical-activities",
16680                        "execution_mode": null,
16681                        "retry_policy": {
16682                            "snapshot_version": 1,
16683                            "max_attempts": 3,
16684                            "backoff_seconds": [2, 4],
16685                            "start_to_close_timeout": 30,
16686                            "schedule_to_start_timeout": 5,
16687                            "schedule_to_close_timeout": 90,
16688                            "heartbeat_timeout": 10,
16689                            "non_retryable_error_types": ["PermanentError"]
16690                        }
16691                    }
16692                }),
16693            ),
16694            history_event(
16695                "ActivityStarted",
16696                json!({
16697                    "sequence": 1,
16698                    "activity_type": "flaky",
16699                    "activity_execution_id": "act-1",
16700                    "activity_attempt_id": "attempt-1",
16701                    "attempt_number": 1
16702                }),
16703            ),
16704            history_event(
16705                "ActivityRetryScheduled",
16706                json!({
16707                    "sequence": 1,
16708                    "activity_type": "flaky",
16709                    "activity_execution_id": "act-1",
16710                    "activity_attempt_id": "attempt-1",
16711                    "attempt_number": 1,
16712                    "retry_after_attempt": 1,
16713                    "retry_backoff_seconds": 2,
16714                    "failure_category": "activity",
16715                    "exception_type": "TransientError"
16716                }),
16717            ),
16718            history_event(
16719                "ActivityStarted",
16720                json!({
16721                    "sequence": 1,
16722                    "activity_type": "flaky",
16723                    "activity_execution_id": "act-1",
16724                    "activity_attempt_id": "attempt-2",
16725                    "attempt_number": 2
16726                }),
16727            ),
16728            history_event(
16729                "ActivityCompleted",
16730                json!({
16731                    "sequence": 1,
16732                    "activity_type": "flaky",
16733                    "activity_execution_id": "act-1",
16734                    "activity_attempt_id": "attempt-2",
16735                    "attempt_number": 2,
16736                    "payload_codec": DEFAULT_CODEC,
16737                    "result": fixture_envelope(json!({"status":"recovered"}))
16738                }),
16739            ),
16740        ]
16741    }
16742
16743    fn retry_activity_options() -> ActivityOptions {
16744        ActivityOptions::new()
16745            .task_queue("critical-activities")
16746            .retry_policy(
16747                ActivityRetryPolicy::new(3)
16748                    .backoff_intervals([Duration::from_secs(2), Duration::from_secs(4)])
16749                    .non_retryable_error_type("PermanentError"),
16750            )
16751            .start_to_close_timeout(Duration::from_secs(30))
16752            .schedule_to_start_timeout(Duration::from_secs(5))
16753            .schedule_to_close_timeout(Duration::from_secs(90))
16754            .heartbeat_timeout(Duration::from_secs(10))
16755    }
16756
16757    #[test]
16758    fn fixed_avro_value_round_trips_json_values() {
16759        let value = json!({"greeting": "hello", "count": 3, "ok": true});
16760        let envelope = PayloadEnvelope::avro(&value).expect("encode");
16761        assert_eq!(envelope.codec, DEFAULT_CODEC);
16762        assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
16763    }
16764
16765    #[tokio::test]
16766    async fn typed_handler_adapters_round_trip_serde_contracts_on_the_fixed_wire() {
16767        let client = Client::new("http://127.0.0.1:8080").expect("client");
16768        let mut worker = Worker::new(client, "rust-workers");
16769        worker.register_typed_workflow(
16770            "typed.contract.workflow",
16771            |_ctx, input: TypedContract| async move { Ok(input) },
16772        );
16773        worker.register_typed_activity(
16774            "typed.contract.activity",
16775            |_ctx, input: TypedContract| async move { Ok(input) },
16776        );
16777
16778        let expected = typed_contract();
16779        let arguments = AvroValue::Array(vec![
16780            AvroValue::from_serialize(&expected).expect("typed request")
16781        ]);
16782        let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("arguments");
16783        let mut workflow = workflow_task("typed.contract.workflow", Vec::new(), DEFAULT_CODEC);
16784        workflow.arguments = Some(envelope.clone());
16785        let commands = worker
16786            .execute_workflow_task(workflow)
16787            .expect("typed workflow task");
16788        let workflow_result: TypedContract =
16789            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
16790                .expect("workflow result envelope")
16791                .deserialize()
16792                .expect("workflow result type");
16793        assert_eq!(workflow_result, expected);
16794
16795        let activity = ActivityTask {
16796            task_id: "typed-contract-activity".to_string(),
16797            activity_attempt_id: Some("typed-contract-attempt".to_string()),
16798            attempt_id: None,
16799            activity_type: "typed.contract.activity".to_string(),
16800            payload_codec: DEFAULT_CODEC.to_string(),
16801            arguments: Some(envelope),
16802            attempt_number: 1,
16803            lease_owner: Some("rust-worker".to_string()),
16804        };
16805        let activity_result: TypedContract = worker
16806            .execute_activity_task(activity)
16807            .await
16808            .expect("typed activity task")
16809            .deserialize()
16810            .expect("activity result type");
16811        assert_eq!(activity_result, expected);
16812    }
16813
16814    #[tokio::test]
16815    async fn typed_handler_errors_include_handler_name_direction_and_rust_type() {
16816        let client = Client::new("http://127.0.0.1:8080").expect("client");
16817        let mut worker = Worker::new(client, "rust-workers");
16818        worker.register_typed_workflow(
16819            "typed.shape.workflow",
16820            |_ctx, input: TypedContract| async move { Ok(input) },
16821        );
16822        worker.register_typed_activity("typed.unsupported.activity", |_ctx, (): ()| async move {
16823            Ok(f64::NAN)
16824        });
16825
16826        let mut workflow = workflow_task("typed.shape.workflow", Vec::new(), DEFAULT_CODEC);
16827        workflow.arguments = Some(
16828            encode_typed_envelope(
16829                &AvroValue::Array(vec![
16830                    AvroValue::String("first".to_string()),
16831                    AvroValue::String("second".to_string()),
16832                ]),
16833                DEFAULT_CODEC,
16834            )
16835            .expect("malformed typed arguments"),
16836        );
16837        let commands = worker
16838            .execute_workflow_task(workflow)
16839            .expect("shape mismatch becomes a workflow failure");
16840        let message = commands[0]["message"].as_str().expect("failure message");
16841        assert!(message.contains("workflow handler \"typed.shape.workflow\" input type"));
16842        assert!(message.contains(type_name::<TypedContract>()));
16843        assert!(message.contains("task carried 2 arguments"));
16844
16845        let activity = ActivityTask {
16846            task_id: "typed-unsupported-activity".to_string(),
16847            activity_attempt_id: Some("typed-unsupported-attempt".to_string()),
16848            attempt_id: None,
16849            activity_type: "typed.unsupported.activity".to_string(),
16850            payload_codec: DEFAULT_CODEC.to_string(),
16851            arguments: Some(
16852                encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
16853                    .expect("unit arguments"),
16854            ),
16855            attempt_number: 1,
16856            lease_owner: Some("rust-worker".to_string()),
16857        };
16858        let Error::HandlerType {
16859            handler_kind,
16860            handler_name,
16861            value_kind,
16862            rust_type,
16863            message,
16864        } = worker
16865            .execute_activity_task(activity)
16866            .await
16867            .expect_err("non-finite handler output must fail")
16868        else {
16869            panic!("expected contextual handler type failure");
16870        };
16871        assert_eq!(handler_kind, HandlerKind::Activity);
16872        assert_eq!(handler_name, "typed.unsupported.activity");
16873        assert_eq!(value_kind, HandlerValueKind::Result);
16874        assert_eq!(rust_type, type_name::<f64>());
16875        assert!(message.contains("non_finite_float"));
16876    }
16877
16878    #[tokio::test]
16879    async fn typed_replayed_workflow_decodes_input_and_activity_result_losslessly() {
16880        #[derive(Clone, Default)]
16881        struct State {
16882            observed: Option<TypedContract>,
16883        }
16884
16885        let client = Client::new("http://127.0.0.1:8080").expect("client");
16886        let mut worker = Worker::new(client, "rust-workers");
16887        worker.register_typed_replayed_workflow(
16888            "typed.contract.replayed",
16889            State::default,
16890            |ctx, input: TypedContract, state| async move {
16891                let result: TypedContract =
16892                    ctx.activity_typed("typed.contract.activity", input).await?;
16893                state.update(|current| current.observed = Some(result.clone()))?;
16894                Ok(result)
16895            },
16896        );
16897        worker.register_replayed_query::<State, _, _>(
16898            "typed.contract.replayed",
16899            "observed",
16900            |_ctx, state, _args| async move {
16901                Ok(json!(state.observed.as_ref().map(|value| value.signed)))
16902            },
16903        );
16904
16905        let expected = typed_contract();
16906        let typed_value = AvroValue::from_serialize(&expected).expect("typed value");
16907        let workflow_arguments =
16908            encode_typed_envelope(&AvroValue::Array(vec![typed_value.clone()]), DEFAULT_CODEC)
16909                .expect("workflow arguments");
16910        let result = encode_typed_envelope(&typed_value, DEFAULT_CODEC).expect("activity result");
16911        let task = QueryTask {
16912            query_task_id: "typed-replay-query".to_string(),
16913            query_task_attempt: 1,
16914            lease_owner: Some("rust-worker".to_string()),
16915            workflow_id: Some("typed-replay".to_string()),
16916            run_id: Some("typed-replay-run".to_string()),
16917            workflow_type: "typed.contract.replayed".to_string(),
16918            query_name: "observed".to_string(),
16919            payload_codec: DEFAULT_CODEC.to_string(),
16920            workflow_arguments: Some(workflow_arguments),
16921            query_arguments: Some(
16922                encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
16923                    .expect("query arguments"),
16924            ),
16925            history_events: vec![
16926                history_event(
16927                    "ActivityScheduled",
16928                    json!({
16929                        "sequence": 1,
16930                        "activity_type": "typed.contract.activity"
16931                    }),
16932                ),
16933                history_event(
16934                    "ActivityCompleted",
16935                    json!({
16936                        "sequence": 1,
16937                        "activity_type": "typed.contract.activity",
16938                        "payload_codec": DEFAULT_CODEC,
16939                        "result": result
16940                    }),
16941                ),
16942            ],
16943            history_export: None,
16944            run_status: Some("completed".to_string()),
16945        };
16946
16947        assert_eq!(
16948            worker
16949                .execute_query_task(task)
16950                .await
16951                .expect("typed replay query")
16952                .deserialize::<i64>()
16953                .expect("query result"),
16954            expected.signed
16955        );
16956    }
16957
16958    #[tokio::test]
16959    async fn typed_worker_surfaces_preserve_bytes_and_map_list_identity() {
16960        let client = Client::new("http://127.0.0.1:8080").expect("client");
16961        let mut worker = Worker::new(client, "rust-workers");
16962        worker.register_workflow_avro_value("typed.echo", |_ctx, input| async move { Ok(input) });
16963        worker
16964            .register_activity_avro_value("typed.activity", |_ctx, input| async move { Ok(input) });
16965        worker.register_query_avro_value("typed.echo", "inspect", |_ctx, input| async move {
16966            Ok(input)
16967        });
16968        worker.register_update_avro_value("typed.echo", "replace", |_ctx, input| async move {
16969            Ok(input)
16970        });
16971        worker.register_workflow_avro_value("typed.signal", |ctx, _input| async move {
16972            Ok(AvroValue::Array(
16973                ctx.wait_signal_avro_value("changed").await?,
16974            ))
16975        });
16976
16977        let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
16978        let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed envelope");
16979
16980        let mut workflow = workflow_task("typed.echo", Vec::new(), DEFAULT_CODEC);
16981        workflow.arguments = Some(envelope.clone());
16982        let commands = worker
16983            .execute_workflow_task(workflow)
16984            .expect("typed workflow task");
16985        assert_eq!(commands[0]["type"], "complete_workflow");
16986        assert_eq!(
16987            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
16988                .expect("typed workflow result"),
16989            arguments
16990        );
16991
16992        let activity = ActivityTask {
16993            task_id: "activity-typed".to_string(),
16994            activity_attempt_id: Some("attempt-typed".to_string()),
16995            attempt_id: None,
16996            activity_type: "typed.activity".to_string(),
16997            payload_codec: DEFAULT_CODEC.to_string(),
16998            arguments: Some(envelope.clone()),
16999            attempt_number: 1,
17000            lease_owner: Some("rust-worker".to_string()),
17001        };
17002        assert_eq!(
17003            worker
17004                .execute_activity_task(activity)
17005                .await
17006                .expect("typed activity result"),
17007            arguments
17008        );
17009
17010        let query = QueryTask {
17011            query_task_id: "query-typed".to_string(),
17012            query_task_attempt: 1,
17013            lease_owner: Some("rust-worker".to_string()),
17014            workflow_id: Some("typed-1".to_string()),
17015            run_id: Some("run-typed".to_string()),
17016            workflow_type: "typed.echo".to_string(),
17017            query_name: "inspect".to_string(),
17018            payload_codec: DEFAULT_CODEC.to_string(),
17019            workflow_arguments: Some(
17020                encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
17021                    .expect("workflow input"),
17022            ),
17023            query_arguments: Some(envelope.clone()),
17024            history_events: Vec::new(),
17025            history_export: None,
17026            run_status: Some("running".to_string()),
17027        };
17028        assert_eq!(
17029            worker
17030                .execute_query_task(query)
17031                .await
17032                .expect("typed query result"),
17033            arguments
17034        );
17035
17036        let mut update = workflow_task(
17037            "typed.echo",
17038            vec![history_event(
17039                "UpdateAccepted",
17040                json!({
17041                    "update_id": "update-typed",
17042                    "update_name": "replace",
17043                    "arguments": envelope.clone(),
17044                }),
17045            )],
17046            DEFAULT_CODEC,
17047        );
17048        update.workflow_update_id = Some("update-typed".to_string());
17049        update.update_name = Some("replace".to_string());
17050        let commands = worker
17051            .execute_workflow_task(update)
17052            .expect("typed update task");
17053        assert_eq!(commands[0]["type"], "complete_update");
17054        assert_eq!(
17055            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17056                .expect("typed update result"),
17057            arguments
17058        );
17059
17060        let mut signal = workflow_task(
17061            "typed.signal",
17062            vec![history_event(
17063                "SignalReceived",
17064                json!({
17065                    "signal_id": "signal-typed",
17066                    "signal_name": "changed",
17067                    "arguments": envelope.clone(),
17068                }),
17069            )],
17070            DEFAULT_CODEC,
17071        );
17072        signal.workflow_signal_id = Some("signal-typed".to_string());
17073        signal.signal_name = Some("changed".to_string());
17074        signal.signal_arguments = Some(envelope);
17075        let commands = worker
17076            .execute_workflow_task(signal)
17077            .expect("typed signal resume");
17078        assert_eq!(
17079            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17080                .expect("typed signal result"),
17081            arguments
17082        );
17083    }
17084
17085    #[tokio::test]
17086    async fn typed_helpers_never_parse_json_inspection_projection() {
17087        let collision_values = projection_collision_probe();
17088        let expected = AvroValue::Array(collision_values.clone());
17089        let envelope = encode_typed_envelope(&expected, DEFAULT_CODEC).expect("collision envelope");
17090
17091        let activity_context = workflow_context_with_codec(
17092            vec![history_event(
17093                "ActivityCompleted",
17094                json!({
17095                    "sequence": 1,
17096                    "activity_type": "collision.activity",
17097                    "payload_codec": DEFAULT_CODEC,
17098                    "result": envelope.clone(),
17099                }),
17100            )],
17101            DEFAULT_CODEC,
17102        );
17103        assert_eq!(
17104            activity_context
17105                .activity_avro_value("collision.activity", AvroValue::Array(Vec::new()))
17106                .await
17107                .expect("typed activity collision result"),
17108            expected
17109        );
17110
17111        let signal_context = workflow_context_with_codec(
17112            vec![
17113                history_event(
17114                    "SignalWaitOpened",
17115                    json!({"sequence": 1, "signal_name": "collision"}),
17116                ),
17117                history_event(
17118                    "SignalApplied",
17119                    json!({
17120                        "sequence": 1,
17121                        "signal_name": "collision",
17122                        "payload_codec": DEFAULT_CODEC,
17123                        "value": envelope.clone(),
17124                    }),
17125                ),
17126            ],
17127            DEFAULT_CODEC,
17128        );
17129        assert_eq!(
17130            signal_context
17131                .wait_signal_avro_value("collision")
17132                .await
17133                .expect("typed signal collision arguments"),
17134            collision_values
17135        );
17136
17137        let child_context = workflow_context_with_codec(
17138            vec![
17139                history_event(
17140                    "ChildWorkflowScheduled",
17141                    json!({
17142                        "sequence": 1,
17143                        "child_workflow_instance_id": "collision-child",
17144                        "child_workflow_run_id": "collision-run",
17145                        "child_workflow_type": "collision.child",
17146                    }),
17147                ),
17148                history_event(
17149                    "ChildRunCompleted",
17150                    json!({
17151                        "sequence": 1,
17152                        "child_workflow_instance_id": "collision-child",
17153                        "child_workflow_run_id": "collision-run",
17154                        "child_workflow_type": "collision.child",
17155                        "payload_codec": DEFAULT_CODEC,
17156                        "result": envelope,
17157                    }),
17158                ),
17159            ],
17160            DEFAULT_CODEC,
17161        );
17162        let child = child_context
17163            .start_child_workflow_avro_value(
17164                "collision.child",
17165                ChildWorkflowOptions::new("collision-workers"),
17166                AvroValue::Array(Vec::new()),
17167            )
17168            .await
17169            .expect("typed child collision result");
17170        assert_eq!(child.result, expected);
17171    }
17172
17173    #[tokio::test]
17174    async fn replayed_typed_query_keeps_lossless_workflow_and_query_inputs() {
17175        let client = Client::new("http://127.0.0.1:8080").expect("client");
17176        let mut worker = Worker::new(client, "rust-workers");
17177        worker.register_replayed_workflow_avro_value(
17178            "typed.replayed",
17179            || (),
17180            |_ctx, input, _state| async move { Ok(input) },
17181        );
17182        worker.register_replayed_query_avro_value::<(), _, _>(
17183            "typed.replayed",
17184            "inspect",
17185            |ctx, _state, args| async move {
17186                let mut signals = ctx.signals_avro_value("collision");
17187                let signal = signals
17188                    .pop()
17189                    .map(AvroValue::Array)
17190                    .unwrap_or_else(|| AvroValue::Array(Vec::new()));
17191                Ok(AvroValue::Array(vec![
17192                    ctx.workflow_input_avro_value().clone(),
17193                    signal,
17194                    args,
17195                ]))
17196            },
17197        );
17198        let arguments = AvroValue::Array(projection_collision_probe());
17199        let signal_arguments =
17200            encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed query signal arguments");
17201        let task = QueryTask {
17202            query_task_id: "query-typed-replay".to_string(),
17203            query_task_attempt: 1,
17204            lease_owner: Some("rust-worker".to_string()),
17205            workflow_id: Some("typed-replay".to_string()),
17206            run_id: Some("run-typed-replay".to_string()),
17207            workflow_type: "typed.replayed".to_string(),
17208            query_name: "inspect".to_string(),
17209            payload_codec: DEFAULT_CODEC.to_string(),
17210            workflow_arguments: Some(
17211                encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("workflow arguments"),
17212            ),
17213            query_arguments: Some(
17214                encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("query arguments"),
17215            ),
17216            history_events: vec![history_event(
17217                "SignalReceived",
17218                json!({
17219                    "signal_id": "collision-signal",
17220                    "signal_name": "collision",
17221                    "workflow_sequence": 1,
17222                    "payload_codec": DEFAULT_CODEC,
17223                    "arguments": signal_arguments,
17224                }),
17225            )],
17226            history_export: None,
17227            run_status: Some("completed".to_string()),
17228        };
17229
17230        assert_eq!(
17231            worker
17232                .execute_query_task(task)
17233                .await
17234                .expect("typed replay query"),
17235            AvroValue::Array(vec![arguments.clone(), arguments.clone(), arguments])
17236        );
17237    }
17238
17239    #[test]
17240    fn public_avro_adapter_rejects_non_string_map_keys_before_json_conversion() {
17241        let value = BTreeMap::from([(1_i32, "integer key")]);
17242        let error = PayloadEnvelope::avro(&value)
17243            .expect_err("integer map keys must fail")
17244            .to_string();
17245
17246        assert!(error.contains("invalid_map_key"));
17247    }
17248
17249    #[test]
17250    fn json_tagged_payload_fails_closed_with_actionable_diagnostic() {
17251        let envelope = PayloadEnvelope {
17252            codec: "json".to_string(),
17253            blob: r#"{"greeting":"hello"}"#.to_string(),
17254        };
17255
17256        let error = decode_payload::<Value>(&envelope).expect_err("JSON payload must fail");
17257        let diagnostic = error.to_string();
17258        assert!(diagnostic.contains("unsupported_payload_codec"));
17259        assert!(diagnostic.contains("codec=\"avro\""));
17260        assert!(diagnostic.contains("HTTP document transport"));
17261    }
17262
17263    #[test]
17264    fn untagged_json_payload_value_fails_closed() {
17265        let error = decode_wire_value(&json!({"stale": true}), DEFAULT_CODEC)
17266            .expect_err("untagged JSON payload values must fail");
17267        let diagnostic = error.to_string();
17268        assert!(diagnostic.contains("unsupported_payload_codec"));
17269        assert!(diagnostic.contains("untagged durable payload"));
17270        assert!(diagnostic.contains("HTTP document transport"));
17271    }
17272
17273    #[test]
17274    fn prerelease_avro_payload_without_single_object_frame_is_rejected() {
17275        let envelope = PayloadEnvelope {
17276            codec: DEFAULT_CODEC.to_string(),
17277            blob: BASE64.encode([0x01]),
17278        };
17279
17280        let error = decode_payload::<Value>(&envelope).expect_err("prerelease payload must fail");
17281        assert!(error.to_string().contains("invalid_payload_framing"));
17282    }
17283
17284    #[tokio::test]
17285    async fn workflow_completion_rejects_invalid_payload_slots_without_transport() {
17286        let server = MockWorkerServer::start();
17287        let client = Client::builder(server.base_url())
17288            .timeout(Duration::from_secs(2))
17289            .build()
17290            .expect("client");
17291        let invalid_commands = [
17292            json!({
17293                "type": "complete_workflow",
17294                "result": {"codec": "json", "blob": null}
17295            }),
17296            json!({
17297                "type": "schedule_activity",
17298                "arguments": {"codec": "yaml", "blob": "ignored"}
17299            }),
17300            json!({
17301                "type": "start_child_workflow",
17302                "arguments": {"codec": DEFAULT_CODEC, "blob": null}
17303            }),
17304            json!({"type": "continue_as_new", "arguments": []}),
17305            json!({"type": "complete_update"}),
17306            json!({"type": "record_side_effect", "result": null}),
17307            json!({
17308                "type": "start_service_operation",
17309                "payload_codec": DEFAULT_CODEC,
17310                "request_payload": "raw-avro-bytes"
17311            }),
17312        ];
17313
17314        for command in invalid_commands {
17315            let error = client
17316                .complete_workflow_task("invalid-codec", "rust-worker", 1, vec![command])
17317                .await
17318                .expect_err("invalid durable payload must fail locally");
17319            let diagnostic = error.to_string();
17320            assert!(
17321                diagnostic.contains("unsupported_payload_codec")
17322                    || diagnostic.contains("invalid_payload_envelope")
17323                    || diagnostic.contains("untagged durable payload"),
17324                "unexpected validation diagnostic: {diagnostic}"
17325            );
17326        }
17327
17328        assert_eq!(
17329            server.request_count("/api/worker/workflow-tasks/invalid-codec/complete"),
17330            0,
17331            "invalid command payloads must not reach HTTP transport"
17332        );
17333    }
17334
17335    #[test]
17336    fn workflow_completion_validates_only_protocol_owned_payload_slots() {
17337        let envelope = fixture_envelope(json!({"codec": "customer-value"}));
17338        let commands = [
17339            json!({"type": "complete_workflow", "result": envelope.clone()}),
17340            json!({"type": "schedule_activity", "arguments": envelope.clone()}),
17341            json!({"type": "start_child_workflow", "arguments": envelope.clone()}),
17342            json!({"type": "continue_as_new", "arguments": envelope.clone()}),
17343            json!({"type": "complete_update", "result": envelope.clone()}),
17344            json!({"type": "record_side_effect", "result": envelope.clone()}),
17345            json!({
17346                "type": "start_service_operation",
17347                "payload_codec": DEFAULT_CODEC,
17348                "request_payload": envelope.clone()
17349            }),
17350            json!({
17351                "type": "complete_workflow",
17352                "result": envelope,
17353                "metadata": {
17354                    "codec": "json",
17355                    "payload_codec": "customer-codec",
17356                    "result": {"codec": "yaml", "blob": null}
17357                }
17358            }),
17359        ];
17360
17361        validate_workflow_task_commands(&commands)
17362            .expect("customer metadata must not become a protocol codec declaration");
17363    }
17364
17365    #[test]
17366    fn valid_avro_tasks_normalize_absent_and_null_arguments_to_empty_lists() {
17367        assert_eq!(
17368            decode_task_avro_arguments(None, DEFAULT_CODEC).expect("absent arguments"),
17369            AvroValue::Array(Vec::new())
17370        );
17371        assert_eq!(
17372            decode_task_avro_arguments(Some(&Value::Null), DEFAULT_CODEC).expect("null arguments"),
17373            AvroValue::Array(Vec::new())
17374        );
17375
17376        let mut signal = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
17377        signal.signal_name = Some("empty-signal".to_string());
17378        signal.signal_arguments = None;
17379        let decoded = decode_resume_signal(&signal)
17380            .expect("valid Avro signal")
17381            .expect("named signal resumes the workflow");
17382        assert!(decoded.arguments.is_empty());
17383    }
17384
17385    #[tokio::test]
17386    async fn malformed_task_level_codecs_become_pre_handler_failures() {
17387        let client = Client::new("http://127.0.0.1:8080").expect("client");
17388        let mut worker = Worker::new(client, "rust-workers");
17389        let handler_calls = Arc::new(AtomicUsize::new(0));
17390
17391        let calls = Arc::clone(&handler_calls);
17392        worker.register_workflow("codec.workflow", move |_ctx, _args| {
17393            calls.fetch_add(1, Ordering::SeqCst);
17394            async move { Ok(Value::Null) }
17395        });
17396        let calls = Arc::clone(&handler_calls);
17397        worker.register_activity("codec.activity", move |_ctx, _args| {
17398            calls.fetch_add(1, Ordering::SeqCst);
17399            async move { Ok(Value::Null) }
17400        });
17401        let calls = Arc::clone(&handler_calls);
17402        worker.register_query("codec.workflow", "known", move |_ctx, _args| {
17403            calls.fetch_add(1, Ordering::SeqCst);
17404            async move { Ok(Value::Null) }
17405        });
17406
17407        let mut failures = Vec::new();
17408        for codec_case in [
17409            InvalidTaskPayloadCodec::Missing,
17410            InvalidTaskPayloadCodec::Null,
17411            InvalidTaskPayloadCodec::NonString,
17412        ] {
17413            let mut workflow = json!({
17414                "task_id": format!("workflow-{}", codec_case.label()),
17415                "workflow_type": "codec.workflow"
17416            });
17417            codec_case.apply(&mut workflow);
17418            match serde_json::from_value::<WorkflowTask>(workflow) {
17419                Ok(task) => match worker.execute_workflow_task(task) {
17420                    Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
17421                    outcome => failures.push(format!(
17422                        "workflow {} codec returned {outcome:?}",
17423                        codec_case.label()
17424                    )),
17425                },
17426                Err(error) => failures.push(format!(
17427                    "workflow {} codec failed transport deserialization: {error}",
17428                    codec_case.label()
17429                )),
17430            }
17431
17432            let mut activity = json!({
17433                "task_id": format!("activity-{}", codec_case.label()),
17434                "activity_attempt_id": format!("attempt-{}", codec_case.label()),
17435                "activity_type": "codec.activity",
17436                "attempt_number": 1
17437            });
17438            codec_case.apply(&mut activity);
17439            match serde_json::from_value::<ActivityTask>(activity) {
17440                Ok(task) => match worker.execute_activity_task(task).await {
17441                    Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
17442                    outcome => failures.push(format!(
17443                        "activity {} codec returned {outcome:?}",
17444                        codec_case.label()
17445                    )),
17446                },
17447                Err(error) => failures.push(format!(
17448                    "activity {} codec failed transport deserialization: {error}",
17449                    codec_case.label()
17450                )),
17451            }
17452
17453            let mut query = json!({
17454                "query_task_id": format!("query-{}", codec_case.label()),
17455                "workflow_type": "codec.workflow",
17456                "query_name": "known"
17457            });
17458            codec_case.apply(&mut query);
17459            match serde_json::from_value::<QueryTask>(query) {
17460                Ok(task) => match worker.execute_query_task(task).await {
17461                    Err(failure) if failure.message.contains("unsupported_payload_codec") => {}
17462                    outcome => failures.push(format!(
17463                        "query {} codec returned {outcome:?}",
17464                        codec_case.label()
17465                    )),
17466                },
17467                Err(error) => failures.push(format!(
17468                    "query {} codec failed transport deserialization: {error}",
17469                    codec_case.label()
17470                )),
17471            }
17472        }
17473
17474        assert!(failures.is_empty(), "{}", failures.join("\n"));
17475        assert_eq!(
17476            handler_calls.load(Ordering::SeqCst),
17477            0,
17478            "invalid task codecs must not invoke a handler"
17479        );
17480    }
17481
17482    #[tokio::test]
17483    async fn polled_malformed_task_codecs_are_settled_without_handler_execution() {
17484        for codec_case in [
17485            InvalidTaskPayloadCodec::Missing,
17486            InvalidTaskPayloadCodec::Null,
17487            InvalidTaskPayloadCodec::NonString,
17488        ] {
17489            let server = MockWorkerServer::invalid_task_payload_codec(codec_case);
17490            let client = Client::builder(server.base_url())
17491                .timeout(Duration::from_secs(2))
17492                .build()
17493                .expect("client");
17494            let mut worker = Worker::new(client, "rust-workers")
17495                .worker_id("codec-worker")
17496                .poll_timeout(Duration::from_millis(10));
17497            let handler_calls = Arc::new(AtomicUsize::new(0));
17498
17499            let calls = Arc::clone(&handler_calls);
17500            worker.register_workflow("codec.workflow", move |_ctx, _args| {
17501                calls.fetch_add(1, Ordering::SeqCst);
17502                async move { Ok(Value::Null) }
17503            });
17504            let calls = Arc::clone(&handler_calls);
17505            worker.register_activity("codec.activity", move |_ctx, _args| {
17506                calls.fetch_add(1, Ordering::SeqCst);
17507                async move { Ok(Value::Null) }
17508            });
17509            let calls = Arc::clone(&handler_calls);
17510            worker.register_query("codec.workflow", "known", move |_ctx, _args| {
17511                calls.fetch_add(1, Ordering::SeqCst);
17512                async move { Ok(Value::Null) }
17513            });
17514
17515            assert_eq!(
17516                worker.run_once().await.expect("invalid tasks are settled"),
17517                3,
17518                "all {} codec tasks must be handled",
17519                codec_case.label()
17520            );
17521            assert_eq!(
17522                handler_calls.load(Ordering::SeqCst),
17523                0,
17524                "{} task codecs must fail before every handler",
17525                codec_case.label()
17526            );
17527
17528            for path in [
17529                "/api/worker/workflow-tasks/codec-workflow/fail",
17530                "/api/worker/activity-tasks/codec-activity/fail",
17531                "/api/worker/query-tasks/codec-query/fail",
17532            ] {
17533                let body = server.request_body(path);
17534                assert!(
17535                    body["failure"]["message"]
17536                        .as_str()
17537                        .is_some_and(|message| message.contains("unsupported_payload_codec")),
17538                    "{path} must receive the stable codec diagnostic for the {} case: {body}",
17539                    codec_case.label()
17540                );
17541            }
17542            assert_eq!(
17543                server.request_body("/api/worker/query-tasks/codec-query/fail")["failure"]
17544                    ["reason"],
17545                "query_payload_decode_failed"
17546            );
17547            for path in [
17548                "/api/worker/workflow-tasks/codec-workflow/complete",
17549                "/api/worker/activity-tasks/codec-activity/complete",
17550                "/api/worker/query-tasks/codec-query/complete",
17551            ] {
17552                assert_eq!(
17553                    server.request_count(path),
17554                    0,
17555                    "invalid {} codec task reached {path}",
17556                    codec_case.label()
17557                );
17558            }
17559        }
17560    }
17561
17562    #[tokio::test]
17563    async fn invalid_inbound_codecs_precede_handlers_and_unrelated_outcomes() {
17564        let client = Client::new("http://127.0.0.1:8080").expect("client");
17565        let mut worker = Worker::new(client, "rust-workers");
17566        let handler_calls = Arc::new(AtomicUsize::new(0));
17567
17568        let calls = Arc::clone(&handler_calls);
17569        worker.register_workflow("codec.workflow", move |_ctx, _args| {
17570            calls.fetch_add(1, Ordering::SeqCst);
17571            async move { Ok(Value::Null) }
17572        });
17573        let calls = Arc::clone(&handler_calls);
17574        worker.register_activity("codec.activity", move |_ctx, _args| {
17575            calls.fetch_add(1, Ordering::SeqCst);
17576            async move { Ok(Value::Null) }
17577        });
17578        let calls = Arc::clone(&handler_calls);
17579        worker.register_update("codec.workflow", "known", move |_ctx, _args| {
17580            calls.fetch_add(1, Ordering::SeqCst);
17581            async move { Ok(Value::Null) }
17582        });
17583        let calls = Arc::clone(&handler_calls);
17584        worker.register_query("codec.workflow", "known", move |_ctx, _args| {
17585            calls.fetch_add(1, Ordering::SeqCst);
17586            async move { Ok(Value::Null) }
17587        });
17588
17589        let mut workflow = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
17590        workflow.payload_codec = "json".to_string();
17591        workflow.arguments = None;
17592        let error = worker
17593            .execute_workflow_task(workflow)
17594            .expect_err("task codec must be checked before workflow invocation");
17595        assert!(error.to_string().contains("unsupported_payload_codec"));
17596
17597        let activity = ActivityTask {
17598            task_id: "activity-invalid-codec".to_string(),
17599            activity_attempt_id: None,
17600            attempt_id: None,
17601            activity_type: "codec.activity".to_string(),
17602            payload_codec: "unknown".to_string(),
17603            arguments: None,
17604            attempt_number: 1,
17605            lease_owner: None,
17606        };
17607        let error = worker
17608            .execute_activity_task(activity)
17609            .await
17610            .expect_err("task codec must be checked before activity invocation");
17611        assert!(error.to_string().contains("unsupported_payload_codec"));
17612
17613        let mut update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
17614        update.workflow_update_id = Some("update-invalid-codec".to_string());
17615        update.update_name = Some("known".to_string());
17616        update.history_events.push(history_event(
17617            "UpdateAccepted",
17618            json!({
17619                "update_id": "update-invalid-codec",
17620                "update_name": "known",
17621                "arguments": {"codec": "json", "blob": null}
17622            }),
17623        ));
17624        let error = worker
17625            .execute_workflow_task(update)
17626            .expect_err("nested update codec must be checked before handler lookup");
17627        assert!(error.to_string().contains("unsupported_payload_codec"));
17628
17629        let query: QueryTask = serde_json::from_value(json!({
17630            "query_task_id": "query-invalid-codec",
17631            "workflow_type": "codec.workflow",
17632            "query_name": "known",
17633            "payload_codec": DEFAULT_CODEC,
17634            "workflow_arguments": null,
17635            "query_arguments": null,
17636            "history_export": {
17637                "payloads": {"codec": DEFAULT_CODEC},
17638                "signals": [{
17639                    "name": "empty",
17640                    "payload_codec": "json",
17641                    "arguments": null
17642                }]
17643            }
17644        }))
17645        .expect("query task");
17646        let failure = worker
17647            .execute_query_task(query)
17648            .await
17649            .expect_err("exported signal codec must be checked before query invocation");
17650        assert_eq!(failure.reason, "query_payload_decode_failed");
17651        assert!(failure.message.contains("unsupported_payload_codec"));
17652
17653        let exported_history: QueryTask = serde_json::from_value(json!({
17654            "query_task_id": "query-invalid-history-codec",
17655            "workflow_type": "codec.workflow",
17656            "query_name": "known",
17657            "payload_codec": DEFAULT_CODEC,
17658            "history_export": {
17659                "payloads": {"codec": DEFAULT_CODEC},
17660                "history_events": [{
17661                    "type": "ActivityCompleted",
17662                    "payload": {"payload_codec": "unknown", "result": null}
17663                }]
17664            }
17665        }))
17666        .expect("query task");
17667        let failure = worker
17668            .execute_query_task(exported_history)
17669            .await
17670            .expect_err("exported history codec must be checked before query invocation");
17671        assert_eq!(failure.reason, "query_payload_decode_failed");
17672        assert!(failure.message.contains("unsupported_payload_codec"));
17673        assert_eq!(handler_calls.load(Ordering::SeqCst), 0);
17674
17675        let mut unknown_workflow = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
17676        unknown_workflow.arguments = None;
17677        unknown_workflow.history_events.push(history_event(
17678            "SignalReceived",
17679            json!({
17680                "signal_name": "empty",
17681                "payload_codec": "json",
17682                "arguments": null
17683            }),
17684        ));
17685        let error = worker
17686            .execute_workflow_task(unknown_workflow)
17687            .expect_err("history codec must precede unknown workflow outcome");
17688        assert!(error.to_string().contains("unsupported_payload_codec"));
17689
17690        let unknown_activity = ActivityTask {
17691            task_id: "activity-unknown".to_string(),
17692            activity_attempt_id: None,
17693            attempt_id: None,
17694            activity_type: "missing".to_string(),
17695            payload_codec: "json".to_string(),
17696            arguments: None,
17697            attempt_number: 1,
17698            lease_owner: None,
17699        };
17700        let error = worker
17701            .execute_activity_task(unknown_activity)
17702            .await
17703            .expect_err("codec must precede unknown activity outcome");
17704        assert!(error.to_string().contains("unsupported_payload_codec"));
17705
17706        let mut unknown_update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
17707        unknown_update.payload_codec = "json".to_string();
17708        unknown_update.arguments = None;
17709        unknown_update.workflow_update_id = Some("update-unknown".to_string());
17710        unknown_update.update_name = Some("missing".to_string());
17711        let error = worker
17712            .execute_workflow_task(unknown_update)
17713            .expect_err("codec must precede fail_update shortcut");
17714        assert!(error.to_string().contains("unsupported_payload_codec"));
17715
17716        let unknown_query: QueryTask = serde_json::from_value(json!({
17717            "query_task_id": "query-unknown",
17718            "workflow_type": "missing",
17719            "query_name": "missing",
17720            "payload_codec": "json",
17721            "workflow_arguments": null,
17722            "query_arguments": null
17723        }))
17724        .expect("query task");
17725        let failure = worker
17726            .execute_query_task(unknown_query)
17727            .await
17728            .expect_err("codec must precede unknown query outcome");
17729        assert_eq!(failure.reason, "query_payload_decode_failed");
17730        assert!(failure.message.contains("unsupported_payload_codec"));
17731    }
17732
17733    #[tokio::test]
17734    async fn invalid_signal_history_payload_aliases_precede_shortcuts() {
17735        let client = Client::new("http://127.0.0.1:8080").expect("client");
17736        let worker = Worker::new(client, "rust-workers");
17737
17738        for event_type in ["SignalReceived", "SignalApplied"] {
17739            for (payload_field, codec) in [
17740                ("value", "json"),
17741                ("input", "unknown"),
17742                ("arguments", "json"),
17743            ] {
17744                let payload = json!({
17745                    "signal_name": "empty",
17746                    payload_field: {"codec": codec, "blob": null}
17747                });
17748                let workflow = workflow_task(
17749                    "missing",
17750                    vec![history_event(event_type, payload.clone())],
17751                    DEFAULT_CODEC,
17752                );
17753                let error = worker
17754                    .execute_workflow_task(workflow)
17755                    .expect_err("signal payload codec must precede unknown workflow outcome");
17756                assert!(
17757                    error.to_string().contains("unsupported_payload_codec"),
17758                    "{event_type}.{payload_field} returned an unrelated workflow error: {error}"
17759                );
17760
17761                let query: QueryTask = serde_json::from_value(json!({
17762                    "query_task_id": format!("query-{event_type}-{payload_field}"),
17763                    "workflow_type": "missing",
17764                    "query_name": "missing",
17765                    "payload_codec": DEFAULT_CODEC,
17766                    "workflow_arguments": null,
17767                    "query_arguments": null,
17768                    "history_events": [{
17769                        "event_type": event_type,
17770                        "payload": payload
17771                    }]
17772                }))
17773                .expect("query task");
17774                let failure = worker
17775                    .execute_query_task(query)
17776                    .await
17777                    .expect_err("signal payload codec must precede unknown query outcome");
17778                assert_eq!(
17779                    failure.reason, "query_payload_decode_failed",
17780                    "{event_type}.{payload_field} returned an unrelated query outcome"
17781                );
17782                assert!(
17783                    failure.message.contains("unsupported_payload_codec"),
17784                    "{event_type}.{payload_field} returned an unrelated query error: {}",
17785                    failure.message
17786                );
17787            }
17788        }
17789    }
17790
17791    #[test]
17792    fn workflow_context_schedules_activity_until_completion_is_in_history() {
17793        let ctx = WorkflowContext {
17794            state: Arc::new(Mutex::new(
17795                WorkflowState::new_with_identity(
17796                    Vec::new(),
17797                    Some("wf-parent".to_string()),
17798                    Some("run-parent".to_string()),
17799                    "rust-workers".to_string(),
17800                    DEFAULT_CODEC.to_string(),
17801                    None,
17802                )
17803                .expect("workflow state"),
17804            )),
17805        };
17806
17807        let mut call = Box::pin(ctx.activity("hello.activity", json!(["Ada"])));
17808        let mut task_context = TaskContext::from_waker(noop_waker_ref());
17809        assert!(matches!(
17810            call.as_mut().poll(&mut task_context),
17811            Poll::Pending
17812        ));
17813
17814        let commands = ctx.take_commands().expect("commands");
17815        assert_eq!(commands[0]["type"], "schedule_activity");
17816        assert_eq!(commands[0]["activity_type"], "hello.activity");
17817    }
17818
17819    #[test]
17820    fn activity_options_encode_retry_policy_queue_and_every_timeout() {
17821        let ctx = workflow_context(Vec::new());
17822        let options = ActivityOptions::new()
17823            .task_queue("payments")
17824            .retry_policy(
17825                ActivityRetryPolicy::new(4)
17826                    .exponential_backoff(Duration::from_secs(1), 3, Some(Duration::from_secs(10)))
17827                    .non_retryable_error_type("ValidationError"),
17828            )
17829            .start_to_close_timeout(Duration::from_secs(120))
17830            .schedule_to_start_timeout(Duration::from_secs(10))
17831            .schedule_to_close_timeout(Duration::from_secs(300))
17832            .heartbeat_timeout(Duration::from_secs(15));
17833        let mut call = Box::pin(ctx.activity_with_options(
17834            "charge-card",
17835            options,
17836            json!([{"order_id": "o-1"}]),
17837        ));
17838        let mut task_context = TaskContext::from_waker(noop_waker_ref());
17839
17840        assert!(matches!(
17841            call.as_mut().poll(&mut task_context),
17842            Poll::Pending
17843        ));
17844        assert!(matches!(
17845            call.as_mut().poll(&mut task_context),
17846            Poll::Pending
17847        ));
17848
17849        let commands = ctx.take_commands().expect("activity command");
17850        assert_eq!(commands.len(), 1, "one future emits one logical schedule");
17851        assert_eq!(commands[0]["queue"], "payments");
17852        assert_eq!(
17853            commands[0]["retry_policy"],
17854            json!({
17855                "max_attempts": 4,
17856                "backoff_seconds": [1, 3, 9],
17857                "non_retryable_error_types": ["ValidationError"],
17858            })
17859        );
17860        assert_eq!(commands[0]["start_to_close_timeout"], 120);
17861        assert_eq!(commands[0]["schedule_to_start_timeout"], 10);
17862        assert_eq!(commands[0]["schedule_to_close_timeout"], 300);
17863        assert_eq!(commands[0]["heartbeat_timeout"], 15);
17864    }
17865
17866    #[test]
17867    fn activity_options_encode_explicit_and_rounded_backoff_intervals() {
17868        let ctx = workflow_context(Vec::new());
17869        let options = ActivityOptions::new().retry_policy(
17870            ActivityRetryPolicy::new(3)
17871                .backoff_intervals([Duration::from_millis(1), Duration::from_millis(1_001)]),
17872        );
17873        let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
17874        let mut task_context = TaskContext::from_waker(noop_waker_ref());
17875
17876        assert!(matches!(
17877            call.as_mut().poll(&mut task_context),
17878            Poll::Pending
17879        ));
17880        assert_eq!(
17881            ctx.take_commands().expect("command")[0]["retry_policy"]["backoff_seconds"],
17882            json!([1, 2])
17883        );
17884    }
17885
17886    #[test]
17887    fn invalid_activity_options_return_typed_errors_before_emitting_commands() {
17888        let cases = [
17889            (
17890                ActivityOptions::new().task_queue("  "),
17891                ActivityOptionsErrorKind::EmptyTaskQueue,
17892            ),
17893            (
17894                ActivityOptions::new().retry_policy(ActivityRetryPolicy::default()),
17895                ActivityOptionsErrorKind::EmptyRetryPolicy,
17896            ),
17897            (
17898                ActivityOptions::new().retry_policy(ActivityRetryPolicy::new(0)),
17899                ActivityOptionsErrorKind::InvalidMaxAttempts,
17900            ),
17901            (
17902                ActivityOptions::new().retry_policy(ActivityRetryPolicy {
17903                    max_attempts: None,
17904                    backoff: Some(ActivityBackoff::Explicit(vec![Duration::from_secs(1)])),
17905                    non_retryable_error_types: Vec::new(),
17906                }),
17907                ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
17908            ),
17909            (
17910                ActivityOptions::new().retry_policy(
17911                    ActivityRetryPolicy::new(2)
17912                        .backoff_intervals([Duration::from_secs(1), Duration::from_secs(2)]),
17913                ),
17914                ActivityOptionsErrorKind::TooManyBackoffIntervals,
17915            ),
17916            (
17917                ActivityOptions::new().retry_policy(
17918                    ActivityRetryPolicy::new(2).exponential_backoff(
17919                        Duration::from_secs(1),
17920                        0,
17921                        None,
17922                    ),
17923                ),
17924                ActivityOptionsErrorKind::InvalidBackoffCoefficient,
17925            ),
17926            (
17927                ActivityOptions::new()
17928                    .retry_policy(ActivityRetryPolicy::new(2).non_retryable_error_type("  ")),
17929                ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
17930            ),
17931            (
17932                ActivityOptions::new().retry_policy(
17933                    ActivityRetryPolicy::new(10_002).exponential_backoff(
17934                        Duration::from_secs(1),
17935                        1,
17936                        None,
17937                    ),
17938                ),
17939                ActivityOptionsErrorKind::BackoffGenerationTooLarge,
17940            ),
17941            (
17942                ActivityOptions::new().retry_policy(
17943                    ActivityRetryPolicy::new(2)
17944                        .backoff_intervals([Duration::from_secs(i64::MAX as u64 + 1)]),
17945                ),
17946                ActivityOptionsErrorKind::BackoffOverflow,
17947            ),
17948        ];
17949
17950        for (options, expected_kind) in cases {
17951            let ctx = workflow_context(Vec::new());
17952            let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
17953            let mut task_context = TaskContext::from_waker(noop_waker_ref());
17954            let Poll::Ready(Err(Error::InvalidActivityOptions(error))) =
17955                call.as_mut().poll(&mut task_context)
17956            else {
17957                panic!("expected typed activity validation error");
17958            };
17959            assert_eq!(error.kind, expected_kind);
17960            assert!(ctx.take_commands().expect("commands").is_empty());
17961        }
17962    }
17963
17964    #[test]
17965    fn activity_options_validate_positive_and_ordered_timeouts() {
17966        let zero_timeout_cases = [
17967            ActivityOptions::new().start_to_close_timeout(Duration::ZERO),
17968            ActivityOptions::new().schedule_to_start_timeout(Duration::ZERO),
17969            ActivityOptions::new().schedule_to_close_timeout(Duration::ZERO),
17970            ActivityOptions::new().heartbeat_timeout(Duration::ZERO),
17971        ];
17972        for options in zero_timeout_cases {
17973            assert_eq!(
17974                options.validate().expect_err("zero timeout").kind,
17975                ActivityOptionsErrorKind::TimeoutNotPositive
17976            );
17977        }
17978
17979        let ordering_cases = [
17980            ActivityOptions::new()
17981                .heartbeat_timeout(Duration::from_secs(11))
17982                .start_to_close_timeout(Duration::from_secs(10)),
17983            ActivityOptions::new()
17984                .start_to_close_timeout(Duration::from_secs(31))
17985                .schedule_to_close_timeout(Duration::from_secs(30)),
17986            ActivityOptions::new()
17987                .schedule_to_start_timeout(Duration::from_secs(31))
17988                .schedule_to_close_timeout(Duration::from_secs(30)),
17989        ];
17990        for options in ordering_cases {
17991            assert_eq!(
17992                options.validate().expect_err("timeout order").kind,
17993                ActivityOptionsErrorKind::TimeoutOrder
17994            );
17995        }
17996
17997        assert_eq!(
17998            ActivityOptions::new()
17999                .start_to_close_timeout(Duration::from_secs(i64::MAX as u64 + 1))
18000                .validate()
18001                .expect_err("protocol integer overflow")
18002                .kind,
18003            ActivityOptionsErrorKind::TimeoutOverflow
18004        );
18005    }
18006
18007    #[test]
18008    fn replayed_activity_retry_history_completes_without_duplicate_schedule() {
18009        let ctx = workflow_context(completed_retry_activity_history());
18010        let mut call =
18011            Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18012        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18013
18014        assert!(matches!(
18015            call.as_mut().poll(&mut task_context),
18016            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18017        ));
18018        assert!(ctx.take_commands().expect("commands").is_empty());
18019        ctx.ensure_history_consumed().expect("history consumed");
18020    }
18021
18022    #[test]
18023    fn duplicate_non_retryable_types_use_one_command_and_replay_representation() {
18024        let mut options = retry_activity_options();
18025        options
18026            .retry_policy
18027            .as_mut()
18028            .expect("retry policy")
18029            .non_retryable_error_types
18030            .extend([" PermanentError ".to_string(), "PermanentError".to_string()]);
18031
18032        let new_ctx = workflow_context(Vec::new());
18033        let mut new_call =
18034            Box::pin(new_ctx.activity_with_options("flaky", options.clone(), json!([])));
18035        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18036        assert!(matches!(
18037            new_call.as_mut().poll(&mut task_context),
18038            Poll::Pending
18039        ));
18040        let commands = new_ctx.take_commands().expect("commands");
18041        assert_eq!(commands.len(), 1);
18042        assert_eq!(
18043            commands[0]["retry_policy"]["non_retryable_error_types"],
18044            json!(["PermanentError"])
18045        );
18046
18047        let replay_ctx = workflow_context(completed_retry_activity_history());
18048        let mut replay_call =
18049            Box::pin(replay_ctx.activity_with_options("flaky", options, json!([])));
18050        assert!(matches!(
18051            replay_call.as_mut().poll(&mut task_context),
18052            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18053        ));
18054        assert!(replay_ctx.take_commands().expect("commands").is_empty());
18055        replay_ctx
18056            .ensure_history_consumed()
18057            .expect("history consumed");
18058    }
18059
18060    #[test]
18061    fn replayed_intermediate_retry_remains_pending_across_restarts() {
18062        let history = completed_retry_activity_history()
18063            .into_iter()
18064            .take(3)
18065            .collect::<Vec<_>>();
18066
18067        for _restart in 0..2 {
18068            let ctx = workflow_context(history.clone());
18069            let mut call =
18070                Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18071            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18072            assert!(matches!(
18073                call.as_mut().poll(&mut task_context),
18074                Poll::Pending
18075            ));
18076            assert!(ctx.take_commands().expect("commands").is_empty());
18077        }
18078    }
18079
18080    #[test]
18081    fn replayed_activity_rejects_changed_queue_retry_and_every_timeout_field() {
18082        let mut changed_queue = retry_activity_options();
18083        changed_queue.task_queue = Some("different-queue".to_string());
18084
18085        let mut changed_max_attempts = retry_activity_options();
18086        let retry_policy = changed_max_attempts
18087            .retry_policy
18088            .as_mut()
18089            .expect("retry policy");
18090        retry_policy.max_attempts = Some(4);
18091
18092        let mut changed_backoff = retry_activity_options();
18093        let retry_policy = changed_backoff.retry_policy.as_mut().expect("retry policy");
18094        retry_policy.backoff = Some(ActivityBackoff::Explicit(vec![
18095            Duration::from_secs(3),
18096            Duration::from_secs(4),
18097        ]));
18098
18099        let mut changed_non_retryable_types = retry_activity_options();
18100        let retry_policy = changed_non_retryable_types
18101            .retry_policy
18102            .as_mut()
18103            .expect("retry policy");
18104        retry_policy.non_retryable_error_types = vec!["AnotherPermanentError".to_string()];
18105
18106        let mut changed_start_to_close = retry_activity_options();
18107        changed_start_to_close.start_to_close_timeout = Some(Duration::from_secs(31));
18108        let mut changed_schedule_to_start = retry_activity_options();
18109        changed_schedule_to_start.schedule_to_start_timeout = Some(Duration::from_secs(6));
18110        let mut changed_schedule_to_close = retry_activity_options();
18111        changed_schedule_to_close.schedule_to_close_timeout = Some(Duration::from_secs(91));
18112        let mut changed_heartbeat = retry_activity_options();
18113        changed_heartbeat.heartbeat_timeout = Some(Duration::from_secs(11));
18114
18115        let cases = [
18116            (changed_queue, "activity_task_queue_mismatch"),
18117            (changed_max_attempts, "activity_retry_policy_mismatch"),
18118            (changed_backoff, "activity_retry_policy_mismatch"),
18119            (
18120                changed_non_retryable_types,
18121                "activity_retry_policy_mismatch",
18122            ),
18123            (changed_start_to_close, "activity_retry_policy_mismatch"),
18124            (changed_schedule_to_start, "activity_retry_policy_mismatch"),
18125            (changed_schedule_to_close, "activity_retry_policy_mismatch"),
18126            (changed_heartbeat, "activity_retry_policy_mismatch"),
18127        ];
18128
18129        for (options, expected_reason) in cases {
18130            let ctx = workflow_context(completed_retry_activity_history());
18131            let mut call = Box::pin(ctx.activity_with_options("flaky", options, json!([])));
18132            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18133            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18134                call.as_mut().poll(&mut task_context)
18135            else {
18136                panic!("changed activity options must fail replay");
18137            };
18138            assert_eq!(failure.reason, expected_reason);
18139            assert_eq!(failure.sequence, Some(1));
18140            assert!(ctx.take_commands().expect("commands").is_empty());
18141        }
18142    }
18143
18144    #[test]
18145    fn replayed_activity_rejects_changed_execution_mode_and_snapshot_version() {
18146        let cases = [
18147            (
18148                "execution_mode",
18149                json!("local"),
18150                "activity_execution_mode_mismatch",
18151            ),
18152            (
18153                "snapshot_version",
18154                json!(2),
18155                "activity_retry_policy_mismatch",
18156            ),
18157        ];
18158
18159        for (field, value, expected_reason) in cases {
18160            let mut history = completed_retry_activity_history();
18161            let activity = history[0].payload["activity"]
18162                .as_object_mut()
18163                .expect("activity snapshot");
18164            if field == "execution_mode" {
18165                activity.insert(field.to_string(), value);
18166            } else {
18167                activity["retry_policy"]
18168                    .as_object_mut()
18169                    .expect("retry snapshot")
18170                    .insert(field.to_string(), value);
18171            }
18172
18173            let ctx = workflow_context(history);
18174            let mut call =
18175                Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18176            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18177            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18178                call.as_mut().poll(&mut task_context)
18179            else {
18180                panic!("changed {field} must fail replay");
18181            };
18182            assert_eq!(failure.reason, expected_reason);
18183            assert_eq!(failure.sequence, Some(1));
18184            assert!(ctx.take_commands().expect("commands").is_empty());
18185        }
18186    }
18187
18188    #[test]
18189    fn replayed_legacy_activity_treats_missing_option_snapshot_as_unknown() {
18190        let mut history = completed_retry_activity_history();
18191        let activity = history[0].payload["activity"]
18192            .as_object_mut()
18193            .expect("activity snapshot");
18194        activity.remove("execution_mode");
18195        activity.remove("retry_policy");
18196
18197        let mut current = retry_activity_options();
18198        current.start_to_close_timeout = Some(Duration::from_secs(45));
18199        current.schedule_to_start_timeout = Some(Duration::from_secs(8));
18200        current.schedule_to_close_timeout = Some(Duration::from_secs(120));
18201        current.heartbeat_timeout = Some(Duration::from_secs(12));
18202
18203        let ctx = workflow_context(history);
18204        let mut call = Box::pin(ctx.activity_with_options("flaky", current, json!([])));
18205        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18206        assert!(matches!(
18207            call.as_mut().poll(&mut task_context),
18208            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18209        ));
18210        assert!(ctx.take_commands().expect("commands").is_empty());
18211        ctx.ensure_history_consumed().expect("history consumed");
18212    }
18213
18214    #[test]
18215    fn terminal_activity_failed_after_start_returns_typed_failure() {
18216        let history = vec![
18217            history_event(
18218                "ActivityScheduled",
18219                json!({
18220                    "sequence": 1,
18221                    "activity_type": "flaky",
18222                    "activity_execution_id": "act-terminal",
18223                    "activity": {
18224                        "id": "act-terminal",
18225                        "sequence": 1,
18226                        "type": "flaky",
18227                        "queue": "critical-activities",
18228                        "retry_policy": {
18229                            "snapshot_version": 1,
18230                            "max_attempts": 3,
18231                            "backoff_seconds": [2, 4],
18232                            "non_retryable_error_types": ["PermanentError"]
18233                        }
18234                    }
18235                }),
18236            ),
18237            history_event(
18238                "ActivityStarted",
18239                json!({
18240                    "sequence": 1,
18241                    "activity_type": "flaky",
18242                    "activity_execution_id": "act-terminal",
18243                    "activity_attempt_id": "attempt-1",
18244                    "attempt_number": 1
18245                }),
18246            ),
18247            history_event(
18248                "ActivityFailed",
18249                json!({
18250                    "sequence": 1,
18251                    "activity_type": "flaky",
18252                    "activity_execution_id": "act-terminal",
18253                    "activity_attempt_id": "attempt-1",
18254                    "attempt_number": 1,
18255                    "failure_id": "failure-terminal",
18256                    "failure_category": "activity",
18257                    "exception_type": "PermanentError",
18258                    "message": "cannot retry",
18259                    "non_retryable": true
18260                }),
18261            ),
18262        ];
18263        let ctx = workflow_context(history);
18264        let mut call =
18265            Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18266        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18267
18268        let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18269            call.as_mut().poll(&mut task_context)
18270        else {
18271            panic!("terminal ActivityFailed must settle the activity future");
18272        };
18273        assert_eq!(failure.kind, ActivityFailureKind::Failed);
18274        assert_eq!(
18275            failure.activity_execution_id.as_deref(),
18276            Some("act-terminal")
18277        );
18278        assert_eq!(failure.exception_type.as_deref(), Some("PermanentError"));
18279        assert!(failure.non_retryable);
18280        assert!(ctx.take_commands().expect("commands").is_empty());
18281        ctx.ensure_history_consumed().expect("history consumed");
18282    }
18283
18284    #[test]
18285    fn activity_terminal_events_return_machine_readable_failures() {
18286        let cases = [
18287            (
18288                "ActivityFailed",
18289                json!({
18290                    "sequence": 1,
18291                    "activity_type": "charge-card",
18292                    "activity_execution_id": "act-1",
18293                    "activity_attempt_id": "attempt-2",
18294                    "attempt_number": 2,
18295                    "failure_id": "failure-1",
18296                    "failure_category": "activity",
18297                    "exception_type": "PaymentDeclined",
18298                    "exception_class": "payments.PaymentDeclined",
18299                    "message": "card declined",
18300                    "non_retryable": true
18301                }),
18302                ActivityFailureKind::Failed,
18303                "activity",
18304            ),
18305            (
18306                "ActivityCancelled",
18307                json!({
18308                    "sequence": 1,
18309                    "activity_type": "charge-card",
18310                    "activity_execution_id": "act-1",
18311                    "activity_attempt_id": "attempt-1"
18312                }),
18313                ActivityFailureKind::Cancelled,
18314                "cancelled",
18315            ),
18316        ];
18317
18318        for (event_type, payload, expected_kind, expected_reason) in cases {
18319            let ctx = workflow_context(vec![history_event(event_type, payload)]);
18320            let mut call = Box::pin(ctx.activity("charge-card", json!([])));
18321            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18322            let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18323                call.as_mut().poll(&mut task_context)
18324            else {
18325                panic!("expected terminal activity failure");
18326            };
18327            assert_eq!(failure.kind, expected_kind);
18328            assert_eq!(failure.reason, expected_reason);
18329            assert_eq!(failure.activity_execution_id.as_deref(), Some("act-1"));
18330            assert_eq!(failure.activity_type.as_deref(), Some("charge-card"));
18331        }
18332    }
18333
18334    #[test]
18335    fn every_activity_timeout_class_is_typed() {
18336        for timeout_kind in [
18337            "start_to_close",
18338            "schedule_to_start",
18339            "schedule_to_close",
18340            "heartbeat",
18341        ] {
18342            let ctx = workflow_context(vec![history_event(
18343                "ActivityTimedOut",
18344                json!({
18345                    "sequence": 1,
18346                    "activity_type": "slow",
18347                    "activity_execution_id": "act-timeout",
18348                    "activity_attempt_id": "attempt-timeout",
18349                    "failure_category": "timeout",
18350                    "timeout_kind": timeout_kind,
18351                    "message": "deadline expired"
18352                }),
18353            )]);
18354            let mut call = Box::pin(ctx.activity("slow", json!([])));
18355            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18356            let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18357                call.as_mut().poll(&mut task_context)
18358            else {
18359                panic!("expected timeout failure");
18360            };
18361            assert_eq!(failure.kind, ActivityFailureKind::TimedOut);
18362            assert_eq!(failure.reason, timeout_kind);
18363            assert_eq!(failure.timeout_kind.as_deref(), Some(timeout_kind));
18364            assert_eq!(failure.failure_category.as_deref(), Some("timeout"));
18365        }
18366    }
18367
18368    #[test]
18369    fn workflow_sleep_emits_one_durable_timer_and_rounds_up() {
18370        let ctx = workflow_context(Vec::new());
18371        let mut sleep = Box::pin(ctx.sleep(Duration::from_millis(1_001)));
18372        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18373
18374        assert!(matches!(
18375            sleep.as_mut().poll(&mut task_context),
18376            Poll::Pending
18377        ));
18378        assert!(matches!(
18379            sleep.as_mut().poll(&mut task_context),
18380            Poll::Pending
18381        ));
18382
18383        let commands = ctx.take_commands().expect("timer command");
18384        assert_eq!(
18385            commands,
18386            vec![json!({
18387                "type": "start_timer",
18388                "delay_seconds": 2,
18389            })]
18390        );
18391    }
18392
18393    #[test]
18394    fn workflow_sleep_replays_matching_schedule_and_fire_without_a_command() {
18395        let history = vec![
18396            history_event(
18397                "TimerScheduled",
18398                json!({
18399                    "sequence": 1,
18400                    "timer_id": "timer-1",
18401                    "delay_seconds": 5,
18402                    "fire_at": "2026-07-11T12:00:05Z",
18403                }),
18404            ),
18405            history_event(
18406                "TimerFired",
18407                json!({
18408                    "sequence": 1,
18409                    "timer_id": "timer-1",
18410                    "delay_seconds": 5,
18411                    "fire_at": "2026-07-11T12:00:05Z",
18412                    "fired_at": "2026-07-11T12:00:05Z",
18413                }),
18414            ),
18415        ];
18416
18417        for _restart in 0..2 {
18418            let ctx = workflow_context(history.clone());
18419            let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(5)));
18420            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18421            assert!(matches!(
18422                sleep.as_mut().poll(&mut task_context),
18423                Poll::Ready(Ok(()))
18424            ));
18425            assert!(ctx.take_commands().expect("commands").is_empty());
18426            ctx.ensure_history_consumed().expect("history consumed");
18427        }
18428    }
18429
18430    #[test]
18431    fn workflow_sleep_rejects_changed_delay_during_replay() {
18432        let ctx = workflow_context(vec![
18433            history_event(
18434                "TimerScheduled",
18435                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
18436            ),
18437            history_event(
18438                "TimerFired",
18439                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
18440            ),
18441        ]);
18442        let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(500)));
18443        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18444
18445        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18446            sleep.as_mut().poll(&mut task_context)
18447        else {
18448            panic!("changed timer delay must be rejected");
18449        };
18450        assert_eq!(failure.reason, "timer_delay_mismatch");
18451        assert_eq!(failure.sequence, Some(1));
18452    }
18453
18454    #[test]
18455    fn workflow_condition_wait_emits_published_identity_and_timeout_contract() {
18456        let ctx = workflow_context(Vec::new());
18457        let mut wait = Box::pin(
18458            ctx.wait_condition(
18459                ConditionWaitOptions::new("approval.ready", "sha256:approval-v1")
18460                    .timeout(Duration::from_millis(60_001)),
18461                || Ok(false),
18462            ),
18463        );
18464        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18465
18466        assert!(matches!(
18467            wait.as_mut().poll(&mut task_context),
18468            Poll::Pending
18469        ));
18470        assert!(matches!(
18471            wait.as_mut().poll(&mut task_context),
18472            Poll::Pending
18473        ));
18474        assert_eq!(
18475            ctx.take_commands().expect("condition command"),
18476            vec![json!({
18477                "type": "open_condition_wait",
18478                "condition_wait_occurrence_id": "rust:condition-wait:0",
18479                "condition_key": "approval.ready",
18480                "condition_definition_fingerprint": "sha256:approval-v1",
18481                "timeout_seconds": 61,
18482            })]
18483        );
18484    }
18485
18486    #[test]
18487    fn workflow_condition_wait_returns_explicit_immediate_results_without_commands() {
18488        let ctx = workflow_context(Vec::new());
18489        let mut satisfied = Box::pin(wait_condition!(ctx, "already-ready", || Ok(true)));
18490        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18491        assert!(matches!(
18492            satisfied.as_mut().poll(&mut task_context),
18493            Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18494        ));
18495
18496        let mut timed_out = Box::pin(wait_condition!(
18497            ctx,
18498            "no-wait",
18499            timeout: Duration::ZERO,
18500            || Ok(false),
18501        ));
18502        assert!(matches!(
18503            timed_out.as_mut().poll(&mut task_context),
18504            Poll::Ready(Ok(ConditionWaitResult::TimedOut))
18505        ));
18506        assert!(ctx.take_commands().expect("commands").is_empty());
18507    }
18508
18509    #[test]
18510    fn signal_and_update_history_reevaluate_open_conditions_after_restart() {
18511        let signal_history = vec![
18512            history_event(
18513                "ConditionWaitOpened",
18514                json!({
18515                    "sequence": 4,
18516                    "condition_wait_id": "condition:4",
18517                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18518                    "condition_key": "approval",
18519                    "condition_definition_fingerprint": "sha256:approval-v1",
18520                    "timeout_seconds": 30,
18521                }),
18522            ),
18523            history_event(
18524                "SignalReceived",
18525                json!({
18526                    "workflow_sequence": 4,
18527                    "signal_name": "approve",
18528                    "arguments": fixture_envelope(json!(["Ada"])),
18529                }),
18530            ),
18531        ];
18532        for _worker_before_or_after_restart in 0..2 {
18533            let ctx = workflow_context(signal_history.clone());
18534            let predicate_ctx = ctx.clone();
18535            let mut wait = Box::pin(
18536                ctx.wait_condition(
18537                    ConditionWaitOptions::new("approval", "sha256:approval-v1")
18538                        .timeout(Duration::from_secs(30)),
18539                    move || Ok(!predicate_ctx.signals("approve")?.is_empty()),
18540                ),
18541            );
18542            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18543            assert!(matches!(
18544                wait.as_mut().poll(&mut task_context),
18545                Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18546            ));
18547            assert!(ctx.take_commands().expect("commands").is_empty());
18548            ctx.ensure_history_consumed().expect("condition consumed");
18549        }
18550
18551        let update_history = vec![
18552            history_event(
18553                "ConditionWaitOpened",
18554                json!({
18555                    "sequence": 7,
18556                    "condition_wait_id": "condition:7",
18557                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18558                    "condition_key": "update-approval",
18559                    "condition_definition_fingerprint": "sha256:update-approval-v1",
18560                }),
18561            ),
18562            history_event(
18563                "UpdateApplied",
18564                json!({
18565                    "sequence": 7,
18566                    "update_id": "update-1",
18567                    "update_name": "approve",
18568                    "arguments": fixture_envelope(json!([true])),
18569                }),
18570            ),
18571        ];
18572        let ctx = workflow_context(update_history);
18573        let predicate_ctx = ctx.clone();
18574        let mut wait = Box::pin(ctx.wait_condition(
18575            ConditionWaitOptions::new("update-approval", "sha256:update-approval-v1"),
18576            move || {
18577                Ok(predicate_ctx
18578                    .updates("approve")?
18579                    .first()
18580                    .and_then(|arguments| arguments.first())
18581                    .and_then(Value::as_bool)
18582                    == Some(true))
18583            },
18584        ));
18585        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18586        assert!(matches!(
18587            wait.as_mut().poll(&mut task_context),
18588            Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18589        ));
18590        assert!(ctx.take_commands().expect("commands").is_empty());
18591        ctx.ensure_history_consumed().expect("condition consumed");
18592    }
18593
18594    #[test]
18595    fn condition_wait_preserves_open_satisfied_and_timed_out_replay_states() {
18596        let open_history = vec![
18597            history_event(
18598                "ConditionWaitOpened",
18599                json!({
18600                    "sequence": 3,
18601                    "condition_wait_id": "condition:3",
18602                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18603                    "condition_key": "two-votes",
18604                    "condition_definition_fingerprint": "sha256:two-votes-v1",
18605                    "timeout_seconds": 120,
18606                }),
18607            ),
18608            history_event(
18609                "SignalReceived",
18610                json!({
18611                    "workflow_sequence": 3,
18612                    "signal_name": "vote",
18613                    "arguments": fixture_envelope(json!(["first"])),
18614                }),
18615            ),
18616        ];
18617        for _worker_before_or_after_restart in 0..2 {
18618            let ctx = workflow_context(open_history.clone());
18619            let predicate_ctx = ctx.clone();
18620            let mut wait = Box::pin(
18621                ctx.wait_condition(
18622                    ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1")
18623                        .timeout(Duration::from_secs(120)),
18624                    move || Ok(predicate_ctx.signals("vote")?.len() >= 2),
18625                ),
18626            );
18627            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18628            assert!(matches!(
18629                wait.as_mut().poll(&mut task_context),
18630                Poll::Pending
18631            ));
18632            assert_eq!(
18633                ctx.take_commands().expect("reopened condition"),
18634                vec![json!({
18635                    "type": "open_condition_wait",
18636                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18637                    "condition_key": "two-votes",
18638                    "condition_definition_fingerprint": "sha256:two-votes-v1",
18639                    "timeout_seconds": 120,
18640                })]
18641            );
18642        }
18643
18644        let satisfied_ctx = workflow_context(vec![
18645            history_event(
18646                "ConditionWaitOpened",
18647                json!({
18648                    "sequence": 5,
18649                    "condition_wait_id": "condition:5",
18650                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18651                    "condition_key": "approval",
18652                    "condition_definition_fingerprint": "sha256:approval-v1",
18653                }),
18654            ),
18655            history_event(
18656                "ConditionWaitSatisfied",
18657                json!({
18658                    "sequence": 5,
18659                    "condition_wait_id": "condition:5",
18660                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18661                    "condition_key": "approval",
18662                    "condition_definition_fingerprint": "sha256:approval-v1",
18663                }),
18664            ),
18665        ]);
18666        let mut satisfied = Box::pin(satisfied_ctx.wait_condition(
18667            ConditionWaitOptions::new("approval", "sha256:approval-v1"),
18668            || Ok(false),
18669        ));
18670        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18671        assert!(matches!(
18672            satisfied.as_mut().poll(&mut task_context),
18673            Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18674        ));
18675
18676        let timed_out_ctx = workflow_context(vec![
18677            history_event(
18678                "ConditionWaitOpened",
18679                json!({
18680                    "sequence": 8,
18681                    "condition_wait_id": "condition:8",
18682                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18683                    "condition_key": "approval-timeout",
18684                    "condition_definition_fingerprint": "sha256:approval-timeout-v1",
18685                    "timeout_seconds": 5,
18686                }),
18687            ),
18688            history_event(
18689                "TimerScheduled",
18690                json!({
18691                    "sequence": 9,
18692                    "timer_id": "condition-timer:9",
18693                    "timer_kind": "condition_timeout",
18694                    "condition_wait_id": "condition:8",
18695                    "delay_seconds": 5,
18696                }),
18697            ),
18698            history_event(
18699                "TimerFired",
18700                json!({
18701                    "sequence": 9,
18702                    "timer_id": "condition-timer:9",
18703                    "timer_kind": "condition_timeout",
18704                    "condition_wait_id": "condition:8",
18705                    "delay_seconds": 5,
18706                }),
18707            ),
18708        ]);
18709        let mut timed_out = Box::pin(
18710            timed_out_ctx.wait_condition(
18711                ConditionWaitOptions::new("approval-timeout", "sha256:approval-timeout-v1")
18712                    .timeout(Duration::from_secs(5)),
18713                || Ok(true),
18714            ),
18715        );
18716        assert!(matches!(
18717            timed_out.as_mut().poll(&mut task_context),
18718            Poll::Ready(Ok(ConditionWaitResult::TimedOut))
18719        ));
18720    }
18721
18722    #[test]
18723    fn condition_wait_replays_repeated_physical_opens_as_one_logical_wait() {
18724        let history = vec![
18725            history_event(
18726                "ConditionWaitOpened",
18727                json!({
18728                    "sequence": 3,
18729                    "condition_wait_id": "condition:3",
18730                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18731                    "condition_key": "two-votes",
18732                    "condition_definition_fingerprint": "sha256:two-votes-v1",
18733                }),
18734            ),
18735            history_event(
18736                "SignalReceived",
18737                json!({
18738                    "workflow_sequence": 3,
18739                    "signal_name": "vote",
18740                    "arguments": fixture_envelope(json!(["first"])),
18741                }),
18742            ),
18743            history_event(
18744                "ConditionWaitSatisfied",
18745                json!({
18746                    "sequence": 3,
18747                    "condition_wait_id": "condition:3",
18748                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18749                    "condition_key": "two-votes",
18750                    "condition_definition_fingerprint": "sha256:two-votes-v1",
18751                }),
18752            ),
18753            history_event(
18754                "ConditionWaitOpened",
18755                json!({
18756                    "sequence": 5,
18757                    "condition_wait_id": "condition:5",
18758                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18759                    "condition_key": "two-votes",
18760                    "condition_definition_fingerprint": "sha256:two-votes-v1",
18761                }),
18762            ),
18763            history_event(
18764                "SignalReceived",
18765                json!({
18766                    "workflow_sequence": 5,
18767                    "signal_name": "vote",
18768                    "arguments": fixture_envelope(json!(["second"])),
18769                }),
18770            ),
18771            history_event(
18772                "ConditionWaitSatisfied",
18773                json!({
18774                    "sequence": 5,
18775                    "condition_wait_id": "condition:5",
18776                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18777                    "condition_key": "two-votes",
18778                    "condition_definition_fingerprint": "sha256:two-votes-v1",
18779                }),
18780            ),
18781        ];
18782        for _cold_worker_or_restart in 0..2 {
18783            let ctx = workflow_context(history.clone());
18784            let predicate_ctx = ctx.clone();
18785            let mut wait = Box::pin(ctx.wait_condition(
18786                ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1"),
18787                move || Ok(predicate_ctx.signals("vote")?.len() >= 2),
18788            ));
18789            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18790
18791            assert!(matches!(
18792                wait.as_mut().poll(&mut task_context),
18793                Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18794            ));
18795            assert!(ctx.take_commands().expect("commands").is_empty());
18796            ctx.ensure_history_consumed()
18797                .expect("every physical wait-open is consumed");
18798        }
18799    }
18800
18801    #[test]
18802    fn condition_wait_replays_update_driven_physical_opens_as_one_occurrence() {
18803        let history = vec![
18804            history_event(
18805                "ConditionWaitOpened",
18806                json!({
18807                    "sequence": 3,
18808                    "condition_wait_id": "condition:3",
18809                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18810                    "condition_key": "approved",
18811                    "condition_definition_fingerprint": "sha256:approved-v1",
18812                }),
18813            ),
18814            history_event(
18815                "UpdateApplied",
18816                json!({
18817                    "sequence": 3,
18818                    "update_id": "update-1",
18819                    "update_name": "approve",
18820                    "arguments": fixture_envelope(json!([false])),
18821                }),
18822            ),
18823            history_event(
18824                "ConditionWaitOpened",
18825                json!({
18826                    "sequence": 5,
18827                    "condition_wait_id": "condition:5",
18828                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18829                    "condition_key": "approved",
18830                    "condition_definition_fingerprint": "sha256:approved-v1",
18831                }),
18832            ),
18833            history_event(
18834                "UpdateApplied",
18835                json!({
18836                    "sequence": 5,
18837                    "update_id": "update-2",
18838                    "update_name": "approve",
18839                    "arguments": fixture_envelope(json!([true])),
18840                }),
18841            ),
18842        ];
18843
18844        for _cold_worker_or_restart in 0..2 {
18845            let ctx = workflow_context(history.clone());
18846            let predicate_ctx = ctx.clone();
18847            let mut wait = Box::pin(ctx.wait_condition(
18848                ConditionWaitOptions::new("approved", "sha256:approved-v1"),
18849                move || {
18850                    Ok(predicate_ctx
18851                        .updates("approve")?
18852                        .last()
18853                        .and_then(|arguments| arguments.first())
18854                        .and_then(Value::as_bool)
18855                        == Some(true))
18856                },
18857            ));
18858            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18859
18860            assert!(matches!(
18861                wait.as_mut().poll(&mut task_context),
18862                Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18863            ));
18864            assert!(ctx.take_commands().expect("commands").is_empty());
18865            ctx.ensure_history_consumed()
18866                .expect("every update-driven reopen is consumed");
18867        }
18868    }
18869
18870    #[test]
18871    fn condition_wait_replay_keeps_every_adjacent_authored_occurrence_distinct() {
18872        for (first_key, first_fingerprint, second_key, second_fingerprint) in [
18873            ("shared", "sha256:first", "shared", "sha256:second"),
18874            ("first", "sha256:shared", "second", "sha256:shared"),
18875            ("shared", "sha256:shared", "shared", "sha256:shared"),
18876            ("first", "sha256:first", "second", "sha256:second"),
18877        ] {
18878            let history = vec![
18879                history_event(
18880                    "ConditionWaitOpened",
18881                    json!({
18882                        "sequence": 3,
18883                        "condition_wait_id": "condition:3",
18884                        "condition_wait_occurrence_id": "rust:condition-wait:0",
18885                        "condition_key": first_key,
18886                        "condition_definition_fingerprint": first_fingerprint,
18887                    }),
18888                ),
18889                history_event(
18890                    "ConditionWaitSatisfied",
18891                    json!({
18892                        "sequence": 3,
18893                        "condition_wait_id": "condition:3",
18894                        "condition_wait_occurrence_id": "rust:condition-wait:0",
18895                        "condition_key": first_key,
18896                        "condition_definition_fingerprint": first_fingerprint,
18897                    }),
18898                ),
18899                history_event(
18900                    "ConditionWaitOpened",
18901                    json!({
18902                        "sequence": 4,
18903                        "condition_wait_id": "condition:4",
18904                        "condition_wait_occurrence_id": "rust:condition-wait:1",
18905                        "condition_key": second_key,
18906                        "condition_definition_fingerprint": second_fingerprint,
18907                    }),
18908                ),
18909                history_event(
18910                    "ConditionWaitSatisfied",
18911                    json!({
18912                        "sequence": 4,
18913                        "condition_wait_id": "condition:4",
18914                        "condition_wait_occurrence_id": "rust:condition-wait:1",
18915                        "condition_key": second_key,
18916                        "condition_definition_fingerprint": second_fingerprint,
18917                    }),
18918                ),
18919            ];
18920            for _cold_worker_or_restart in 0..2 {
18921                let ctx = workflow_context(history.clone());
18922                let mut task_context = TaskContext::from_waker(noop_waker_ref());
18923                let mut first = Box::pin(ctx.wait_condition(
18924                    ConditionWaitOptions::new(first_key, first_fingerprint),
18925                    || Ok(false),
18926                ));
18927                assert!(matches!(
18928                    first.as_mut().poll(&mut task_context),
18929                    Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18930                ));
18931
18932                let mut second = Box::pin(ctx.wait_condition(
18933                    ConditionWaitOptions::new(second_key, second_fingerprint),
18934                    || Ok(false),
18935                ));
18936                assert!(matches!(
18937                    second.as_mut().poll(&mut task_context),
18938                    Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18939                ));
18940                assert!(ctx.take_commands().expect("commands").is_empty());
18941                ctx.ensure_history_consumed()
18942                    .expect("each authored wait consumes one occurrence");
18943            }
18944        }
18945    }
18946
18947    #[test]
18948    fn cold_workers_replay_adjacent_condition_waits_from_one_loop_call_site() {
18949        fn worker() -> Worker {
18950            let client = Client::new("http://127.0.0.1:8080").expect("client");
18951            let mut worker = Worker::new(client, "rust-workers");
18952            worker.register_workflow("rust.condition-loop", |ctx, _input| async move {
18953                let mut outcomes = Vec::new();
18954                for _ in 0..2 {
18955                    outcomes.push(
18956                        ctx.wait_condition(
18957                            ConditionWaitOptions::new("shared", "sha256:shared"),
18958                            || Ok(false),
18959                        )
18960                        .await?,
18961                    );
18962                }
18963                Ok(json!(outcomes))
18964            });
18965            worker
18966        }
18967
18968        let task = workflow_task(
18969            "rust.condition-loop",
18970            vec![
18971                history_event(
18972                    "ConditionWaitOpened",
18973                    json!({
18974                        "sequence": 1,
18975                        "condition_wait_id": "condition:1",
18976                        "condition_wait_occurrence_id": "rust:condition-wait:0",
18977                        "condition_key": "shared",
18978                        "condition_definition_fingerprint": "sha256:shared",
18979                    }),
18980                ),
18981                history_event(
18982                    "ConditionWaitSatisfied",
18983                    json!({
18984                        "sequence": 1,
18985                        "condition_wait_id": "condition:1",
18986                        "condition_wait_occurrence_id": "rust:condition-wait:0",
18987                        "condition_key": "shared",
18988                        "condition_definition_fingerprint": "sha256:shared",
18989                    }),
18990                ),
18991                history_event(
18992                    "ConditionWaitOpened",
18993                    json!({
18994                        "sequence": 2,
18995                        "condition_wait_id": "condition:2",
18996                        "condition_wait_occurrence_id": "rust:condition-wait:1",
18997                        "condition_key": "shared",
18998                        "condition_definition_fingerprint": "sha256:shared",
18999                    }),
19000                ),
19001                history_event(
19002                    "ConditionWaitSatisfied",
19003                    json!({
19004                        "sequence": 2,
19005                        "condition_wait_id": "condition:2",
19006                        "condition_wait_occurrence_id": "rust:condition-wait:1",
19007                        "condition_key": "shared",
19008                        "condition_definition_fingerprint": "sha256:shared",
19009                    }),
19010                ),
19011            ],
19012            DEFAULT_CODEC,
19013        );
19014
19015        for _cold_worker_or_restart in 0..2 {
19016            let commands = worker()
19017                .execute_workflow_task(task.clone())
19018                .expect("adjacent loop waits replay deterministically");
19019            assert_eq!(commands.len(), 1);
19020            assert_eq!(commands[0]["type"], "complete_workflow");
19021            assert_eq!(
19022                decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
19023                json!(["satisfied", "satisfied"])
19024            );
19025        }
19026    }
19027
19028    #[test]
19029    fn condition_wait_replay_rejects_identity_predicate_and_timeout_changes() {
19030        let history = vec![history_event(
19031            "ConditionWaitOpened",
19032            json!({
19033                "sequence": 12,
19034                "condition_wait_id": "condition:12",
19035                "condition_wait_occurrence_id": "rust:condition-wait:0",
19036                "condition_key": "approval",
19037                "condition_definition_fingerprint": "sha256:approval-v1",
19038                "timeout_seconds": 30,
19039            }),
19040        )];
19041        for (options, expected_reason) in [
19042            (
19043                ConditionWaitOptions::new("changed", "sha256:approval-v1")
19044                    .timeout(Duration::from_secs(30)),
19045                "condition_wait_key_mismatch",
19046            ),
19047            (
19048                ConditionWaitOptions::new("approval", "sha256:approval-v2")
19049                    .timeout(Duration::from_secs(30)),
19050                "condition_wait_predicate_mismatch",
19051            ),
19052            (
19053                ConditionWaitOptions::new("approval", "sha256:approval-v1")
19054                    .timeout(Duration::from_secs(29)),
19055                "condition_wait_timeout_mismatch",
19056            ),
19057        ] {
19058            let ctx = workflow_context(history.clone());
19059            let mut wait = Box::pin(ctx.wait_condition(options, || Ok(false)));
19060            let mut task_context = TaskContext::from_waker(noop_waker_ref());
19061            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
19062                wait.as_mut().poll(&mut task_context)
19063            else {
19064                panic!("changed condition definition must fail replay");
19065            };
19066            assert_eq!(failure.reason, expected_reason);
19067            assert_eq!(failure.sequence, Some(12));
19068        }
19069    }
19070
19071    #[test]
19072    fn condition_wait_history_requires_the_canonical_predicate_fingerprint() {
19073        let error = WorkflowState::new(
19074            vec![history_event(
19075                "ConditionWaitOpened",
19076                json!({
19077                    "sequence": 12,
19078                    "condition_wait_id": "condition:12",
19079                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19080                    "condition_key": "approval",
19081                }),
19082            )],
19083            "rust-workers".to_string(),
19084            DEFAULT_CODEC.to_string(),
19085            None,
19086        )
19087        .expect_err("condition history without a predicate fingerprint must fail");
19088
19089        assert!(matches!(
19090            error,
19091            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19092                if reason == "condition_wait_predicate_fingerprint_missing"
19093        ));
19094    }
19095
19096    #[test]
19097    fn condition_wait_history_requires_authored_occurrence_identity() {
19098        let error = WorkflowState::new(
19099            vec![history_event(
19100                "ConditionWaitOpened",
19101                json!({
19102                    "sequence": 12,
19103                    "condition_wait_id": "condition:12",
19104                    "condition_key": "approval",
19105                    "condition_definition_fingerprint": "sha256:approval-v1",
19106                }),
19107            )],
19108            "rust-workers".to_string(),
19109            DEFAULT_CODEC.to_string(),
19110            None,
19111        )
19112        .expect_err("condition history without occurrence identity must fail");
19113
19114        assert!(matches!(
19115            error,
19116            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19117                if reason == "condition_wait_occurrence_id_missing"
19118        ));
19119    }
19120
19121    #[test]
19122    fn typed_search_attribute_updates_validate_emit_and_replay() {
19123        let update = SearchAttributeUpdate::new()
19124            .keyword("OrderStatus", " waiting ")
19125            .expect("keyword")
19126            .int("Attempt", 3)
19127            .expect("int")
19128            .bool("Escalated", false)
19129            .expect("bool")
19130            .keyword_list("Regions", ["us-east", "eu-west"])
19131            .expect("list")
19132            .datetime("UpdatedAt", "2026-08-22T04:00:00Z")
19133            .expect("datetime")
19134            .delete("LegacyStatus")
19135            .expect("delete");
19136        let ctx = workflow_context(Vec::new());
19137        ctx.upsert_search_attributes(update.clone())
19138            .expect("typed update");
19139        assert_eq!(
19140            ctx.take_commands().expect("search-attribute command"),
19141            vec![json!({
19142                "type": "upsert_search_attributes",
19143                "attributes": {
19144                    "Attempt": 3,
19145                    "Escalated": false,
19146                    "LegacyStatus": null,
19147                    "OrderStatus": "waiting",
19148                    "Regions": ["us-east", "eu-west"],
19149                    "UpdatedAt": "2026-08-22T04:00:00Z",
19150                },
19151                "attribute_types": {
19152                    "Attempt": "int",
19153                    "Escalated": "bool",
19154                    "OrderStatus": "keyword",
19155                    "Regions": "keyword_list",
19156                    "UpdatedAt": "datetime",
19157                },
19158            })]
19159        );
19160
19161        let replay = workflow_context(vec![history_event(
19162            "SearchAttributesUpserted",
19163            json!({
19164                "sequence": 6,
19165                "attributes": {
19166                    "Attempt": 3,
19167                    "Escalated": false,
19168                    "LegacyStatus": null,
19169                    "OrderStatus": "waiting",
19170                    "Regions": ["us-east", "eu-west"],
19171                    "UpdatedAt": "2026-08-22T04:00:00Z",
19172                },
19173                "attribute_types": {
19174                    "Attempt": "int",
19175                    "Escalated": "bool",
19176                    "OrderStatus": "keyword",
19177                    "Regions": "keyword_list",
19178                    "UpdatedAt": "datetime",
19179                },
19180                "merged": {},
19181            }),
19182        )]);
19183        replay
19184            .upsert_search_attributes(update)
19185            .expect("matching update replays");
19186        assert!(replay.take_commands().expect("commands").is_empty());
19187        replay.ensure_history_consumed().expect("history consumed");
19188
19189        let type_drift = workflow_context(vec![history_event(
19190            "SearchAttributesUpserted",
19191            json!({
19192                "sequence": 7,
19193                "attributes": {"OrderStatus": "waiting"},
19194                "attribute_types": {"OrderStatus": "keyword"},
19195                "merged": {"OrderStatus": "waiting"},
19196            }),
19197        )]);
19198        let error = type_drift
19199            .upsert_search_attributes(
19200                SearchAttributeUpdate::new()
19201                    .string("OrderStatus", "waiting")
19202                    .expect("string update"),
19203            )
19204            .expect_err("same JSON value with a changed type must fail replay");
19205        assert!(matches!(
19206            error,
19207            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19208                if reason == "search_attribute_type_mismatch"
19209        ));
19210
19211        let malformed_types = WorkflowState::new(
19212            vec![history_event(
19213                "SearchAttributesUpserted",
19214                json!({
19215                    "sequence": 8,
19216                    "attributes": {"OrderStatus": "waiting"},
19217                    "attribute_types": {"OrderStatus": "unsupported"},
19218                    "merged": {"OrderStatus": "waiting"},
19219                }),
19220            )],
19221            "rust-workers".to_string(),
19222            DEFAULT_CODEC.to_string(),
19223            None,
19224        )
19225        .expect_err("unsupported search-attribute type metadata must fail");
19226        assert!(matches!(
19227            malformed_types,
19228            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19229                if reason == "search_attribute_types_malformed"
19230        ));
19231
19232        assert!(matches!(
19233            SearchAttributeUpdate::new().keyword("bad key", "value"),
19234            Err(SearchAttributeUpdateError::InvalidKey(_))
19235        ));
19236        assert!(matches!(
19237            SearchAttributeUpdate::new().float("Ratio", f64::NAN),
19238            Err(SearchAttributeUpdateError::NonFiniteFloat(_))
19239        ));
19240        assert!(matches!(
19241            SearchAttributeUpdate::new().keyword("UnicodeKeyword", "é".repeat(128)),
19242            Err(SearchAttributeUpdateError::ValueTooLong { .. })
19243        ));
19244        assert!(matches!(
19245            SearchAttributeUpdate::new().datetime("UpdatedAt", "2026-02-30T04:00:00Z"),
19246            Err(SearchAttributeUpdateError::InvalidDateTime(_))
19247        ));
19248        assert!(matches!(
19249            workflow_context(Vec::new()).upsert_search_attributes(SearchAttributeUpdate::new()),
19250            Err(Error::InvalidSearchAttributeUpdate(
19251                SearchAttributeUpdateError::Empty
19252            ))
19253        ));
19254    }
19255
19256    #[test]
19257    fn workflow_history_rejects_unpaired_or_mismatched_timer_events() {
19258        let lone_fire = WorkflowState::new(
19259            vec![history_event(
19260                "TimerFired",
19261                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19262            )],
19263            "rust-workers".to_string(),
19264            DEFAULT_CODEC.to_string(),
19265            None,
19266        )
19267        .expect_err("TimerFired requires TimerScheduled");
19268        assert!(matches!(
19269            lone_fire,
19270            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19271                if reason == "timer_schedule_missing_or_duplicate"
19272        ));
19273
19274        let wrong_identity = WorkflowState::new(
19275            vec![
19276                history_event(
19277                    "TimerScheduled",
19278                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19279                ),
19280                history_event(
19281                    "TimerFired",
19282                    json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
19283                ),
19284            ],
19285            "rust-workers".to_string(),
19286            DEFAULT_CODEC.to_string(),
19287            None,
19288        )
19289        .expect_err("fire must match scheduled timer identity");
19290        assert!(matches!(
19291            wrong_identity,
19292            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19293                if reason == "timer_identity_mismatch"
19294        ));
19295
19296        let duplicate_fire = WorkflowState::new(
19297            vec![
19298                history_event(
19299                    "TimerScheduled",
19300                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19301                ),
19302                history_event(
19303                    "TimerFired",
19304                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19305                ),
19306                history_event(
19307                    "TimerFired",
19308                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19309                ),
19310            ],
19311            "rust-workers".to_string(),
19312            DEFAULT_CODEC.to_string(),
19313            None,
19314        )
19315        .expect_err("a durable timer cannot fire twice");
19316        assert!(matches!(
19317            duplicate_fire,
19318            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19319                if reason == "duplicate_timer_fire"
19320        ));
19321
19322        let wrong_fired_delay = WorkflowState::new(
19323            vec![
19324                history_event(
19325                    "TimerScheduled",
19326                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19327                ),
19328                history_event(
19329                    "TimerFired",
19330                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 6}),
19331                ),
19332            ],
19333            "rust-workers".to_string(),
19334            DEFAULT_CODEC.to_string(),
19335            None,
19336        )
19337        .expect_err("timer schedule and fire delays must agree");
19338        assert!(matches!(
19339            wrong_fired_delay,
19340            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19341                if reason == "timer_history_delay_mismatch"
19342        ));
19343    }
19344
19345    #[test]
19346    fn replay_rejects_activity_moved_before_recorded_timer() {
19347        let ctx = workflow_context(vec![
19348            history_event(
19349                "TimerScheduled",
19350                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19351            ),
19352            history_event(
19353                "TimerFired",
19354                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19355            ),
19356            history_event(
19357                "ActivityCompleted",
19358                json!({
19359                    "sequence": 2,
19360                    "activity_type": "after-timer",
19361                    "payload_codec": DEFAULT_CODEC,
19362                    "result": fixture_envelope(json!("done")),
19363                }),
19364            ),
19365        ]);
19366        let mut activity = Box::pin(ctx.activity("after-timer", json!([])));
19367        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19368
19369        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
19370            activity.as_mut().poll(&mut task_context)
19371        else {
19372            panic!("reordered durable command must be rejected");
19373        };
19374        assert_eq!(failure.reason, "recorded_command_mismatch");
19375        assert_eq!(failure.sequence, Some(1));
19376        assert_eq!(failure.expected.as_deref(), Some("timer"));
19377        assert_eq!(failure.actual.as_deref(), Some("activity:after-timer"));
19378    }
19379
19380    #[test]
19381    fn workflow_context_emits_a_typed_named_signal_wait() {
19382        let ctx = workflow_context(Vec::new());
19383        let mut signal = Box::pin(ctx.wait_signal("finish"));
19384        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19385
19386        assert!(matches!(
19387            signal.as_mut().poll(&mut task_context),
19388            Poll::Pending
19389        ));
19390        assert_eq!(
19391            ctx.take_commands().expect("signal-wait command"),
19392            vec![json!({
19393                "type": "open_signal_wait",
19394                "signal_name": "finish",
19395            })]
19396        );
19397    }
19398
19399    #[test]
19400    fn runtime_message_stream_transport_cannot_be_opened_as_a_user_signal() {
19401        let ctx = workflow_context(Vec::new());
19402        let mut signal = Box::pin(ctx.wait_signal(MESSAGE_STREAM_SIGNAL));
19403        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19404
19405        let Poll::Ready(Err(Error::Codec(message))) = signal.as_mut().poll(&mut task_context)
19406        else {
19407            panic!("runtime-reserved signal should be rejected");
19408        };
19409        assert!(message.contains("reserved by the workflow runtime"));
19410        assert!(ctx.take_commands().expect("commands").is_empty());
19411    }
19412
19413    #[tokio::test]
19414    async fn runtime_message_stream_transport_cannot_be_sent_as_a_user_signal() {
19415        let client = Client::builder("http://127.0.0.1:9")
19416            .build()
19417            .expect("client");
19418        let error = client
19419            .signal_workflow("workflow-1", MESSAGE_STREAM_SIGNAL, json!(["forged"]))
19420            .await
19421            .expect_err("runtime-reserved signal should be rejected before transport");
19422
19423        assert!(
19424            matches!(error, Error::Codec(ref message) if message.contains("reserved by the workflow runtime"))
19425        );
19426    }
19427
19428    #[test]
19429    fn message_stream_worker_task_consumes_current_contiguous_bounded_batch() {
19430        fn delivery(message_id: &str, position: u64, value: &str) -> Value {
19431            let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
19432                value.to_string(),
19433            )]))
19434            .expect("message payload");
19435            json!({
19436                "schema": MESSAGE_STREAM_SCHEMA,
19437                "stream_name": "orders",
19438                "message_id": message_id,
19439                "position": position,
19440                "payload_envelope": payload,
19441            })
19442        }
19443
19444        fn opened(sequence: u64) -> HistoryEvent {
19445            history_event(
19446                "SignalWaitOpened",
19447                json!({
19448                    "sequence": sequence,
19449                    "signal_name": MESSAGE_STREAM_SIGNAL,
19450                }),
19451            )
19452        }
19453
19454        fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
19455            history_event(
19456                "SignalApplied",
19457                json!({
19458                    "sequence": sequence,
19459                    "signal_name": MESSAGE_STREAM_SIGNAL,
19460                    "value": fixture_envelope(json!([delivery])),
19461                }),
19462            )
19463        }
19464
19465        fn received(delivery: Value) -> HistoryEvent {
19466            history_event(
19467                "SignalReceived",
19468                json!({
19469                    "signal_name": MESSAGE_STREAM_SIGNAL,
19470                    "arguments": fixture_envelope(json!([delivery])),
19471                    "payload_codec": DEFAULT_CODEC,
19472                }),
19473            )
19474        }
19475
19476        let client = Client::new("http://127.0.0.1:8080").expect("client");
19477        let mut worker = Worker::new(client, "rust-workers");
19478        worker.register_workflow("rust.message-stream-batch", |ctx, _input| async move {
19479            let messages = ctx.message_stream("orders")?.receive(2).await?;
19480            Ok(json!(messages
19481                .into_iter()
19482                .map(|message| message.message_id)
19483                .collect::<Vec<_>>()))
19484        });
19485
19486        let first = delivery("message-1", 1, "one");
19487        let second = delivery("message-2", 2, "two");
19488        let batch = worker
19489            .execute_workflow_task_decision(workflow_task(
19490                "rust.message-stream-batch",
19491                vec![
19492                    opened(1),
19493                    received(first.clone()),
19494                    applied(1, first.clone()),
19495                    received(first.clone()),
19496                    received(second),
19497                ],
19498                DEFAULT_CODEC,
19499            ))
19500            .expect("worker task consumes the available batch");
19501
19502        assert_eq!(batch.commands.len(), 1);
19503        assert_eq!(batch.commands[0]["type"], "complete_workflow");
19504        assert_eq!(
19505            decode_wire_value(&batch.commands[0]["result"], DEFAULT_CODEC)
19506                .expect("workflow result"),
19507            json!(["message-1", "message-2"])
19508        );
19509        assert_eq!(
19510            batch.message_stream_cursors,
19511            vec![json!({"stream_name": "orders", "through_position": 2})]
19512        );
19513        assert!(batch.message_stream_waits.is_empty());
19514
19515        let partial = worker
19516            .execute_workflow_task_decision(workflow_task(
19517                "rust.message-stream-batch",
19518                vec![opened(1), received(first.clone()), applied(1, first)],
19519                DEFAULT_CODEC,
19520            ))
19521            .expect("worker task returns without waiting for a missing second item");
19522        assert_eq!(partial.commands.len(), 1);
19523        assert_eq!(partial.commands[0]["type"], "complete_workflow");
19524        assert_eq!(
19525            decode_wire_value(&partial.commands[0]["result"], DEFAULT_CODEC)
19526                .expect("workflow result"),
19527            json!(["message-1"])
19528        );
19529        assert_eq!(
19530            partial.message_stream_cursors,
19531            vec![json!({"stream_name": "orders", "through_position": 1})]
19532        );
19533        assert!(partial.message_stream_waits.is_empty());
19534    }
19535
19536    #[test]
19537    fn message_stream_replay_preserves_partial_batch_boundary_before_later_wait() {
19538        fn delivery(message_id: &str, position: u64, value: &str) -> Value {
19539            let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
19540                value.to_string(),
19541            )]))
19542            .expect("message payload");
19543            json!({
19544                "schema": MESSAGE_STREAM_SCHEMA,
19545                "stream_name": "orders",
19546                "message_id": message_id,
19547                "position": position,
19548                "payload_envelope": payload,
19549            })
19550        }
19551
19552        fn opened(sequence: u64) -> HistoryEvent {
19553            history_event(
19554                "SignalWaitOpened",
19555                json!({
19556                    "sequence": sequence,
19557                    "signal_name": MESSAGE_STREAM_SIGNAL,
19558                }),
19559            )
19560        }
19561
19562        fn received(delivery: Value) -> HistoryEvent {
19563            history_event(
19564                "SignalReceived",
19565                json!({
19566                    "signal_name": MESSAGE_STREAM_SIGNAL,
19567                    "arguments": fixture_envelope(json!([delivery])),
19568                    "payload_codec": DEFAULT_CODEC,
19569                }),
19570            )
19571        }
19572
19573        fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
19574            history_event(
19575                "SignalApplied",
19576                json!({
19577                    "sequence": sequence,
19578                    "signal_name": MESSAGE_STREAM_SIGNAL,
19579                    "value": fixture_envelope(json!([delivery])),
19580                }),
19581            )
19582        }
19583
19584        let client = Client::new("http://127.0.0.1:8080").expect("client");
19585        let mut worker = Worker::new(client, "rust-workers");
19586        worker.register_workflow(
19587            "rust.message-stream-partial-batches",
19588            |ctx, _input| async move {
19589                let stream = ctx.message_stream("orders")?;
19590                let first = stream.receive(10).await?;
19591                let second = stream.receive(10).await?;
19592                Ok(json!([
19593                    first
19594                        .into_iter()
19595                        .map(|message| message.message_id)
19596                        .collect::<Vec<_>>(),
19597                    second
19598                        .into_iter()
19599                        .map(|message| message.message_id)
19600                        .collect::<Vec<_>>(),
19601                ]))
19602            },
19603        );
19604
19605        let first = delivery("message-1", 1, "one");
19606        let second = delivery("message-2", 2, "two");
19607        let decision = worker
19608            .execute_workflow_task_decision(workflow_task(
19609                "rust.message-stream-partial-batches",
19610                vec![
19611                    opened(1),
19612                    received(first.clone()),
19613                    applied(1, first),
19614                    opened(2),
19615                    received(second.clone()),
19616                    applied(2, second),
19617                ],
19618                DEFAULT_CODEC,
19619            ))
19620            .expect("cold replay preserves both authored receive boundaries");
19621
19622        assert_eq!(decision.commands.len(), 1);
19623        assert_eq!(decision.commands[0]["type"], "complete_workflow");
19624        assert_eq!(
19625            decode_wire_value(&decision.commands[0]["result"], DEFAULT_CODEC)
19626                .expect("workflow result"),
19627            json!([["message-1"], ["message-2"]])
19628        );
19629        assert_eq!(
19630            decision.message_stream_cursors,
19631            vec![json!({"stream_name": "orders", "through_position": 2})]
19632        );
19633        assert!(decision.message_stream_waits.is_empty());
19634    }
19635
19636    #[test]
19637    fn empty_message_stream_opens_internal_signal_wait_and_reports_position() {
19638        let ctx = workflow_context(Vec::new());
19639        let stream = ctx.message_stream("orders").expect("message stream");
19640        let mut receive = Box::pin(stream.receive(10));
19641        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19642
19643        assert!(matches!(
19644            receive.as_mut().poll(&mut task_context),
19645            Poll::Pending
19646        ));
19647        assert_eq!(
19648            ctx.take_commands().expect("message-stream wait command"),
19649            vec![json!({
19650                "type": "open_signal_wait",
19651                "signal_name": MESSAGE_STREAM_SIGNAL,
19652            })]
19653        );
19654        let (cursors, waits) = ctx.message_stream_metadata().expect("stream metadata");
19655        assert!(cursors.is_empty());
19656        assert_eq!(
19657            waits,
19658            vec![json!({"stream_name": "orders", "after_position": 0})]
19659        );
19660    }
19661
19662    #[test]
19663    fn continue_as_new_cursor_checkpoint_preserves_global_pending_position() {
19664        let ctx = workflow_context(vec![history_event(
19665            "SignalReceived",
19666            json!({
19667                "signal_name": MESSAGE_STREAM_SIGNAL,
19668                "arguments": fixture_envelope(json!([{
19669                    "schema": MESSAGE_STREAM_CURSOR_SCHEMA,
19670                    "stream_name": "orders",
19671                    "through_position": 2,
19672                }])),
19673                "payload_codec": DEFAULT_CODEC,
19674            }),
19675        )]);
19676        let stream = ctx.message_stream("orders").expect("message stream");
19677        let mut receive = Box::pin(stream.receive(10));
19678        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19679
19680        assert!(matches!(
19681            receive.as_mut().poll(&mut task_context),
19682            Poll::Pending
19683        ));
19684        let (cursors, waits) = ctx.message_stream_metadata().expect("stream metadata");
19685        assert_eq!(
19686            cursors,
19687            vec![json!({"stream_name": "orders", "through_position": 2})]
19688        );
19689        assert_eq!(
19690            waits,
19691            vec![json!({"stream_name": "orders", "after_position": 2})]
19692        );
19693    }
19694
19695    #[test]
19696    fn message_stream_delivery_preserves_typed_avro_arguments_across_replay() {
19697        let mut empty_map = BTreeMap::new();
19698        let mut nested = BTreeMap::new();
19699        nested.insert(
19700            "value".to_string(),
19701            AvroValue::Array(vec![AvroValue::Bytes(b"nested".to_vec())]),
19702        );
19703        let values = vec![
19704            AvroValue::Bytes(vec![0, 255]),
19705            AvroValue::Long(1),
19706            AvroValue::Double(1.0),
19707            AvroValue::Array(Vec::new()),
19708            AvroValue::Map(std::mem::take(&mut empty_map)),
19709            AvroValue::Map(nested),
19710        ];
19711        let payload = encode_avro_value(&AvroValue::Array(values.clone())).expect("payload");
19712        let transport = vec![json!({
19713            "schema": MESSAGE_STREAM_SCHEMA,
19714            "stream_name": "orders",
19715            "message_id": "message-1",
19716            "position": 1,
19717            "payload_envelope": payload,
19718        })];
19719
19720        for _ in 0..2 {
19721            let Some(MessageStreamDelivery::Message(message)) =
19722                decode_message_stream_delivery(transport.clone()).expect("delivery")
19723            else {
19724                panic!("message delivery expected");
19725            };
19726            assert_eq!(message.arguments, values);
19727            assert!(matches!(message.arguments[1], AvroValue::Long(1)));
19728            assert!(matches!(message.arguments[2], AvroValue::Double(1.0)));
19729        }
19730    }
19731
19732    #[test]
19733    fn cold_worker_replacement_consumes_message_stream_wait_arrivals_once_in_order() {
19734        fn delivery(message_id: &str, position: u64, value: &str) -> Value {
19735            let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
19736                value.to_string(),
19737            )]))
19738            .expect("message payload");
19739            json!({
19740                "schema": MESSAGE_STREAM_SCHEMA,
19741                "stream_name": "orders",
19742                "message_id": message_id,
19743                "position": position,
19744                "payload_envelope": payload,
19745            })
19746        }
19747
19748        fn opened(sequence: u64) -> HistoryEvent {
19749            history_event(
19750                "SignalWaitOpened",
19751                json!({
19752                    "sequence": sequence,
19753                    "signal_name": MESSAGE_STREAM_SIGNAL,
19754                }),
19755            )
19756        }
19757
19758        fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
19759            history_event(
19760                "SignalApplied",
19761                json!({
19762                    "sequence": sequence,
19763                    "signal_name": MESSAGE_STREAM_SIGNAL,
19764                    "value": fixture_envelope(json!([delivery])),
19765                }),
19766            )
19767        }
19768
19769        fn worker() -> Worker {
19770            let client = Client::new("http://127.0.0.1:8080").expect("client");
19771            let mut worker = Worker::new(client, "rust-workers");
19772            worker.register_workflow("rust.message-stream", |ctx, _input| async move {
19773                let stream = ctx.message_stream("orders")?;
19774                let first = stream.receive_one().await?;
19775                let second = stream.receive_one().await?;
19776                Ok(json!([first.message_id, second.message_id]))
19777            });
19778            worker
19779        }
19780
19781        fn task_with_resume(history: Vec<HistoryEvent>, delivery: Value) -> WorkflowTask {
19782            let mut task = workflow_task("rust.message-stream", history, DEFAULT_CODEC);
19783            task.signal_name = Some(MESSAGE_STREAM_SIGNAL.to_string());
19784            task.signal_arguments = Some(fixture_envelope(json!([delivery])));
19785            task
19786        }
19787
19788        let waiting = worker()
19789            .execute_workflow_task_decision(workflow_task(
19790                "rust.message-stream",
19791                Vec::new(),
19792                DEFAULT_CODEC,
19793            ))
19794            .expect("first worker opens the stream wait");
19795        assert_eq!(
19796            waiting.commands,
19797            vec![json!({
19798                "type": "open_signal_wait",
19799                "signal_name": MESSAGE_STREAM_SIGNAL,
19800            })]
19801        );
19802        assert!(waiting.message_stream_cursors.is_empty());
19803        assert_eq!(
19804            waiting.message_stream_waits,
19805            vec![json!({"stream_name": "orders", "after_position": 0})]
19806        );
19807
19808        let first_delivery = delivery("message-1", 1, "one");
19809        let first_arrival = worker()
19810            .execute_workflow_task_decision(task_with_resume(
19811                vec![opened(1)],
19812                first_delivery.clone(),
19813            ))
19814            .expect("replacement worker consumes the first arrival");
19815        assert_eq!(
19816            first_arrival.commands,
19817            vec![json!({
19818                "type": "open_signal_wait",
19819                "signal_name": MESSAGE_STREAM_SIGNAL,
19820            })]
19821        );
19822        assert_eq!(
19823            first_arrival.message_stream_cursors,
19824            vec![json!({"stream_name": "orders", "through_position": 1})]
19825        );
19826        assert_eq!(
19827            first_arrival.message_stream_waits,
19828            vec![json!({"stream_name": "orders", "after_position": 1})]
19829        );
19830
19831        let second_delivery = delivery("message-2", 2, "two");
19832        let first_applied = applied(1, first_delivery);
19833        let completed = worker()
19834            .execute_workflow_task_decision(task_with_resume(
19835                vec![opened(1), first_applied.clone(), opened(2)],
19836                second_delivery.clone(),
19837            ))
19838            .expect("next replacement worker consumes the second arrival");
19839        assert_eq!(completed.commands.len(), 1);
19840        assert_eq!(completed.commands[0]["type"], "complete_workflow");
19841        assert_eq!(
19842            decode_wire_value(&completed.commands[0]["result"], DEFAULT_CODEC)
19843                .expect("workflow result"),
19844            json!(["message-1", "message-2"])
19845        );
19846        assert_eq!(
19847            completed.message_stream_cursors,
19848            vec![json!({"stream_name": "orders", "through_position": 2})]
19849        );
19850        assert!(completed.message_stream_waits.is_empty());
19851
19852        let replay_history = vec![
19853            opened(1),
19854            first_applied,
19855            opened(2),
19856            applied(2, second_delivery),
19857        ];
19858        for _cold_worker_or_restart in 0..2 {
19859            let replayed = worker()
19860                .execute_workflow_task_decision(workflow_task(
19861                    "rust.message-stream",
19862                    replay_history.clone(),
19863                    DEFAULT_CODEC,
19864                ))
19865                .expect("cold worker replays each logical message exactly once");
19866            assert_eq!(replayed.commands.len(), 1);
19867            assert_eq!(
19868                decode_wire_value(&replayed.commands[0]["result"], DEFAULT_CODEC)
19869                    .expect("replayed workflow result"),
19870                json!(["message-1", "message-2"])
19871            );
19872            assert_eq!(
19873                replayed.message_stream_cursors,
19874                vec![json!({"stream_name": "orders", "through_position": 2})]
19875            );
19876            assert!(replayed.message_stream_waits.is_empty());
19877        }
19878    }
19879
19880    #[test]
19881    fn message_stream_capability_and_completion_require_protocol_one_fifteen() {
19882        assert!(!worker_protocol_supports_message_streams("1.14"));
19883        assert!(worker_protocol_supports_message_streams("1.15"));
19884        assert!(worker_protocol_supports_message_streams("1.16"));
19885        assert!(worker_protocol_supports_message_streams(
19886            WORKER_PROTOCOL_VERSION
19887        ));
19888        assert_eq!(MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION, "1.15");
19889    }
19890
19891    #[test]
19892    fn condition_wait_history_cannot_be_consumed_as_a_typed_signal_wait() {
19893        let ctx = workflow_context(vec![
19894            history_event(
19895                "ConditionWaitOpened",
19896                json!({
19897                    "sequence": 1,
19898                    "condition_wait_id": "condition:1",
19899                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19900                    "condition_key": "signal:finish",
19901                    "condition_definition_fingerprint": "sha256:signal-finish-v1",
19902                }),
19903            ),
19904            history_event(
19905                "ConditionWaitSatisfied",
19906                json!({
19907                    "sequence": 1,
19908                    "condition_wait_id": "condition:1",
19909                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19910                    "condition_key": "signal:finish",
19911                    "condition_definition_fingerprint": "sha256:signal-finish-v1",
19912                }),
19913            ),
19914            history_event(
19915                "SignalReceived",
19916                json!({"signal_name": "finish", "arguments": []}),
19917            ),
19918        ]);
19919        let mut signal = Box::pin(ctx.wait_signal("finish"));
19920        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19921
19922        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
19923            signal.as_mut().poll(&mut task_context)
19924        else {
19925            panic!("condition history must not resolve as a typed signal wait");
19926        };
19927        assert_eq!(failure.reason, "recorded_command_mismatch");
19928        assert_eq!(failure.expected.as_deref(), Some("condition wait"));
19929    }
19930
19931    #[test]
19932    fn replay_orders_signal_waits_and_timers_in_one_command_stream() {
19933        let signal_then_timer = vec![
19934            history_event(
19935                "SignalWaitOpened",
19936                json!({"sequence": 1, "signal_name": "go"}),
19937            ),
19938            history_event(
19939                "SignalApplied",
19940                json!({
19941                    "sequence": 1,
19942                    "signal_name": "go",
19943                    "value": fixture_envelope(json!(["now"])),
19944                }),
19945            ),
19946            history_event(
19947                "TimerScheduled",
19948                json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
19949            ),
19950            history_event(
19951                "TimerFired",
19952                json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
19953            ),
19954        ];
19955
19956        let ctx = workflow_context(signal_then_timer.clone());
19957        let mut signal = Box::pin(ctx.wait_signal("go"));
19958        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19959        assert!(matches!(
19960            signal.as_mut().poll(&mut task_context),
19961            Poll::Ready(Ok(arguments)) if arguments == vec![json!("now")]
19962        ));
19963        let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5)));
19964        assert!(matches!(
19965            timer.as_mut().poll(&mut task_context),
19966            Poll::Ready(Ok(()))
19967        ));
19968        ctx.ensure_history_consumed()
19969            .expect("signal and timer history consumed in order");
19970
19971        let reordered = workflow_context(signal_then_timer);
19972        let mut timer_first = Box::pin(reordered.sleep(Duration::from_secs(5)));
19973        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
19974            timer_first.as_mut().poll(&mut task_context)
19975        else {
19976            panic!("timer cannot consume signal-wait-first history");
19977        };
19978        assert_eq!(failure.reason, "recorded_command_mismatch");
19979        assert_eq!(failure.sequence, Some(1));
19980        assert_eq!(failure.expected.as_deref(), Some("signal wait"));
19981
19982        let timer_then_signal = vec![
19983            history_event(
19984                "TimerScheduled",
19985                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19986            ),
19987            history_event(
19988                "TimerFired",
19989                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19990            ),
19991            history_event(
19992                "SignalWaitOpened",
19993                json!({"sequence": 2, "signal_name": "go"}),
19994            ),
19995            history_event(
19996                "SignalApplied",
19997                json!({
19998                    "sequence": 2,
19999                    "signal_name": "go",
20000                    "value": fixture_envelope(json!([])),
20001                }),
20002            ),
20003        ];
20004        let reordered = workflow_context(timer_then_signal);
20005        let mut signal_first = Box::pin(reordered.wait_signal("go"));
20006        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
20007            signal_first.as_mut().poll(&mut task_context)
20008        else {
20009            panic!("signal wait cannot consume timer-first history");
20010        };
20011        assert_eq!(failure.reason, "recorded_command_mismatch");
20012        assert_eq!(failure.sequence, Some(1));
20013        assert_eq!(failure.expected.as_deref(), Some("timer"));
20014    }
20015
20016    #[test]
20017    fn workflow_history_rejects_duplicate_or_colliding_command_sequences() {
20018        let duplicate_timer = WorkflowState::new(
20019            vec![
20020                history_event(
20021                    "TimerScheduled",
20022                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20023                ),
20024                history_event(
20025                    "TimerScheduled",
20026                    json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
20027                ),
20028            ],
20029            "rust-workers".to_string(),
20030            DEFAULT_CODEC.to_string(),
20031            None,
20032        )
20033        .expect_err("one workflow sequence cannot schedule two timers");
20034        assert!(matches!(
20035            duplicate_timer,
20036            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20037                if reason == "timer_schedule_missing_or_duplicate"
20038        ));
20039
20040        let colliding_kinds = WorkflowState::new(
20041            vec![
20042                history_event(
20043                    "TimerScheduled",
20044                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20045                ),
20046                history_event(
20047                    "ActivityCompleted",
20048                    json!({"sequence": 1, "activity_type": "same-sequence"}),
20049                ),
20050            ],
20051            "rust-workers".to_string(),
20052            DEFAULT_CODEC.to_string(),
20053            None,
20054        )
20055        .expect_err("one workflow sequence cannot identify two command kinds");
20056        assert!(matches!(
20057            colliding_kinds,
20058            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20059                if reason == "durable_command_sequence_collision"
20060        ));
20061
20062        let duplicate_signal_wait = WorkflowState::new(
20063            vec![
20064                history_event(
20065                    "SignalWaitOpened",
20066                    json!({"sequence": 1, "signal_name": "go"}),
20067                ),
20068                history_event(
20069                    "SignalWaitOpened",
20070                    json!({"sequence": 1, "signal_name": "go"}),
20071                ),
20072            ],
20073            "rust-workers".to_string(),
20074            DEFAULT_CODEC.to_string(),
20075            None,
20076        )
20077        .expect_err("one workflow sequence cannot open two signal waits");
20078        assert!(matches!(
20079            duplicate_signal_wait,
20080            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20081                if reason == "signal_wait_open_missing_or_duplicate"
20082        ));
20083    }
20084
20085    #[test]
20086    fn workflow_history_accepts_a_first_command_after_global_sequence_gaps() {
20087        let result = encode_value_envelope(&json!({"captured": true}), DEFAULT_CODEC)
20088            .expect("side-effect result");
20089        let ctx = workflow_context(vec![history_event(
20090            "SideEffectRecorded",
20091            json!({"sequence": 99, "result": result}),
20092        )]);
20093
20094        let replayed: Value = ctx
20095            .side_effect(|| panic!("recorded side effect must not run"))
20096            .expect("positive global workflow sequence is valid");
20097        assert_eq!(replayed, json!({"captured": true}));
20098        ctx.ensure_history_consumed().expect("history consumed");
20099    }
20100
20101    #[test]
20102    fn workflow_history_rejects_zero_and_descending_command_sequences() {
20103        let result =
20104            encode_value_envelope(&json!("captured"), DEFAULT_CODEC).expect("side-effect result");
20105        let zero = WorkflowState::new(
20106            vec![history_event(
20107                "SideEffectRecorded",
20108                json!({"sequence": 0, "result": result.clone()}),
20109            )],
20110            "rust-workers".to_string(),
20111            DEFAULT_CODEC.to_string(),
20112            None,
20113        )
20114        .expect_err("durable command sequences must be positive");
20115        assert!(matches!(
20116            zero,
20117            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20118                if reason == "durable_command_sequence_invalid"
20119        ));
20120
20121        let descending = WorkflowState::new(
20122            vec![
20123                history_event(
20124                    "SideEffectRecorded",
20125                    json!({"sequence": 3, "result": result}),
20126                ),
20127                history_event(
20128                    "VersionMarkerRecorded",
20129                    json!({
20130                        "sequence": 2,
20131                        "change_id": "descending-marker",
20132                        "version": 1,
20133                        "min_supported": 1,
20134                        "max_supported": 1,
20135                    }),
20136                ),
20137            ],
20138            "rust-workers".to_string(),
20139            DEFAULT_CODEC.to_string(),
20140            None,
20141        )
20142        .expect_err("new durable commands must remain strictly ordered");
20143        let Error::NonDeterministicReplay(failure) = descending else {
20144            panic!("expected typed replay failure");
20145        };
20146        assert_eq!(failure.reason, "durable_command_sequence_mismatch");
20147        assert_eq!(failure.sequence, Some(2));
20148        assert_eq!(
20149            failure.expected.as_deref(),
20150            Some("workflow sequence greater than 3")
20151        );
20152        assert_eq!(failure.actual.as_deref(), Some("2"));
20153    }
20154
20155    #[test]
20156    fn workflow_task_replay_completes_after_signals_create_sequence_gaps() {
20157        fn worker() -> Worker {
20158            let client = Client::new("http://127.0.0.1:8080").expect("client");
20159            let mut worker = Worker::new(client, "rust-workers");
20160            worker.register_workflow("rust.finish-after-gaps", |ctx, _input| async move {
20161                ctx.wait_signal("finish").await?;
20162                let marker: String =
20163                    ctx.side_effect(|| panic!("recorded side effect must not run"))?;
20164                assert_eq!(marker, "after-finish");
20165                Ok(json!("finished"))
20166            });
20167            worker
20168        }
20169
20170        let marker = encode_value_envelope(&json!("after-finish"), DEFAULT_CODEC)
20171            .expect("side-effect result");
20172        let task = workflow_task(
20173            "rust.finish-after-gaps",
20174            vec![
20175                history_event(
20176                    "SignalWaitOpened",
20177                    json!({"sequence": 1, "signal_name": "finish"}),
20178                ),
20179                history_event(
20180                    "SignalReceived",
20181                    json!({
20182                        "signal_id": "increment-3",
20183                        "signal_name": "increment",
20184                        "workflow_sequence": 2,
20185                        "payload_codec": DEFAULT_CODEC,
20186                        "arguments": fixture_envelope(json!([3])),
20187                    }),
20188                ),
20189                history_event(
20190                    "SignalReceived",
20191                    json!({
20192                        "signal_id": "increment-5",
20193                        "signal_name": "increment",
20194                        "workflow_sequence": 3,
20195                        "payload_codec": DEFAULT_CODEC,
20196                        "arguments": fixture_envelope(json!([5])),
20197                    }),
20198                ),
20199                history_event(
20200                    "SignalReceived",
20201                    json!({
20202                        "signal_id": "finish",
20203                        "signal_name": "finish",
20204                        "workflow_sequence": 4,
20205                        "payload_codec": DEFAULT_CODEC,
20206                        "arguments": fixture_envelope(json!([])),
20207                    }),
20208                ),
20209                history_event(
20210                    "SignalApplied",
20211                    json!({
20212                        "sequence": 1,
20213                        "signal_id": "finish",
20214                        "signal_name": "finish",
20215                        "payload_codec": DEFAULT_CODEC,
20216                        "value": fixture_envelope(json!([])),
20217                    }),
20218                ),
20219                history_event(
20220                    "SideEffectRecorded",
20221                    json!({"sequence": 5, "result": marker}),
20222                ),
20223            ],
20224            DEFAULT_CODEC,
20225        );
20226
20227        for _original_or_cold_worker in 0..2 {
20228            let commands = worker()
20229                .execute_workflow_task(task.clone())
20230                .expect("signal gaps preserve deterministic replay");
20231            assert_eq!(commands.len(), 1, "replay emits only terminal completion");
20232            assert_eq!(commands[0]["type"], "complete_workflow");
20233            assert_eq!(
20234                decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
20235                json!("finished")
20236            );
20237        }
20238    }
20239
20240    #[test]
20241    fn workflow_sleep_rejects_unrepresentable_rounded_duration() {
20242        let ctx = workflow_context(Vec::new());
20243        let mut sleep = Box::pin(ctx.start_timer(Duration::new(u64::MAX, 1)));
20244        let mut task_context = TaskContext::from_waker(noop_waker_ref());
20245        assert!(matches!(
20246            sleep.as_mut().poll(&mut task_context),
20247            Poll::Ready(Err(Error::TimerDurationOverflow))
20248        ));
20249        assert!(ctx.take_commands().expect("commands").is_empty());
20250    }
20251
20252    #[test]
20253    fn workflow_memo_update_emits_canonical_command_and_replays_once() {
20254        let entries = AvroValue::Map(BTreeMap::from([
20255            ("text".to_string(), AvroValue::String("same".to_string())),
20256            (
20257                "nested".to_string(),
20258                AvroValue::Map(BTreeMap::from([
20259                    ("beta".to_string(), AvroValue::Long(2)),
20260                    ("alpha".to_string(), AvroValue::Long(1)),
20261                ])),
20262            ),
20263            ("long".to_string(), AvroValue::Long(7)),
20264            ("double".to_string(), AvroValue::Double(7.0)),
20265            ("binary".to_string(), AvroValue::Bytes(b"same".to_vec())),
20266        ]));
20267        let ctx = workflow_context(Vec::new());
20268        ctx.upsert_memo(entries.clone()).expect("valid memo update");
20269        let commands = ctx.take_commands().expect("commands");
20270
20271        assert_eq!(commands.len(), 1);
20272        assert_eq!(commands[0]["type"], "upsert_memo");
20273        let server_entries = json!({
20274            "codec": "avro",
20275            "blob": "wwHioz3/VYAiNw4KDGJpbmFyeQgIc2FtZQxkb3VibGUGAAAAAAAAHEAIbG9uZwQODG5lc3RlZA4ECmFscGhhBAIIYmV0YQQEAAh0ZXh0CghzYW1lAA==",
20276        });
20277        assert_eq!(
20278            commands[0]["entries"]
20279                .as_object()
20280                .expect("entries envelope")
20281                .keys()
20282                .collect::<Vec<_>>(),
20283            vec!["blob", "codec"]
20284        );
20285        assert_eq!(commands[0]["entries"], server_entries);
20286        let wire_entries =
20287            decode_wire_avro_value(&commands[0]["entries"], DEFAULT_CODEC).expect("memo entries");
20288        assert_eq!(wire_entries, entries);
20289
20290        let history = vec![history_event(
20291            "MemoUpserted",
20292            json!({
20293                "sequence": 1,
20294                "entries": server_entries.clone(),
20295                "merged": server_entries,
20296            }),
20297        )];
20298        let replay = workflow_context(history.clone());
20299        replay
20300            .upsert_memo(entries.clone())
20301            .expect("matching replay identity");
20302        assert!(replay.take_commands().expect("replay commands").is_empty());
20303
20304        let changed_types = AvroValue::Map(BTreeMap::from([
20305            ("text".to_string(), AvroValue::Bytes(b"same".to_vec())),
20306            (
20307                "nested".to_string(),
20308                AvroValue::Map(BTreeMap::from([
20309                    ("alpha".to_string(), AvroValue::Long(1)),
20310                    ("beta".to_string(), AvroValue::Long(2)),
20311                ])),
20312            ),
20313            ("long".to_string(), AvroValue::Double(7.0)),
20314            ("double".to_string(), AvroValue::Long(7)),
20315            ("binary".to_string(), AvroValue::String("same".to_string())),
20316        ]));
20317        let error = workflow_context(history)
20318            .upsert_memo(changed_types)
20319            .expect_err("memo replay identity must preserve Avro value types");
20320        assert!(matches!(
20321            error,
20322            Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20323        ));
20324    }
20325
20326    #[test]
20327    fn workflow_memo_update_rejects_changed_replay_identity_and_invalid_keys() {
20328        let original = encode_value_envelope(&json!({"stage": "original"}), DEFAULT_CODEC)
20329            .expect("memo envelope");
20330        let replay = workflow_context(vec![history_event(
20331            "MemoUpserted",
20332            json!({
20333                "sequence": 1,
20334                "entries": original.clone(),
20335                "merged": original
20336            }),
20337        )]);
20338        let error = replay
20339            .upsert_memo(json!({"stage": "changed"}))
20340            .expect_err("changed memo update must fail replay");
20341        assert!(matches!(
20342            error,
20343            Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20344        ));
20345
20346        let invalid = workflow_context(Vec::new())
20347            .upsert_memo(
20348                json!({"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx": true}),
20349            )
20350            .expect_err("oversized key");
20351        assert!(matches!(invalid, Error::InvalidMemoUpdate(_)));
20352    }
20353
20354    #[test]
20355    fn workflow_memo_replay_distinguishes_signed_zero_identity() {
20356        let negative_zero = AvroValue::Map(BTreeMap::from([(
20357            "reading".to_string(),
20358            AvroValue::Double(-0.0),
20359        )]));
20360        let negative_zero_envelope =
20361            encode_typed_envelope(&negative_zero, DEFAULT_CODEC).expect("negative zero envelope");
20362        let history = vec![history_event(
20363            "MemoUpserted",
20364            json!({
20365                "sequence": 1,
20366                "entries": negative_zero_envelope.clone(),
20367                "merged": negative_zero_envelope,
20368            }),
20369        )];
20370
20371        workflow_context(history.clone())
20372            .upsert_memo(negative_zero)
20373            .expect("matching negative-zero history identity");
20374
20375        let error = workflow_context(history)
20376            .upsert_memo(AvroValue::Map(BTreeMap::from([(
20377                "reading".to_string(),
20378                AvroValue::Double(0.0),
20379            )])))
20380            .expect_err("positive zero must not consume negative-zero memo history");
20381        assert!(matches!(
20382            error,
20383            Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20384        ));
20385    }
20386
20387    #[test]
20388    fn workflow_memo_capability_requires_flag_and_command_advertisement() {
20389        let supported = json!({
20390            "workflow_memo_updates": {"supported": true, "minimum_protocol_version": "1.14"},
20391            "supported_workflow_task_commands": ["complete_workflow", "upsert_memo"]
20392        });
20393        assert!(runtime_supports_workflow_memo_updates(Some(&supported)));
20394        assert!(!runtime_supports_workflow_memo_updates(Some(&json!({
20395            "workflow_memo_updates": {"supported": false},
20396            "supported_workflow_task_commands": ["upsert_memo"]
20397        }))));
20398        assert!(commands_use_workflow_memo_updates(&[json!({
20399            "type": "upsert_memo",
20400            "entries": {"stage": "processing"}
20401        })]));
20402    }
20403
20404    #[test]
20405    fn workflow_task_replay_completes_without_rescheduling_recorded_commands() {
20406        let client = Client::new("http://127.0.0.1:8080").expect("client");
20407        let mut worker = Worker::new(client, "rust-workers");
20408        worker.register_workflow("rust.timer", |ctx, _input| async move {
20409            ctx.sleep(Duration::from_secs(5)).await?;
20410            ctx.activity("after-timer", json!([])).await
20411        });
20412
20413        let task = |history_events| WorkflowTask {
20414            task_id: "wft-rust-timer-1".to_string(),
20415            workflow_command_id: None,
20416            workflow_id: Some("wf-rust-timer".to_string()),
20417            run_id: Some("run-rust-timer".to_string()),
20418            workflow_type: "rust.timer".to_string(),
20419            cancel_requested: false,
20420            payload_codec: DEFAULT_CODEC.to_string(),
20421            arguments: Some(
20422                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
20423            ),
20424            history_events,
20425            total_history_events: None,
20426            history_size_bytes: None,
20427            continue_as_new_recommended: None,
20428            history_budget_pressure: None,
20429            next_history_page_token: None,
20430            workflow_task_attempt: 1,
20431            workflow_signal_id: None,
20432            signal_name: None,
20433            signal_arguments: None,
20434            workflow_update_id: None,
20435            update_name: None,
20436            lease_owner: Some("rust-worker".to_string()),
20437        };
20438
20439        let initial = worker
20440            .execute_workflow_task(task(Vec::new()))
20441            .expect("initial timer task");
20442        assert_eq!(
20443            initial,
20444            vec![json!({"type": "start_timer", "delay_seconds": 5})]
20445        );
20446
20447        let activity_result =
20448            encode_value_envelope(&json!("done"), DEFAULT_CODEC).expect("activity result");
20449        let replayed = worker
20450            .execute_workflow_task(task(vec![
20451                history_event(
20452                    "TimerScheduled",
20453                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20454                ),
20455                history_event(
20456                    "TimerFired",
20457                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20458                ),
20459                history_event(
20460                    "ActivityCompleted",
20461                    json!({
20462                        "sequence": 2,
20463                        "activity_type": "after-timer",
20464                        "payload_codec": DEFAULT_CODEC,
20465                        "result": activity_result,
20466                    }),
20467                ),
20468            ]))
20469            .expect("replayed workflow task");
20470        assert_eq!(replayed.len(), 1);
20471        assert_eq!(replayed[0]["type"], "complete_workflow");
20472        assert_eq!(
20473            decode_wire_value(&replayed[0]["result"], DEFAULT_CODEC).expect("result"),
20474            json!("done")
20475        );
20476    }
20477
20478    #[test]
20479    fn workflow_continue_as_new_emits_arguments_type_and_queue_once() {
20480        let client = Client::new("http://127.0.0.1:8080").expect("client");
20481        let mut worker = Worker::new(client, "rust-workers");
20482        worker.register_workflow("rust.continue", |ctx, _input| async move {
20483            ctx.continue_as_new_with_options(
20484                ContinueAsNewOptions::new()
20485                    .workflow_type("rust.next")
20486                    .task_queue("next-workers"),
20487                json!([2, {"cursor": "next"}]),
20488            )
20489        });
20490
20491        let commands = worker
20492            .execute_workflow_task(workflow_task("rust.continue", Vec::new(), DEFAULT_CODEC))
20493            .expect("continue-as-new command");
20494
20495        assert_eq!(commands.len(), 1);
20496        assert_eq!(commands[0]["type"], "continue_as_new");
20497        assert_eq!(commands[0]["workflow_type"], "rust.next");
20498        assert_eq!(commands[0]["queue"], "next-workers");
20499        assert_eq!(
20500            decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC)
20501                .expect("continue-as-new arguments"),
20502            json!([2, {"cursor": "next"}])
20503        );
20504    }
20505
20506    #[test]
20507    fn continue_as_new_preserves_typed_arguments() {
20508        let client = Client::new("http://127.0.0.1:8080").expect("client");
20509        let mut worker = Worker::new(client, "rust-workers");
20510        worker.register_workflow_avro_value("rust.typed-continue", |ctx, _input| async move {
20511            ctx.continue_as_new(AvroValue::Array(vec![typed_fidelity_probe()]))?;
20512            unreachable!("continue-as-new returns a control-flow error")
20513        });
20514
20515        let commands = worker
20516            .execute_workflow_task(workflow_task(
20517                "rust.typed-continue",
20518                Vec::new(),
20519                DEFAULT_CODEC,
20520            ))
20521            .expect("typed continue-as-new command");
20522
20523        assert_eq!(commands[0]["type"], "continue_as_new");
20524        assert_eq!(
20525            decode_wire_avro_value(&commands[0]["arguments"], DEFAULT_CODEC)
20526                .expect("typed continue arguments"),
20527            AvroValue::Array(vec![typed_fidelity_probe()])
20528        );
20529    }
20530
20531    #[test]
20532    fn recorded_continue_as_new_is_consumed_without_duplicate_successor_command() {
20533        let client = Client::new("http://127.0.0.1:8080").expect("client");
20534        let mut worker = Worker::new(client, "rust-workers");
20535        worker.register_workflow("rust.continue", |ctx, _input| async move {
20536            ctx.continue_as_new(json!([2]))
20537        });
20538        let task = workflow_task(
20539            "rust.continue",
20540            vec![history_event(
20541                "WorkflowContinuedAsNew",
20542                json!({"sequence": 1, "continued_to_run_id": "run-next"}),
20543            )],
20544            DEFAULT_CODEC,
20545        );
20546
20547        for _worker_restart_or_redelivery in 0..2 {
20548            let commands = worker
20549                .execute_workflow_task(task.clone())
20550                .expect("recorded transition replays");
20551            assert!(
20552                commands.is_empty(),
20553                "replay must not emit another successor"
20554            );
20555        }
20556    }
20557
20558    #[test]
20559    fn continue_as_new_rejects_invalid_overrides_before_emitting_a_command() {
20560        let ctx = workflow_context(Vec::new());
20561        let error = ctx
20562            .continue_as_new_with_options(ContinueAsNewOptions::new().task_queue("  "), json!([1]))
20563            .expect_err("blank queue must be rejected");
20564
20565        let Error::InvalidContinueAsNewOptions(error) = error else {
20566            panic!("expected typed continue-as-new validation error");
20567        };
20568        assert_eq!(error.field, "task_queue");
20569        assert!(ctx.take_commands().expect("commands").is_empty());
20570    }
20571
20572    #[test]
20573    fn workflow_context_exposes_server_history_budget() {
20574        let client = Client::new("http://127.0.0.1:8080").expect("client");
20575        let mut worker = Worker::new(client, "rust-workers");
20576        worker.register_workflow("rust.history-budget", |ctx, _input| async move {
20577            let budget = ctx.history_budget()?;
20578            Ok(json!({
20579                "events": budget.event_count,
20580                "bytes": budget.size_bytes,
20581                "recommended": budget.continue_as_new_recommended,
20582                "pressure": budget.pressure,
20583            }))
20584        });
20585        let task: WorkflowTask = serde_json::from_value(json!({
20586            "task_id": "task-history-budget",
20587            "workflow_type": "rust.history-budget",
20588            "payload_codec": DEFAULT_CODEC,
20589            "history_events": [],
20590            "total_history_events": 480,
20591            "history_size_bytes": 1_048_576,
20592            "continue_as_new_recommended": true,
20593            "history_budget_pressure": "continue_as_new_recommended",
20594        }))
20595        .expect("published workflow task");
20596
20597        let commands = worker
20598            .execute_workflow_task(task)
20599            .expect("history-budget workflow");
20600        let result = decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("result");
20601        assert_eq!(result["events"], 480);
20602        assert_eq!(result["bytes"], 1_048_576);
20603        assert_eq!(result["recommended"], true);
20604        assert_eq!(result["pressure"], "continue_as_new_recommended");
20605    }
20606
20607    #[test]
20608    fn uncaught_workflow_handler_error_emits_terminal_failure_command() {
20609        let client = Client::new("http://127.0.0.1:8080").expect("client");
20610        let mut worker = Worker::new(client, "rust-workers");
20611        worker.register_workflow("rust.failing", |_ctx, _input| async move {
20612            Err(Error::Codec("rust_conformance_failure".to_string()))
20613        });
20614        let task = WorkflowTask {
20615            task_id: "wft-rust-failing-1".to_string(),
20616            workflow_command_id: None,
20617            workflow_id: Some("wf-rust-failing".to_string()),
20618            run_id: Some("run-rust-failing".to_string()),
20619            workflow_type: "rust.failing".to_string(),
20620            cancel_requested: false,
20621            payload_codec: DEFAULT_CODEC.to_string(),
20622            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
20623            history_events: Vec::new(),
20624            total_history_events: Some(0),
20625            history_size_bytes: None,
20626            continue_as_new_recommended: None,
20627            history_budget_pressure: None,
20628            next_history_page_token: None,
20629            workflow_task_attempt: 1,
20630            workflow_signal_id: None,
20631            signal_name: None,
20632            signal_arguments: None,
20633            workflow_update_id: None,
20634            update_name: None,
20635            lease_owner: Some("rust-worker".to_string()),
20636        };
20637
20638        let commands = worker
20639            .execute_workflow_task(task)
20640            .expect("handler failure becomes a workflow command");
20641
20642        assert_eq!(commands.len(), 1);
20643        assert_eq!(commands[0]["type"], "fail_workflow");
20644        assert_eq!(commands[0]["exception_type"], "RustWorkflowError");
20645        assert_eq!(commands[0]["exception_class"], "durable_workflow::Error");
20646        assert_eq!(commands[0]["non_retryable"], false);
20647        assert_eq!(
20648            commands[0]["message"],
20649            "codec error: rust_conformance_failure"
20650        );
20651        assert_eq!(
20652            commands[0]["exception"]["message"],
20653            "codec error: rust_conformance_failure"
20654        );
20655    }
20656
20657    #[test]
20658    fn ordinary_handler_error_preserves_commands_queued_in_the_same_decision() {
20659        let client = Client::new("http://127.0.0.1:8080").expect("client");
20660        let mut worker = Worker::new(client, "rust-workers");
20661        worker.register_workflow("rust.failing-after-side-effect", |ctx, _input| async move {
20662            let _: String = ctx.side_effect(|| "captured".to_string())?;
20663            Err(Error::WorkerLoop("application failure".to_string()))
20664        });
20665
20666        let commands = worker
20667            .execute_workflow_task(workflow_task(
20668                "rust.failing-after-side-effect",
20669                Vec::new(),
20670                DEFAULT_CODEC,
20671            ))
20672            .expect("ordinary failure remains a workflow decision");
20673
20674        assert_eq!(commands.len(), 2);
20675        assert_eq!(commands[0]["type"], "record_side_effect");
20676        assert_eq!(commands[1]["type"], "fail_workflow");
20677    }
20678
20679    #[test]
20680    fn handler_error_cannot_hide_an_unconsumed_committed_side_effect() {
20681        let client = Client::new("http://127.0.0.1:8080").expect("client");
20682        let mut worker = Worker::new(client, "rust-workers");
20683        worker.register_workflow("rust.removed-side-effect", |_ctx, _input| async move {
20684            Err(Error::WorkerLoop("application failure".to_string()))
20685        });
20686        let result =
20687            encode_value_envelope(&json!("committed"), DEFAULT_CODEC).expect("side-effect result");
20688
20689        let error = worker
20690            .execute_workflow_task(workflow_task(
20691                "rust.removed-side-effect",
20692                vec![history_event(
20693                    "SideEffectRecorded",
20694                    json!({"sequence": 1, "result": result}),
20695                )],
20696                DEFAULT_CODEC,
20697            ))
20698            .expect_err("removed committed history must not become fail_workflow");
20699
20700        let Error::NonDeterministicReplay(failure) = error else {
20701            panic!("expected typed replay failure");
20702        };
20703        assert_eq!(failure.reason, "recorded_commands_unconsumed");
20704        assert_eq!(failure.sequence, Some(1));
20705        assert_eq!(failure.expected.as_deref(), Some("side effect"));
20706    }
20707
20708    #[test]
20709    fn replay_error_discards_side_effect_queued_before_incompatible_marker_check() {
20710        let client = Client::new("http://127.0.0.1:8080").expect("client");
20711        let mut worker = Worker::new(client, "rust-workers");
20712        worker.register_workflow(
20713            "rust.side-effect-before-marker-error",
20714            |ctx, _input| async move {
20715                assert_eq!(ctx.get_version("restart-safe", 1, 1)?, 1);
20716                let _: String = ctx.side_effect(|| "must-not-commit".to_string())?;
20717                ctx.get_version("restart-safe", 2, 2)?;
20718                Ok(Value::Null)
20719            },
20720        );
20721
20722        let error = worker
20723            .execute_workflow_task(workflow_task(
20724                "rust.side-effect-before-marker-error",
20725                vec![history_event(
20726                    "VersionMarkerRecorded",
20727                    json!({
20728                        "sequence": 1,
20729                        "change_id": "restart-safe",
20730                        "version": 1,
20731                        "min_supported": 1,
20732                        "max_supported": 1,
20733                    }),
20734                )],
20735                DEFAULT_CODEC,
20736            ))
20737            .expect_err("replay error must return no queued workflow commands");
20738
20739        let Error::NonDeterministicReplay(failure) = error else {
20740            panic!("expected typed replay failure");
20741        };
20742        assert_eq!(failure.reason, "version_marker_incompatible_range");
20743        assert_eq!(failure.sequence, Some(1));
20744    }
20745
20746    #[test]
20747    fn workflow_task_replay_keeps_recorded_unfired_timer_pending_without_rescheduling() {
20748        let client = Client::new("http://127.0.0.1:8080").expect("client");
20749        let mut worker = Worker::new(client, "rust-workers");
20750        worker.register_workflow("rust.timer.pending", |ctx, _input| async move {
20751            ctx.sleep(Duration::from_secs(5)).await?;
20752            Ok(json!({"status": "timer fired"}))
20753        });
20754
20755        let task = WorkflowTask {
20756            task_id: "wft-rust-timer-pending".to_string(),
20757            workflow_command_id: None,
20758            workflow_id: Some("wf-rust-timer".to_string()),
20759            run_id: Some("run-rust-timer".to_string()),
20760            workflow_type: "rust.timer.pending".to_string(),
20761            cancel_requested: false,
20762            payload_codec: DEFAULT_CODEC.to_string(),
20763            arguments: Some(
20764                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
20765            ),
20766            history_events: vec![history_event(
20767                "TimerScheduled",
20768                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20769            )],
20770            total_history_events: Some(1),
20771            history_size_bytes: None,
20772            continue_as_new_recommended: None,
20773            history_budget_pressure: None,
20774            next_history_page_token: None,
20775            workflow_task_attempt: 1,
20776            workflow_signal_id: None,
20777            signal_name: None,
20778            signal_arguments: None,
20779            workflow_update_id: None,
20780            update_name: None,
20781            lease_owner: Some("rust-worker".to_string()),
20782        };
20783
20784        for _redelivery_or_restart in 0..2 {
20785            let commands = worker
20786                .execute_workflow_task(task.clone())
20787                .expect("recorded timer remains pending");
20788            assert!(
20789                commands.is_empty(),
20790                "recorded timer must not be rescheduled"
20791            );
20792        }
20793    }
20794
20795    #[test]
20796    fn workflow_task_rejects_recorded_command_removed_from_workflow_code() {
20797        let client = Client::new("http://127.0.0.1:8080").expect("client");
20798        let mut worker = Worker::new(client, "rust-workers");
20799        worker.register_workflow("rust.timer.removed", |_ctx, _input| async move {
20800            Ok(json!({"status": "completed"}))
20801        });
20802        let task = WorkflowTask {
20803            task_id: "wft-rust-timer-removed".to_string(),
20804            workflow_command_id: None,
20805            workflow_id: Some("wf-rust-timer".to_string()),
20806            run_id: Some("run-rust-timer".to_string()),
20807            workflow_type: "rust.timer.removed".to_string(),
20808            cancel_requested: false,
20809            payload_codec: DEFAULT_CODEC.to_string(),
20810            arguments: Some(
20811                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
20812            ),
20813            history_events: vec![
20814                history_event(
20815                    "TimerScheduled",
20816                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20817                ),
20818                history_event(
20819                    "TimerFired",
20820                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20821                ),
20822            ],
20823            total_history_events: Some(2),
20824            history_size_bytes: None,
20825            continue_as_new_recommended: None,
20826            history_budget_pressure: None,
20827            next_history_page_token: None,
20828            workflow_task_attempt: 1,
20829            workflow_signal_id: None,
20830            signal_name: None,
20831            signal_arguments: None,
20832            workflow_update_id: None,
20833            update_name: None,
20834            lease_owner: Some("rust-worker".to_string()),
20835        };
20836
20837        let Error::NonDeterministicReplay(failure) = worker
20838            .execute_workflow_task(task)
20839            .expect_err("removed timer must fail replay")
20840        else {
20841            panic!("expected typed replay failure");
20842        };
20843        assert_eq!(failure.reason, "recorded_commands_unconsumed");
20844        assert_eq!(failure.sequence, Some(1));
20845    }
20846
20847    #[test]
20848    fn workflow_context_emits_explicit_child_workflow_contract() {
20849        let ctx = WorkflowContext {
20850            state: Arc::new(Mutex::new(
20851                WorkflowState::new_with_identity(
20852                    Vec::new(),
20853                    Some("wf-parent".to_string()),
20854                    Some("run-parent".to_string()),
20855                    "parent-workers".to_string(),
20856                    DEFAULT_CODEC.to_string(),
20857                    None,
20858                )
20859                .expect("workflow state"),
20860            )),
20861        };
20862        let options = ChildWorkflowOptions::new("python-workers")
20863            .parent_close_policy(ParentClosePolicy::RequestCancel)
20864            .retry_policy(ChildWorkflowRetryPolicy {
20865                max_attempts: Some(3),
20866                backoff_seconds: vec![1, 5],
20867                non_retryable_error_types: vec!["ValidationError".to_string()],
20868            })
20869            .execution_timeout_seconds(600)
20870            .run_timeout_seconds(120);
20871        let mut call = Box::pin(ctx.start_child_workflow(
20872            "python.fulfil-order",
20873            options,
20874            json!([{"order_id": "order-42"}]),
20875        ));
20876        let mut task_context = TaskContext::from_waker(noop_waker_ref());
20877
20878        assert!(matches!(
20879            call.as_mut().poll(&mut task_context),
20880            Poll::Pending
20881        ));
20882        let commands = ctx.take_commands().expect("commands");
20883        assert_eq!(commands.len(), 1);
20884        let command = &commands[0];
20885        assert_eq!(command["type"], "start_child_workflow");
20886        assert_eq!(command["workflow_type"], "python.fulfil-order");
20887        assert_eq!(command["queue"], "python-workers");
20888        assert_eq!(command["parent_close_policy"], "request_cancel");
20889        assert_eq!(command["retry_policy"]["max_attempts"], 3);
20890        assert_eq!(command["execution_timeout_seconds"], 600);
20891        assert_eq!(command["run_timeout_seconds"], 120);
20892        assert_eq!(
20893            decode_wire_value(&command["arguments"], DEFAULT_CODEC).expect("child args"),
20894            json!([{"order_id": "order-42"}])
20895        );
20896    }
20897
20898    fn child_parent_worker() -> Worker {
20899        let client = Client::new("http://127.0.0.1:8080").expect("client");
20900        let mut worker = Worker::new(client, "rust-parent-workers");
20901        worker.register_workflow("rust.parent", |ctx, _input| async move {
20902            let child = ctx
20903                .start_child_workflow(
20904                    "python.child",
20905                    ChildWorkflowOptions::new("python-child-workers")
20906                        .parent_close_policy(ParentClosePolicy::Terminate),
20907                    json!([{"codec_probe": [1, true, "rust"]}]),
20908                )
20909                .await?;
20910            Ok(json!({
20911                "parent_workflow_id": child.parent.workflow_id,
20912                "parent_run_id": child.parent.run_id,
20913                "child_workflow_id": child.child.workflow_id,
20914                "child_run_id": child.child.run_id,
20915                "child_workflow_type": child.child_workflow_type,
20916                "result": child.result,
20917            }))
20918        });
20919        worker
20920    }
20921
20922    fn child_parent_task(event_type: &str, payload: Value) -> WorkflowTask {
20923        WorkflowTask {
20924            task_id: "wft-child-parent".to_string(),
20925            workflow_command_id: None,
20926            workflow_id: Some("wf-parent".to_string()),
20927            run_id: Some("run-parent".to_string()),
20928            workflow_type: "rust.parent".to_string(),
20929            cancel_requested: false,
20930            payload_codec: DEFAULT_CODEC.to_string(),
20931            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
20932            history_events: vec![
20933                HistoryEvent {
20934                    event_type: "ChildWorkflowScheduled".to_string(),
20935                    payload: json!({
20936                        "sequence": 1,
20937                        "child_call_id": "call-child",
20938                        "child_workflow_instance_id": "wf-child",
20939                        "child_workflow_run_id": "run-child",
20940                        "child_workflow_type": "python.child",
20941                    }),
20942                    raw: HashMap::new(),
20943                },
20944                HistoryEvent {
20945                    event_type: event_type.to_string(),
20946                    payload,
20947                    raw: HashMap::new(),
20948                },
20949            ],
20950            total_history_events: Some(2),
20951            history_size_bytes: None,
20952            continue_as_new_recommended: None,
20953            history_budget_pressure: None,
20954            next_history_page_token: None,
20955            workflow_task_attempt: 1,
20956            workflow_signal_id: None,
20957            signal_name: None,
20958            signal_arguments: None,
20959            workflow_update_id: None,
20960            update_name: None,
20961            lease_owner: Some("rust-worker".to_string()),
20962        }
20963    }
20964
20965    #[test]
20966    fn committed_child_result_replays_without_starting_a_duplicate() {
20967        let worker = child_parent_worker();
20968        let task = child_parent_task(
20969            "ChildRunCompleted",
20970            json!({
20971                "sequence": 1,
20972                "child_call_id": "call-child",
20973                "child_workflow_instance_id": "wf-child",
20974                "child_workflow_run_id": "run-child",
20975                "child_workflow_type": "python.child",
20976                "payload_codec": DEFAULT_CODEC,
20977                "result": fixture_envelope(json!({"from":"python","ok":true})),
20978            }),
20979        );
20980
20981        for _restart in 0..2 {
20982            let commands = worker
20983                .execute_workflow_task(task.clone())
20984                .expect("replayed parent task");
20985            assert_eq!(commands.len(), 1);
20986            assert_eq!(commands[0]["type"], "complete_workflow");
20987            assert!(!commands
20988                .iter()
20989                .any(|command| command["type"] == "start_child_workflow"));
20990            let output =
20991                decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
20992            assert_eq!(output["parent_workflow_id"], "wf-parent");
20993            assert_eq!(output["parent_run_id"], "run-parent");
20994            assert_eq!(output["child_workflow_id"], "wf-child");
20995            assert_eq!(output["child_run_id"], "run-child");
20996            assert_eq!(output["result"], json!({"from": "python", "ok": true}));
20997        }
20998    }
20999
21000    #[test]
21001    fn typed_child_arguments_and_results_survive_replay() {
21002        let client = Client::new("http://127.0.0.1:8080").expect("client");
21003        let mut worker = Worker::new(client, "rust-parent-workers");
21004        worker.register_workflow_avro_value("rust.typed-parent", |ctx, _input| async move {
21005            let child = ctx
21006                .start_child_workflow_avro_value(
21007                    "python.typed-child",
21008                    ChildWorkflowOptions::new("python-workers"),
21009                    AvroValue::Array(vec![typed_fidelity_probe()]),
21010                )
21011                .await?;
21012            Ok(child.result)
21013        });
21014
21015        let initial = worker
21016            .execute_workflow_task(workflow_task(
21017                "rust.typed-parent",
21018                Vec::new(),
21019                DEFAULT_CODEC,
21020            ))
21021            .expect("typed child start");
21022        assert_eq!(initial[0]["type"], "start_child_workflow");
21023        assert_eq!(
21024            decode_wire_avro_value(&initial[0]["arguments"], DEFAULT_CODEC)
21025                .expect("typed child arguments"),
21026            AvroValue::Array(vec![typed_fidelity_probe()])
21027        );
21028
21029        let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
21030            .expect("typed child result");
21031        let task = workflow_task(
21032            "rust.typed-parent",
21033            vec![
21034                history_event(
21035                    "ChildWorkflowScheduled",
21036                    json!({
21037                        "sequence": 1,
21038                        "child_call_id": "call-typed",
21039                        "child_workflow_instance_id": "wf-child",
21040                        "child_workflow_run_id": "run-child",
21041                        "child_workflow_type": "python.typed-child",
21042                    }),
21043                ),
21044                history_event(
21045                    "ChildRunCompleted",
21046                    json!({
21047                        "sequence": 1,
21048                        "child_call_id": "call-typed",
21049                        "child_workflow_instance_id": "wf-child",
21050                        "child_workflow_run_id": "run-child",
21051                        "child_workflow_type": "python.typed-child",
21052                        "payload_codec": DEFAULT_CODEC,
21053                        "result": result,
21054                    }),
21055                ),
21056            ],
21057            DEFAULT_CODEC,
21058        );
21059
21060        let commands = worker
21061            .execute_workflow_task(task)
21062            .expect("typed child replay");
21063        assert_eq!(commands[0]["type"], "complete_workflow");
21064        assert_eq!(
21065            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
21066                .expect("typed parent result"),
21067            typed_fidelity_probe()
21068        );
21069    }
21070
21071    #[test]
21072    fn pending_child_replays_after_restart_without_starting_a_duplicate() {
21073        let worker = child_parent_worker();
21074        let mut task = child_parent_task("unused", Value::Null);
21075        task.history_events.truncate(1);
21076        task.total_history_events = Some(1);
21077
21078        for _redelivery_or_restart in 0..2 {
21079            let commands = worker
21080                .execute_workflow_task(task.clone())
21081                .expect("recorded child remains pending");
21082            assert!(
21083                commands.is_empty(),
21084                "recorded pending child must not be started again"
21085            );
21086        }
21087    }
21088
21089    #[test]
21090    fn child_cancellation_becomes_stable_parent_failure_command() {
21091        let worker = child_parent_worker();
21092        let task = child_parent_task(
21093            "ChildRunCancelled",
21094            json!({
21095                "sequence": 1,
21096                "child_workflow_instance_id": "wf-child",
21097                "child_workflow_run_id": "run-child",
21098                "child_workflow_type": "python.child",
21099                "failure_id": "failure-child",
21100                "failure_category": "cancelled",
21101                "message": "cancelled by parent-close policy",
21102            }),
21103        );
21104
21105        let commands = worker
21106            .execute_workflow_task(task)
21107            .expect("parent settlement");
21108        assert_eq!(commands.len(), 1);
21109        assert_eq!(commands[0]["type"], "fail_workflow");
21110        assert_eq!(commands[0]["exception_type"], "ChildWorkflowCancelled");
21111        assert_eq!(
21112            commands[0]["exception"]["properties"]["reason"],
21113            "cancelled"
21114        );
21115        assert_eq!(
21116            commands[0]["exception"]["properties"]["child_workflow_run_id"],
21117            "run-child"
21118        );
21119    }
21120
21121    #[test]
21122    fn workflow_can_handle_typed_child_failure() {
21123        let client = Client::new("http://127.0.0.1:8080").expect("client");
21124        let mut worker = Worker::new(client, "rust-parent-workers");
21125        worker.register_workflow("rust.handled-parent", |ctx, _input| async move {
21126            match ctx
21127                .start_child_workflow(
21128                    "python.child",
21129                    ChildWorkflowOptions::new("python-child-workers"),
21130                    json!([]),
21131                )
21132                .await
21133            {
21134                Err(Error::ChildWorkflowFailed(failure)) => Ok(json!({
21135                    "reason": failure.reason,
21136                    "failure_id": failure.failure_id,
21137                    "exception_class": failure.exception_class,
21138                    "child_run_id": failure.child_workflow_run_id,
21139                })),
21140                Err(error) => Err(error),
21141                Ok(_) => Err(Error::WorkerLoop(
21142                    "child unexpectedly succeeded".to_string(),
21143                )),
21144            }
21145        });
21146        let mut task = child_parent_task(
21147            "ChildRunFailed",
21148            json!({
21149                "sequence": 1,
21150                "child_workflow_instance_id": "wf-child",
21151                "child_workflow_run_id": "run-child",
21152                "child_workflow_type": "python.child",
21153                "failure_id": "failure-child",
21154                "failure_category": "child_workflow",
21155                "message": "payment rejected",
21156                "exception": {
21157                    "type": "PaymentRejected",
21158                    "class": "payments.PaymentRejected",
21159                    "message": "payment rejected"
21160                }
21161            }),
21162        );
21163        task.workflow_type = "rust.handled-parent".to_string();
21164
21165        let commands = worker.execute_workflow_task(task).expect("handled failure");
21166        assert_eq!(commands[0]["type"], "complete_workflow");
21167        let output =
21168            decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
21169        assert_eq!(output["reason"], "child_workflow");
21170        assert_eq!(output["failure_id"], "failure-child");
21171        assert_eq!(output["exception_class"], "payments.PaymentRejected");
21172        assert_eq!(output["child_run_id"], "run-child");
21173    }
21174
21175    #[test]
21176    fn rust_hello_world_uses_signal_arguments_from_resume_payload() {
21177        let client = Client::new("http://127.0.0.1:8080").expect("client");
21178        let mut worker = Worker::new(client, "rust-workers");
21179
21180        worker.register_workflow("rust.hello_workflow", |ctx, _input| async move {
21181            let signal = ctx.wait_signal("start").await?;
21182            let name = signal
21183                .first()
21184                .and_then(|value| value.as_str())
21185                .unwrap_or("world");
21186            let greeting = ctx.activity("rust.hello_activity", json!([name])).await?;
21187            Ok(json!({
21188                "greeting": greeting,
21189                "language": "rust"
21190            }))
21191        });
21192
21193        let signal_arguments =
21194            encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC).expect("signal arguments");
21195        let task = WorkflowTask {
21196            task_id: "wft-rust-signal-1".to_string(),
21197            workflow_command_id: None,
21198            workflow_id: Some("wf-rust-hello".to_string()),
21199            run_id: Some("run-rust-hello".to_string()),
21200            workflow_type: "rust.hello_workflow".to_string(),
21201            cancel_requested: false,
21202            payload_codec: DEFAULT_CODEC.to_string(),
21203            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21204            history_events: vec![HistoryEvent {
21205                event_type: "SignalReceived".to_string(),
21206                payload: json!({
21207                    "signal_id": "sig-rust-1",
21208                    "signal_name": "start"
21209                }),
21210                raw: HashMap::new(),
21211            }],
21212            total_history_events: Some(1),
21213            history_size_bytes: None,
21214            continue_as_new_recommended: None,
21215            history_budget_pressure: None,
21216            next_history_page_token: None,
21217            workflow_task_attempt: 1,
21218            workflow_signal_id: Some("sig-rust-1".to_string()),
21219            signal_name: Some("start".to_string()),
21220            signal_arguments: Some(signal_arguments),
21221            workflow_update_id: None,
21222            update_name: None,
21223            lease_owner: Some("rust-worker".to_string()),
21224        };
21225
21226        let commands = worker.execute_workflow_task(task).expect("workflow task");
21227
21228        assert_eq!(commands.len(), 1);
21229        assert_eq!(commands[0]["type"], "schedule_activity");
21230        assert_eq!(commands[0]["activity_type"], "rust.hello_activity");
21231        assert_eq!(
21232            decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC).expect("activity args"),
21233            json!(["Rust"])
21234        );
21235    }
21236
21237    #[test]
21238    fn workflow_task_appends_paginated_history_events() {
21239        let mut task = WorkflowTask {
21240            task_id: "wft-rust-pages-1".to_string(),
21241            workflow_command_id: None,
21242            workflow_id: Some("wf-rust-pages".to_string()),
21243            run_id: Some("run-rust-pages".to_string()),
21244            workflow_type: "rust.hello_workflow".to_string(),
21245            cancel_requested: false,
21246            payload_codec: DEFAULT_CODEC.to_string(),
21247            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21248            history_events: vec![HistoryEvent {
21249                event_type: "WorkflowStarted".to_string(),
21250                payload: json!({}),
21251                raw: HashMap::new(),
21252            }],
21253            total_history_events: Some(3),
21254            history_size_bytes: None,
21255            continue_as_new_recommended: None,
21256            history_budget_pressure: None,
21257            next_history_page_token: Some("MQ==".to_string()),
21258            workflow_task_attempt: 1,
21259            workflow_signal_id: None,
21260            signal_name: None,
21261            signal_arguments: None,
21262            workflow_update_id: None,
21263            update_name: None,
21264            lease_owner: Some("rust-worker".to_string()),
21265        };
21266
21267        task.append_history_page(WorkflowTaskHistoryPage {
21268            history_events: vec![
21269                HistoryEvent {
21270                    event_type: "SignalReceived".to_string(),
21271                    payload: json!({
21272                        "signal_id": "sig-rust-1",
21273                        "signal_name": "start",
21274                        "arguments": encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC)
21275                            .expect("signal arguments")
21276                    }),
21277                    raw: HashMap::new(),
21278                },
21279                HistoryEvent {
21280                    event_type: "MarkerRecorded".to_string(),
21281                    payload: json!({"sequence": 3}),
21282                    raw: HashMap::new(),
21283                },
21284            ],
21285            total_history_events: Some(3),
21286            next_history_page_token: None,
21287        });
21288
21289        assert_eq!(task.history_events.len(), 3);
21290        assert_eq!(task.total_history_events, Some(3));
21291        assert_eq!(task.next_history_page_token, None);
21292
21293        let signal = task
21294            .history_events
21295            .iter()
21296            .find(|event| event.event_type == "SignalReceived")
21297            .expect("signal event");
21298        assert_eq!(
21299            decode_signal_event_arguments(signal, DEFAULT_CODEC).expect("signal arguments"),
21300            vec![AvroValue::String("Rust".to_string())]
21301        );
21302    }
21303
21304    #[tokio::test]
21305    async fn query_handler_reads_ordered_cross_codec_signals_without_commands() {
21306        let client = Client::new("http://127.0.0.1:8080").expect("client");
21307        let mut worker = Worker::new(client, "rust-workers");
21308        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21309        worker.register_query("counter", "current", |ctx, _args| async move {
21310            let mut count = 0_i64;
21311            for signal in ctx.signal_events() {
21312                let value = signal
21313                    .arguments
21314                    .first()
21315                    .and_then(Value::as_i64)
21316                    .unwrap_or_default();
21317                match signal.name.as_str() {
21318                    "increment" => count += value,
21319                    "set" => count = value,
21320                    _ => {}
21321                }
21322            }
21323            Ok(json!(count))
21324        });
21325
21326        let task = QueryTask {
21327            query_task_id: "query-rust-counter".to_string(),
21328            query_task_attempt: 1,
21329            lease_owner: Some("rust-worker".to_string()),
21330            workflow_id: Some("counter-1".to_string()),
21331            run_id: Some("run-counter-1".to_string()),
21332            workflow_type: "counter".to_string(),
21333            query_name: "current".to_string(),
21334            payload_codec: DEFAULT_CODEC.to_string(),
21335            workflow_arguments: Some(
21336                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
21337            ),
21338            query_arguments: Some(
21339                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("query arguments"),
21340            ),
21341            history_events: vec![
21342                HistoryEvent {
21343                    event_type: "SignalReceived".to_string(),
21344                    payload: json!({
21345                        "signal_id": "php-signal-1",
21346                        "signal_name": "increment",
21347                        "workflow_sequence": 1,
21348                        "payload_codec": DEFAULT_CODEC,
21349                        "arguments": encode_value_envelope(&json!([3]), DEFAULT_CODEC).expect("php avro signal")
21350                    }),
21351                    raw: HashMap::new(),
21352                },
21353                HistoryEvent {
21354                    event_type: "SignalReceived".to_string(),
21355                    payload: json!({
21356                        "signal_id": "python-signal-2",
21357                        "signal_name": "increment",
21358                        "workflow_sequence": 2,
21359                        "payload_codec": DEFAULT_CODEC,
21360                        "arguments": encode_value_envelope(&json!([5]), DEFAULT_CODEC).expect("python avro signal")
21361                    }),
21362                    raw: HashMap::new(),
21363                },
21364                HistoryEvent {
21365                    event_type: "SignalReceived".to_string(),
21366                    payload: json!({
21367                        "signal_id": "rust-signal-3",
21368                        "signal_name": "set",
21369                        "workflow_sequence": 3,
21370                        "payload_codec": DEFAULT_CODEC,
21371                        "arguments": encode_value_envelope(&json!([0]), DEFAULT_CODEC).expect("rust avro signal")
21372                    }),
21373                    raw: HashMap::new(),
21374                },
21375            ],
21376            history_export: None,
21377            run_status: Some("completed".to_string()),
21378        };
21379
21380        let result = worker.execute_query_task(task).await.expect("query result");
21381        assert_eq!(result.into_json().expect("query projection"), json!(0));
21382    }
21383
21384    #[tokio::test]
21385    async fn replayed_queries_read_running_completed_and_cold_restarted_instance_state() {
21386        let worker = replay_counter_worker();
21387        let running_history = json!([
21388            {
21389                "type": "ActivityCompleted",
21390                "payload": {
21391                    "sequence": 1,
21392                    "activity_type": "load-counter",
21393                    "payload_codec": DEFAULT_CODEC,
21394                    "result": fixture_envelope(json!("loaded"))
21395                }
21396            },
21397            {
21398                "type": "SignalWaitOpened",
21399                "payload": {
21400                    "sequence": 3,
21401                    "signal_name": "increment"
21402                }
21403            },
21404            {
21405                "type": "SignalReceived",
21406                "payload": {
21407                    "signal_id": "signal-3",
21408                    "signal_name": "increment",
21409                    "workflow_sequence": 2,
21410                    "payload_codec": DEFAULT_CODEC,
21411                    "arguments": fixture_envelope(json!([3]))
21412                }
21413            },
21414            {
21415                "type": "SignalApplied",
21416                "payload": {
21417                    "sequence": 3,
21418                    "signal_id": "signal-3",
21419                    "signal_name": "increment",
21420                    "payload_codec": DEFAULT_CODEC,
21421                    "value": fixture_envelope(json!([3]))
21422                }
21423            }
21424        ]);
21425
21426        let running = worker
21427            .execute_query_task(replay_counter_query(
21428                "current",
21429                running_history.clone(),
21430                "running",
21431            ))
21432            .await
21433            .expect("running replay query");
21434        assert_eq!(
21435            running.clone().into_json().expect("query projection"),
21436            json!({"loaded": "loaded", "count": 3, "finished": false})
21437        );
21438
21439        let detached = worker
21440            .execute_query_task(replay_counter_query(
21441                "detached-mutation",
21442                running_history.clone(),
21443                "running",
21444            ))
21445            .await
21446            .expect("query mutates only its detached state clone");
21447        assert_eq!(detached.into_json().expect("query projection"), json!(999));
21448        let failed = worker
21449            .execute_query_task(replay_counter_query(
21450                "failed-mutation",
21451                running_history.clone(),
21452                "running",
21453            ))
21454            .await
21455            .expect_err("failed query");
21456        assert_eq!(failed.reason, "query_rejected");
21457        let unchanged = worker
21458            .execute_query_task(replay_counter_query("current", running_history, "running"))
21459            .await
21460            .expect("later query reconstructs unchanged state");
21461        assert_eq!(unchanged, running);
21462
21463        let restarted_worker = replay_counter_worker();
21464        let empty_arguments = fixture_envelope(json!([]));
21465        let loaded_result = fixture_envelope(json!("loaded"));
21466        let signal_three = fixture_blob(json!([3]));
21467        let signal_five = fixture_blob(json!([5]));
21468        let restarted_task: QueryTask = serde_json::from_value(json!({
21469            "query_task_id": "query-after-restart",
21470            "workflow_id": "counter-1",
21471            "run_id": "run-counter-1",
21472            "workflow_type": "replay-counter",
21473            "query_name": "current",
21474            "payload_codec": DEFAULT_CODEC,
21475            "workflow_arguments": empty_arguments.clone(),
21476            "query_arguments": empty_arguments,
21477            "history_events": [],
21478            "history_export": {
21479                "payloads": {"codec": DEFAULT_CODEC},
21480                "history_events": [
21481                    {
21482                        "type": "ActivityCompleted",
21483                        "payload": {
21484                            "sequence": 1,
21485                            "activity_type": "load-counter",
21486                            "payload_codec": DEFAULT_CODEC,
21487                            "result": null
21488                        }
21489                    },
21490                    {
21491                        "type": "SignalWaitOpened",
21492                        "payload": {
21493                            "sequence": 3,
21494                            "signal_name": "increment"
21495                        }
21496                    },
21497                    {
21498                        "type": "SignalReceived",
21499                        "payload": {
21500                            "signal_id": "signal-3",
21501                            "signal_name": "increment",
21502                            "workflow_sequence": 2
21503                        }
21504                    },
21505                    {
21506                        "type": "SignalApplied",
21507                        "payload": {
21508                            "sequence": 3,
21509                            "signal_id": "signal-3",
21510                            "signal_name": "increment"
21511                        }
21512                    },
21513                    {
21514                        "type": "SignalWaitOpened",
21515                        "payload": {
21516                            "sequence": 5,
21517                            "signal_name": "increment"
21518                        }
21519                    },
21520                    {
21521                        "type": "SignalReceived",
21522                        "payload": {
21523                            "signal_id": "signal-5",
21524                            "signal_name": "increment",
21525                            "workflow_sequence": 4
21526                        }
21527                    },
21528                    {
21529                        "type": "SignalApplied",
21530                        "payload": {
21531                            "sequence": 5,
21532                            "signal_id": "signal-5",
21533                            "signal_name": "increment"
21534                        }
21535                    }
21536                ],
21537                "activities": [{
21538                    "sequence": 1,
21539                    "activity_type": "load-counter",
21540                    "payload_codec": DEFAULT_CODEC,
21541                    "result": loaded_result
21542                }],
21543                "signals": [
21544                    {
21545                        "id": "signal-3",
21546                        "name": "increment",
21547                        "workflow_sequence": 2,
21548                        "payload_codec": DEFAULT_CODEC,
21549                        "arguments": signal_three
21550                    },
21551                    {
21552                        "id": "signal-5",
21553                        "name": "increment",
21554                        "workflow_sequence": 4,
21555                        "payload_codec": DEFAULT_CODEC,
21556                        "arguments": signal_five
21557                    }
21558                ]
21559            },
21560            "run_status": "completed"
21561        }))
21562        .expect("cold replay query task");
21563        let completed = restarted_worker
21564            .execute_query_task(restarted_task)
21565            .await
21566            .expect("completed cold replay query");
21567        assert_eq!(
21568            completed.into_json().expect("query projection"),
21569            json!({"loaded": "loaded", "count": 8, "finished": true})
21570        );
21571    }
21572
21573    #[tokio::test]
21574    async fn replayed_query_replay_failures_are_machine_readable() {
21575        let worker = replay_counter_worker();
21576        let task = replay_counter_query(
21577            "current",
21578            json!([{
21579                "type": "ActivityCompleted",
21580                "payload": {
21581                    "sequence": 1,
21582                    "payload_codec": DEFAULT_CODEC,
21583                    "result": {"codec": DEFAULT_CODEC, "blob": "{"}
21584                }
21585            }]),
21586            "running",
21587        );
21588        let failure = worker
21589            .execute_query_task(task)
21590            .await
21591            .expect_err("invalid replay history payload");
21592        assert_eq!(failure.reason, "query_payload_decode_failed");
21593        assert_eq!(failure.failure_type, "QueryPayloadDecodeFailed");
21594        assert!(failure.message.contains("invalid_payload_framing"));
21595    }
21596
21597    #[tokio::test]
21598    async fn query_task_restores_compact_history_from_export() {
21599        let client = Client::new("http://127.0.0.1:8080").expect("client");
21600        let mut worker = Worker::new(client, "rust-workers");
21601        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21602        worker.register_query("counter", "current", |ctx, _args| async move {
21603            Ok(json!(ctx.signals("increment")[0][0]))
21604        });
21605        let empty_arguments = fixture_envelope(json!([]));
21606        let exported_signal = fixture_blob(json!([9]));
21607        let task: QueryTask = serde_json::from_value(json!({
21608            "query_task_id": "query-export",
21609            "workflow_type": "counter",
21610            "query_name": "current",
21611            "payload_codec": DEFAULT_CODEC,
21612            "workflow_arguments": empty_arguments.clone(),
21613            "query_arguments": empty_arguments,
21614            "history_events": [],
21615            "history_export": {
21616                "payloads": {"codec": DEFAULT_CODEC},
21617                "history_events": [{
21618                    "type": "SignalReceived",
21619                    "payload": {"signal_id": "signal-export", "signal_name": "increment"}
21620                }],
21621                "signals": [{
21622                    "id": "signal-export",
21623                    "name": "increment",
21624                    "status": "applied",
21625                    "workflow_sequence": 1,
21626                    "payload_codec": DEFAULT_CODEC,
21627                    "arguments": exported_signal
21628                }]
21629            }
21630        }))
21631        .expect("query task");
21632
21633        let result = worker.execute_query_task(task).await.expect("query result");
21634        assert_eq!(result.into_json().expect("query projection"), json!(9));
21635    }
21636
21637    #[tokio::test]
21638    async fn query_task_failures_have_stable_reasons() {
21639        let client = Client::new("http://127.0.0.1:8080").expect("client");
21640        let mut worker = Worker::new(client, "rust-workers");
21641        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21642        worker.register_query(
21643            "counter",
21644            "current",
21645            |_ctx, _args| async move { Ok(json!(0)) },
21646        );
21647
21648        let base_task = QueryTask {
21649            query_task_id: "query-errors".to_string(),
21650            query_task_attempt: 1,
21651            lease_owner: None,
21652            workflow_id: Some("counter-errors".to_string()),
21653            run_id: Some("run-errors".to_string()),
21654            workflow_type: "counter".to_string(),
21655            query_name: "missing".to_string(),
21656            payload_codec: DEFAULT_CODEC.to_string(),
21657            workflow_arguments: Some(fixture_envelope(json!([]))),
21658            query_arguments: Some(fixture_envelope(json!([]))),
21659            history_events: Vec::new(),
21660            history_export: None,
21661            run_status: Some("running".to_string()),
21662        };
21663
21664        let unknown = worker
21665            .execute_query_task(base_task.clone())
21666            .await
21667            .expect_err("unknown query");
21668        assert_eq!(unknown.reason, "rejected_unknown_query");
21669
21670        let mut malformed = base_task;
21671        malformed.query_name = "current".to_string();
21672        malformed.query_arguments = Some(json!({"codec": DEFAULT_CODEC, "blob": "{"}));
21673        let malformed = worker
21674            .execute_query_task(malformed)
21675            .await
21676            .expect_err("malformed payload");
21677        assert_eq!(malformed.reason, "query_payload_decode_failed");
21678
21679        let client = Client::new("http://127.0.0.1:8080").expect("client");
21680        let mut unavailable_worker = Worker::new(client, "rust-workers");
21681        unavailable_worker
21682            .register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21683        let empty_arguments = fixture_envelope(json!([]));
21684        let unavailable_task: QueryTask = serde_json::from_value(json!({
21685            "query_task_id": "query-unavailable",
21686            "workflow_type": "counter",
21687            "query_name": "current",
21688            "payload_codec": DEFAULT_CODEC,
21689            "workflow_arguments": empty_arguments.clone(),
21690            "query_arguments": empty_arguments
21691        }))
21692        .expect("query task");
21693        let unavailable = unavailable_worker
21694            .execute_query_task(unavailable_task)
21695            .await
21696            .expect_err("query handler unavailable");
21697        assert_eq!(unavailable.reason, "query_handler_unavailable");
21698    }
21699
21700    #[tokio::test]
21701    async fn client_query_decodes_result_and_typed_failure() {
21702        let server = MockWorkerServer::start();
21703        let client = Client::builder(server.base_url())
21704            .timeout(Duration::from_secs(2))
21705            .build()
21706            .expect("client");
21707
21708        let result = client
21709            .query_workflow("counter-1", "current", json!([]))
21710            .await
21711            .expect("query result");
21712        assert_eq!(result, json!({"count": 8}));
21713
21714        let error = client
21715            .query_workflow("counter-1", "missing", json!([]))
21716            .await
21717            .expect_err("unknown query");
21718        let Error::QueryFailed(failure) = error else {
21719            panic!("expected typed query failure");
21720        };
21721        assert_eq!(failure.status, 404);
21722        assert_eq!(failure.reason, "rejected_unknown_query");
21723    }
21724
21725    #[tokio::test]
21726    async fn public_client_surfaces_send_and_receive_lossless_avro_values() {
21727        let server = MockWorkerServer::start();
21728        let client = Client::builder(server.base_url())
21729            .timeout(Duration::from_secs(2))
21730            .build()
21731            .expect("client");
21732        let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
21733
21734        client
21735            .start_workflow(
21736                "typed.echo",
21737                "rust-workers",
21738                "typed-start",
21739                arguments.clone(),
21740            )
21741            .await
21742            .expect("typed workflow start");
21743        assert_eq!(
21744            decode_wire_avro_value(
21745                &server.request_body("/api/workflows")["input"],
21746                DEFAULT_CODEC,
21747            )
21748            .expect("typed start input"),
21749            arguments
21750        );
21751
21752        client
21753            .signal_workflow("typed-1", "changed", arguments.clone())
21754            .await
21755            .expect("typed signal");
21756        assert_eq!(
21757            decode_wire_avro_value(
21758                &server.request_body("/api/workflows/typed-1/signal/changed")["input"],
21759                DEFAULT_CODEC,
21760            )
21761            .expect("typed signal input"),
21762            arguments
21763        );
21764
21765        assert_eq!(
21766            client
21767                .query_workflow_avro_value("typed-1", "inspect", arguments.clone())
21768                .await
21769                .expect("typed query"),
21770            typed_fidelity_probe()
21771        );
21772        assert_eq!(
21773            decode_wire_avro_value(
21774                &server.request_body("/api/workflows/typed-1/query/inspect")["input"],
21775                DEFAULT_CODEC,
21776            )
21777            .expect("typed query input"),
21778            arguments
21779        );
21780
21781        assert_eq!(
21782            client
21783                .update_workflow_avro_value(
21784                    "typed-1",
21785                    "replace",
21786                    arguments.clone(),
21787                    Some("typed-request"),
21788                )
21789                .await
21790                .expect("typed update"),
21791            typed_fidelity_probe()
21792        );
21793        let update = server.request_body("/api/workflows/typed-1/update/replace");
21794        assert_eq!(update["request_id"], "typed-request");
21795        assert_eq!(
21796            decode_wire_avro_value(&update["input"], DEFAULT_CODEC).expect("typed update input"),
21797            arguments
21798        );
21799
21800        let handle = WorkflowHandle {
21801            client: client.clone(),
21802            workflow_id: "typed-1".to_string(),
21803            run_id: Some("run-typed-1".to_string()),
21804            workflow_type: "typed.echo".to_string(),
21805        };
21806        assert_eq!(
21807            handle
21808                .result_avro_value(WorkflowResultOptions::default())
21809                .await
21810                .expect("typed workflow result"),
21811            typed_fidelity_probe()
21812        );
21813
21814        client
21815            .complete_activity_task(
21816                "activity-typed",
21817                "attempt-typed",
21818                "rust-worker",
21819                typed_fidelity_probe(),
21820                DEFAULT_CODEC,
21821            )
21822            .await
21823            .expect("typed activity completion");
21824        assert_eq!(
21825            decode_wire_avro_value(
21826                &server.request_body("/api/worker/activity-tasks/activity-typed/complete")
21827                    ["result"],
21828                DEFAULT_CODEC,
21829            )
21830            .expect("typed activity result"),
21831            typed_fidelity_probe()
21832        );
21833        client
21834            .fail_activity_task(
21835                "activity-typed",
21836                "attempt-typed",
21837                "rust-worker",
21838                "typed failure",
21839                true,
21840            )
21841            .await
21842            .expect("activity failure");
21843    }
21844
21845    #[tokio::test]
21846    async fn lifecycle_commands_support_instance_and_selected_run_targets() {
21847        let server = MockWorkerServer::start();
21848        let client = Client::builder(server.base_url())
21849            .timeout(Duration::from_secs(2))
21850            .build()
21851            .expect("client");
21852
21853        let options = WorkflowCommandOptions::new()
21854            .reason("cleanup requested")
21855            .request_id("cancel-17");
21856        let cancelled = client
21857            .cancel_workflow("wf-lifecycle", options)
21858            .await
21859            .expect("instance cancellation");
21860        assert_eq!(cancelled.command, WorkflowCommandKind::Cancel);
21861        assert_eq!(cancelled.run_id.as_deref(), Some("run-current"));
21862        assert_eq!(cancelled.outcome.as_deref(), Some("cancelled"));
21863        assert_eq!(
21864            server.request_body("/api/workflows/wf-lifecycle/cancel"),
21865            json!({"reason":"cleanup requested","request_id":"cancel-17"})
21866        );
21867
21868        let terminated = client
21869            .terminate_workflow(
21870                "wf-lifecycle",
21871                WorkflowCommandOptions::new().reason("forced stop"),
21872            )
21873            .await
21874            .expect("instance termination");
21875        assert_eq!(terminated.command, WorkflowCommandKind::Terminate);
21876        assert_eq!(terminated.outcome.as_deref(), Some("terminated"));
21877
21878        client
21879            .cancel_workflow_run(
21880                "wf-lifecycle",
21881                "run-current",
21882                WorkflowCommandOptions::default(),
21883            )
21884            .await
21885            .expect("selected run cancellation");
21886        client
21887            .terminate_workflow_run(
21888                "wf-lifecycle",
21889                "run-current",
21890                WorkflowCommandOptions::default(),
21891            )
21892            .await
21893            .expect("selected run termination");
21894
21895        for (command, error) in [
21896            (
21897                WorkflowCommandKind::Cancel,
21898                client
21899                    .cancel_workflow_run(
21900                        "wf-lifecycle",
21901                        "run-stale",
21902                        WorkflowCommandOptions::default(),
21903                    )
21904                    .await
21905                    .expect_err("stale cancellation must be rejected"),
21906            ),
21907            (
21908                WorkflowCommandKind::Terminate,
21909                client
21910                    .terminate_workflow_run(
21911                        "wf-lifecycle",
21912                        "run-stale",
21913                        WorkflowCommandOptions::default(),
21914                    )
21915                    .await
21916                    .expect_err("stale termination must be rejected"),
21917            ),
21918        ] {
21919            let Error::WorkflowCommandRejected(rejection) = error else {
21920                panic!("expected typed command rejection");
21921            };
21922            assert_eq!(rejection.command, command);
21923            assert_eq!(rejection.status, 409);
21924            assert_eq!(rejection.reason, "historical_run_command_rejected");
21925            assert_eq!(rejection.run_id.as_deref(), Some("run-stale"));
21926            assert_eq!(rejection.target_scope.as_deref(), Some("run"));
21927        }
21928    }
21929
21930    #[tokio::test]
21931    async fn workflow_start_options_send_server_enforced_deadlines() {
21932        let server = MockWorkerServer::start();
21933        let client = Client::builder(server.base_url())
21934            .timeout(Duration::from_secs(2))
21935            .build()
21936            .expect("client");
21937
21938        let handle = client
21939            .start_workflow_with_options(
21940                "rust.timeout",
21941                "rust-timeouts",
21942                "wf-start-options",
21943                WorkflowStartOptions::new()
21944                    .execution_timeout_seconds(30)
21945                    .run_timeout_seconds(1),
21946                json!([]),
21947            )
21948            .await
21949            .expect("workflow start");
21950
21951        assert_eq!(handle.run_id.as_deref(), Some("run-start-options"));
21952        let body = server.request_body("/api/workflows");
21953        assert_eq!(body["execution_timeout_seconds"], 30);
21954        assert_eq!(body["run_timeout_seconds"], 1);
21955
21956        let invalid = client
21957            .start_workflow_with_options(
21958                "rust.timeout",
21959                "rust-timeouts",
21960                "wf-invalid-options",
21961                WorkflowStartOptions::new()
21962                    .execution_timeout_seconds(1)
21963                    .run_timeout_seconds(2),
21964                json!([]),
21965            )
21966            .await
21967            .expect_err("invalid deadline ordering");
21968        assert!(invalid
21969            .to_string()
21970            .contains("run_timeout_seconds cannot exceed execution_timeout_seconds"));
21971    }
21972
21973    #[tokio::test]
21974    async fn workflow_result_returns_each_typed_terminal_outcome() {
21975        let server = MockWorkerServer::start();
21976        let client = Client::builder(server.base_url())
21977            .timeout(Duration::from_secs(2))
21978            .build()
21979            .expect("client");
21980        let options = WorkflowResultOptions {
21981            poll_interval: Duration::ZERO,
21982            timeout: Duration::from_secs(1),
21983        };
21984
21985        let failed = WorkflowHandle {
21986            client: client.clone(),
21987            workflow_id: "wf-failed".to_string(),
21988            run_id: Some("run-failed".to_string()),
21989            workflow_type: "failure".to_string(),
21990        }
21991        .result(options)
21992        .await
21993        .expect_err("failed outcome");
21994        let Error::WorkflowFailed(failure) = failed else {
21995            panic!("expected WorkflowFailed");
21996        };
21997        assert_eq!(failure.workflow_id, "wf-failed");
21998        assert_eq!(failure.run_id.as_deref(), Some("run-failed"));
21999        assert_eq!(failure.failure_id.as_deref(), Some("failure-17"));
22000        assert_eq!(failure.failure_category.as_deref(), Some("application"));
22001        assert_eq!(failure.exception_type.as_deref(), Some("PaymentError"));
22002        assert_eq!(
22003            failure.exception_class.as_deref(),
22004            Some("billing::PaymentError")
22005        );
22006        assert_eq!(failure.non_retryable, Some(true));
22007
22008        for (workflow_id, expected_kind, expected_reason) in [
22009            (
22010                "wf-cancelled",
22011                WorkflowTerminalKind::Cancelled,
22012                "cleanup requested",
22013            ),
22014            (
22015                "wf-terminated",
22016                WorkflowTerminalKind::Terminated,
22017                "forced stop",
22018            ),
22019            (
22020                "wf-timed-out",
22021                WorkflowTerminalKind::TimedOut,
22022                "run_timeout",
22023            ),
22024        ] {
22025            let error = WorkflowHandle {
22026                client: client.clone(),
22027                workflow_id: workflow_id.to_string(),
22028                run_id: None,
22029                workflow_type: "terminal".to_string(),
22030            }
22031            .result(options)
22032            .await
22033            .expect_err("typed terminal outcome");
22034            let outcome = match error {
22035                Error::WorkflowCancelled(outcome) => outcome,
22036                Error::WorkflowTerminated(outcome) => outcome,
22037                Error::WorkflowTimedOut(outcome) => outcome,
22038                other => panic!("unexpected terminal error: {other}"),
22039            };
22040            assert_eq!(outcome.kind, expected_kind);
22041            assert_eq!(outcome.workflow_id, workflow_id);
22042            assert_eq!(outcome.reason, expected_reason);
22043        }
22044
22045        let wait_timeout = WorkflowHandle {
22046            client,
22047            workflow_id: "wf-waiting".to_string(),
22048            run_id: Some("run-waiting".to_string()),
22049            workflow_type: "waiting".to_string(),
22050        }
22051        .result(WorkflowResultOptions {
22052            poll_interval: Duration::ZERO,
22053            timeout: Duration::ZERO,
22054        })
22055        .await
22056        .expect_err("client wait timeout");
22057        let Error::WorkflowTimedOut(timeout) = wait_timeout else {
22058            panic!("expected typed client timeout");
22059        };
22060        assert_eq!(timeout.reason, "result_wait_timeout");
22061        assert_eq!(timeout.failure_category.as_deref(), Some("client_timeout"));
22062        assert_eq!(timeout.run_id.as_deref(), Some("run-waiting"));
22063    }
22064
22065    #[tokio::test]
22066    async fn workflow_result_follows_chain_and_selected_result_preserves_history() {
22067        let server = MockWorkerServer::start();
22068        let client = Client::builder(server.base_url())
22069            .timeout(Duration::from_secs(2))
22070            .build()
22071            .expect("client");
22072
22073        let handle = WorkflowHandle {
22074            client,
22075            workflow_id: "wf-selected".to_string(),
22076            run_id: Some("run-selected".to_string()),
22077            workflow_type: "selected".to_string(),
22078        };
22079        let options = WorkflowResultOptions {
22080            poll_interval: Duration::ZERO,
22081            timeout: Duration::from_secs(1),
22082        };
22083
22084        let current = handle
22085            .result(options)
22086            .await
22087            .expect("instance result follows the current run");
22088        assert_eq!(current, json!("current run output"));
22089
22090        let error = handle
22091            .result_selected_run(options)
22092            .await
22093            .expect_err("the selected run is cancelled even though the current run completed");
22094
22095        let Error::WorkflowCancelled(outcome) = error else {
22096            panic!("expected selected run cancellation");
22097        };
22098        assert_eq!(outcome.run_id.as_deref(), Some("run-selected"));
22099        assert_eq!(outcome.reason, "selected run cancelled");
22100        assert_eq!(
22101            server.request_count("/api/workflows/wf-selected/runs/run-selected"),
22102            1
22103        );
22104        assert_eq!(server.request_count("/api/workflows/wf-selected"), 1);
22105    }
22106
22107    #[tokio::test]
22108    async fn poll_responses_decode_http_conflict_drain_as_a_stable_stop() {
22109        let server = MockWorkerServer::draining_polls();
22110        let client = Client::builder(server.base_url())
22111            .timeout(Duration::from_secs(2))
22112            .build()
22113            .expect("client");
22114
22115        let workflow = client
22116            .poll_workflow_task_response("draining-worker", "rust-workers", Duration::ZERO)
22117            .await
22118            .expect("workflow drain response");
22119        let activity = client
22120            .poll_activity_task_response("draining-worker", "rust-workers", Duration::ZERO)
22121            .await
22122            .expect("activity drain response");
22123        let query = client
22124            .poll_query_task_response("draining-worker", "rust-workers", Duration::ZERO)
22125            .await
22126            .expect("query drain response");
22127
22128        for outcome in [workflow.outcome(), activity.outcome(), query.outcome()] {
22129            assert_eq!(
22130                outcome,
22131                WorkerPollOutcome::Stop {
22132                    poll_status: Some("draining".to_string()),
22133                    reason: Some("worker_draining".to_string()),
22134                }
22135            );
22136        }
22137
22138        assert!(client
22139            .poll_workflow_task("draining-worker", "rust-workers", Duration::ZERO)
22140            .await
22141            .expect("compatibility poll")
22142            .is_none());
22143    }
22144
22145    #[tokio::test]
22146    async fn managed_worker_honors_drain_stop_for_every_task_family() {
22147        let server = MockWorkerServer::draining_polls();
22148        let client = Client::builder(server.base_url())
22149            .timeout(Duration::from_secs(2))
22150            .build()
22151            .expect("client");
22152
22153        let mut workflow_worker = Worker::new(client.clone(), "rust-workers")
22154            .worker_id("draining-workflow-worker")
22155            .poll_timeout(Duration::ZERO);
22156        workflow_worker.register_workflow("counter", |_ctx, _args| async { Ok(Value::Null) });
22157        workflow_worker
22158            .run()
22159            .await
22160            .expect("workflow drain is a clean stop");
22161
22162        let mut activity_worker = Worker::new(client.clone(), "rust-workers")
22163            .worker_id("draining-activity-worker")
22164            .poll_timeout(Duration::ZERO);
22165        activity_worker.register_activity("write", |_ctx, _args| async { Ok(Value::Null) });
22166        activity_worker
22167            .run()
22168            .await
22169            .expect("activity drain is a clean stop");
22170
22171        let mut query_worker = Worker::new(client, "rust-workers")
22172            .worker_id("draining-query-worker")
22173            .poll_timeout(Duration::ZERO);
22174        query_worker.register_query("counter", "current", |_ctx, _args| async {
22175            Ok(Value::Null)
22176        });
22177        query_worker
22178            .run()
22179            .await
22180            .expect("query drain is a clean stop");
22181    }
22182
22183    #[tokio::test]
22184    async fn activity_cancellation_and_late_completion_remain_machine_readable() {
22185        let server = MockWorkerServer::start();
22186        let client = Client::builder(server.base_url())
22187            .timeout(Duration::from_secs(2))
22188            .build()
22189            .expect("client");
22190
22191        let heartbeat = client
22192            .heartbeat_activity_task(
22193                "activity-cancel",
22194                "attempt-cancel",
22195                "rust-worker",
22196                typed_fidelity_probe(),
22197            )
22198            .await
22199            .expect("cancellation heartbeat");
22200        assert!(heartbeat.cancel_requested);
22201        assert!(heartbeat.should_stop());
22202        assert_eq!(heartbeat.reason.as_deref(), Some("run_cancelled"));
22203        assert_eq!(heartbeat.run_closed_reason.as_deref(), Some("cancelled"));
22204        let heartbeat_body =
22205            server.request_body("/api/worker/activity-tasks/activity-cancel/heartbeat");
22206        assert_eq!(heartbeat_body["details"]["codec"], DEFAULT_CODEC);
22207        assert_eq!(
22208            decode_wire_avro_value(&heartbeat_body["details"], DEFAULT_CODEC)
22209                .expect("typed heartbeat details"),
22210            typed_fidelity_probe()
22211        );
22212
22213        let error = client
22214            .complete_activity_task(
22215                "activity-cancel",
22216                "attempt-cancel",
22217                "rust-worker",
22218                json!({"late":true}),
22219                DEFAULT_CODEC,
22220            )
22221            .await
22222            .expect_err("late completion must be refused");
22223        assert!(activity_task_rejection_is_final(&error));
22224        let Error::ActivityTaskRejected(rejection) = error else {
22225            panic!("expected typed activity rejection");
22226        };
22227        assert_eq!(rejection.status, 409);
22228        assert_eq!(rejection.reason, "run_cancelled");
22229        assert!(rejection.cancel_requested);
22230        assert_eq!(rejection.can_continue, Some(false));
22231    }
22232
22233    #[tokio::test]
22234    async fn managed_worker_survives_late_completion_and_restart_during_cancellation() {
22235        let server = MockWorkerServer::cancelled_activity();
22236        let client = Client::builder(server.base_url())
22237            .timeout(Duration::from_secs(2))
22238            .build()
22239            .expect("client");
22240        let cancellation_observed = Arc::new(AtomicBool::new(false));
22241        let observed = Arc::clone(&cancellation_observed);
22242        let mut worker = Worker::new(client.clone(), "rust-workers")
22243            .worker_id("rust-cancel-worker")
22244            .poll_timeout(Duration::from_millis(10));
22245        worker.register_activity("cancel-aware", move |ctx, _args| {
22246            let observed = Arc::clone(&observed);
22247            async move {
22248                let heartbeat = ctx.heartbeat(json!({"stage":"running"})).await?;
22249                observed.store(heartbeat.should_stop(), Ordering::SeqCst);
22250                Ok(json!({"late":"completion"}))
22251            }
22252        });
22253
22254        assert_eq!(
22255            worker.run_once().await.expect("cancelled attempt handled"),
22256            1
22257        );
22258        assert!(cancellation_observed.load(Ordering::SeqCst));
22259        assert_eq!(
22260            server.request_count("/api/worker/activity-tasks/activity-cancel/complete"),
22261            1
22262        );
22263
22264        let mut restarted = Worker::new(client, "rust-workers")
22265            .worker_id("rust-cancel-worker-restarted")
22266            .poll_timeout(Duration::from_millis(10));
22267        restarted.register_activity("cancel-aware", |_ctx, _args| async move { Ok(Value::Null) });
22268        assert_eq!(
22269            restarted
22270                .run_once()
22271                .await
22272                .expect("replacement worker continues polling"),
22273            0
22274        );
22275    }
22276
22277    #[tokio::test]
22278    async fn managed_worker_absorbs_selected_run_terminal_timeout_completion_race() {
22279        let response = r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"outcome":"completed","recorded":false,"run_id":"run-selected-timeout","run_status":"failed","created_task_ids":[],"reason":"run_timed_out"}"#;
22280        let server = MockWorkerServer::workflow_completion("409 Conflict", response);
22281        let client = Client::builder(server.base_url())
22282            .timeout(Duration::from_secs(2))
22283            .build()
22284            .expect("client");
22285
22286        let direct_error = client
22287            .complete_workflow_task(
22288                "workflow-timeout-task",
22289                "timeout-worker",
22290                3,
22291                vec![json!({
22292                    "type": "complete_workflow",
22293                    "result": fixture_envelope(Value::Null)
22294                })],
22295            )
22296            .await
22297            .expect_err("the low-level client preserves the completion rejection");
22298        let Error::Http { status, body } = direct_error else {
22299            panic!("expected the original HTTP completion rejection");
22300        };
22301        assert_eq!(status, reqwest::StatusCode::CONFLICT);
22302        assert_eq!(
22303            serde_json::from_str::<Value>(&body).expect("response body")["reason"],
22304            "run_timed_out"
22305        );
22306
22307        let mut worker = Worker::new(client, "rust-workers")
22308            .worker_id("timeout-worker")
22309            .poll_timeout(Duration::from_millis(10));
22310        worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
22311            Ok(json!({"late": "result"}))
22312        });
22313
22314        assert_eq!(
22315            worker
22316                .run_once()
22317                .await
22318                .expect("authoritative selected-run timeout settles the tick"),
22319            1
22320        );
22321        assert_eq!(
22322            server.request_count("/api/worker/workflow-tasks/workflow-timeout-task/complete"),
22323            2,
22324            "both the direct client proof and managed worker must see the rejection"
22325        );
22326    }
22327
22328    #[tokio::test]
22329    async fn managed_worker_does_not_swallow_nearby_completion_errors() {
22330        for (name, status, response) in [
22331            ("bare conflict", "409 Conflict", r#"{"message":"conflict"}"#),
22332            (
22333                "command was recorded",
22334                "409 Conflict",
22335                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":true,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22336            ),
22337            (
22338                "lease conflict",
22339                "409 Conflict",
22340                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"lease_expired"}"#,
22341            ),
22342            (
22343                "nonterminal run",
22344                "409 Conflict",
22345                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"waiting","reason":"run_timed_out"}"#,
22346            ),
22347            (
22348                "different selected run",
22349                "409 Conflict",
22350                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-reused-workflow-current","run_status":"failed","reason":"run_timed_out"}"#,
22351            ),
22352            (
22353                "different task attempt",
22354                "409 Conflict",
22355                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":4,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22356            ),
22357            (
22358                "authentication failure",
22359                "401 Unauthorized",
22360                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22361            ),
22362            (
22363                "authorization failure",
22364                "403 Forbidden",
22365                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22366            ),
22367            (
22368                "protocol failure",
22369                "400 Bad Request",
22370                r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.2","requested_version":"1.3"}"#,
22371            ),
22372            (
22373                "malformed command",
22374                "422 Unprocessable Entity",
22375                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22376            ),
22377            (
22378                "transient server failure",
22379                "503 Service Unavailable",
22380                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22381            ),
22382        ] {
22383            let server = MockWorkerServer::workflow_completion(status, response);
22384            let client = Client::builder(server.base_url())
22385                .timeout(Duration::from_secs(2))
22386                .build()
22387                .expect("client");
22388            let mut worker = Worker::new(client, "rust-workers")
22389                .worker_id("timeout-worker")
22390                .poll_timeout(Duration::from_millis(10));
22391            worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
22392                Ok(json!({"late": "result"}))
22393            });
22394
22395            let error = worker
22396                .run_once()
22397                .await
22398                .expect_err(&format!("{name} must remain an error"));
22399            assert!(
22400                matches!(error, Error::Http { .. } | Error::Protocol(_)),
22401                "{name} returned an unexpected error variant: {error}"
22402            );
22403        }
22404    }
22405
22406    #[tokio::test]
22407    async fn worker_deregistration_uses_worker_plane_method_path_headers_and_result() {
22408        let server = MockWorkerServer::start();
22409        let client = Client::builder(server.base_url())
22410            .worker_token(Some("worker-secret".to_string()))
22411            .namespace("orders")
22412            .timeout(Duration::from_secs(2))
22413            .build()
22414            .expect("client");
22415        let path = "/api/worker/registrations/worker%2F%CE%B1%20space";
22416
22417        let result = client
22418            .deregister_worker_registration("worker/α space")
22419            .await
22420            .expect("deregister worker registration");
22421
22422        assert_eq!(server.method_for(path).as_deref(), Some("DELETE"));
22423        assert_eq!(
22424            server.worker_protocol_for(path).as_deref(),
22425            Some(WORKER_PROTOCOL_VERSION)
22426        );
22427        assert_eq!(server.control_protocol_for(path), None);
22428        assert_eq!(server.namespace_for(path).as_deref(), Some("orders"));
22429        assert_eq!(
22430            server.authorization_for(path).as_deref(),
22431            Some("Bearer worker-secret")
22432        );
22433        assert_eq!(
22434            result,
22435            WorkerDeregistrationEnvelope {
22436                worker_id: "deregistered-worker".to_string(),
22437                outcome: "deregistered".to_string(),
22438                recovered_workflow_task_count: 2,
22439            }
22440        );
22441    }
22442
22443    #[tokio::test]
22444    async fn low_level_registration_rejects_update_validators_before_transport() {
22445        let server = MockWorkerServer::start();
22446        let client = Client::builder(server.base_url())
22447            .timeout(Duration::from_secs(2))
22448            .build()
22449            .expect("client");
22450
22451        for update_validators in [json!(["approve"]), json!("approve")] {
22452            let error = client
22453                .register_worker_with_command_contracts(
22454                    "validator-claiming-worker",
22455                    "rust-workers",
22456                    vec!["orders".to_string()],
22457                    vec![],
22458                    1,
22459                    1,
22460                    vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
22461                    json!({
22462                        "orders": {
22463                            "queries": ["current"],
22464                            "updates": ["approve"],
22465                            "update_validators": update_validators,
22466                        },
22467                    }),
22468                )
22469                .await
22470                .expect_err("unsupported validator claims must fail before registration");
22471
22472            let Error::UnsupportedUpdateValidators { workflow_type } = error else {
22473                panic!("expected typed unsupported-validator failure");
22474            };
22475            assert_eq!(workflow_type, "orders");
22476        }
22477        assert_eq!(server.request_count("/api/worker/register"), 0);
22478    }
22479
22480    #[tokio::test]
22481    async fn low_level_registration_preserves_query_and_update_contracts() {
22482        let server = MockWorkerServer::start();
22483        let client = Client::builder(server.base_url())
22484            .timeout(Duration::from_secs(2))
22485            .build()
22486            .expect("client");
22487        let contracts = json!({
22488            "orders": {
22489                "queries": ["current"],
22490                "updates": ["approve"],
22491                "update_validators": [],
22492            },
22493            "payments": {
22494                "queries": ["status"],
22495                "updates": ["capture"],
22496            },
22497        });
22498
22499        client
22500            .register_worker_with_command_contracts(
22501                "command-worker",
22502                "rust-workers",
22503                vec!["orders".to_string(), "payments".to_string()],
22504                vec![],
22505                1,
22506                1,
22507                vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
22508                contracts.clone(),
22509            )
22510            .await
22511            .expect("query and update contracts must remain supported");
22512
22513        assert_eq!(
22514            server.request_body("/api/worker/register")["workflow_command_contracts"],
22515            contracts
22516        );
22517    }
22518
22519    #[tokio::test]
22520    async fn role_scoped_tokens_are_never_used_for_the_opposite_plane() {
22521        let server = MockWorkerServer::start();
22522        let control_only = Client::builder(server.base_url())
22523            .control_token(Some("control-secret".to_string()))
22524            .build()
22525            .expect("control client");
22526
22527        let error = control_only
22528            .register_worker("worker", "queue", vec![], vec![], 1, 1)
22529            .await
22530            .expect_err("control token must not authorize a worker request");
22531        assert!(matches!(
22532            error,
22533            Error::MissingRoleCredentials { role: "worker", .. }
22534        ));
22535        assert_eq!(server.request_count("/api/worker/register"), 0);
22536
22537        let worker_only = Client::builder(server.base_url())
22538            .worker_token(Some("worker-secret".to_string()))
22539            .build()
22540            .expect("worker client");
22541        let error = worker_only
22542            .health()
22543            .await
22544            .expect_err("worker token must not authorize a control request");
22545        assert!(matches!(
22546            error,
22547            Error::MissingRoleCredentials {
22548                role: "control",
22549                ..
22550            }
22551        ));
22552        assert_eq!(server.request_count("/api/health"), 0);
22553    }
22554
22555    #[tokio::test]
22556    async fn shared_token_supports_worker_and_control_planes() {
22557        let server = MockWorkerServer::start();
22558        let client = Client::builder(server.base_url())
22559            .token(Some("shared-secret".to_string()))
22560            .build()
22561            .expect("client");
22562
22563        client.health().await.expect("control request");
22564        client
22565            .register_worker("worker", "queue", vec![], vec![], 1, 1)
22566            .await
22567            .expect("worker request");
22568
22569        assert_eq!(
22570            server.authorization_for("/api/health").as_deref(),
22571            Some("Bearer shared-secret")
22572        );
22573        assert_eq!(
22574            server.control_protocol_for("/api/health").as_deref(),
22575            Some(CONTROL_PLANE_VERSION)
22576        );
22577        assert_eq!(
22578            server.authorization_for("/api/worker/register").as_deref(),
22579            Some("Bearer shared-secret")
22580        );
22581        assert_eq!(
22582            server
22583                .worker_protocol_for("/api/worker/register")
22584                .as_deref(),
22585            Some(WORKER_PROTOCOL_VERSION)
22586        );
22587    }
22588
22589    #[tokio::test]
22590    async fn baseline_worker_endpoints_send_the_baseline_protocol() {
22591        let server = MockWorkerServer::start();
22592        let client = Client::builder(server.base_url())
22593            .timeout(Duration::from_secs(2))
22594            .build()
22595            .expect("client");
22596
22597        client
22598            .register_worker("capture-worker", "capture", vec![], vec![], 1, 1)
22599            .await
22600            .expect("register");
22601        client
22602            .heartbeat_worker("capture-worker", 1, 1)
22603            .await
22604            .expect("heartbeat");
22605        client
22606            .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
22607            .await
22608            .expect("workflow poll");
22609        client
22610            .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
22611            .await
22612            .expect("activity poll");
22613
22614        for path in [
22615            "/api/worker/register",
22616            "/api/worker/heartbeat",
22617            "/api/worker/workflow-tasks/poll",
22618            "/api/worker/activity-tasks/poll",
22619        ] {
22620            assert_eq!(
22621                server.worker_protocol_for(path).as_deref(),
22622                Some(WORKER_PROTOCOL_VERSION),
22623                "unexpected protocol for {path}"
22624            );
22625        }
22626
22627        assert_eq!(
22628            server.request_body("/api/worker/workflow-tasks/poll")["timeout_seconds"],
22629            1
22630        );
22631        assert_eq!(
22632            server.request_body("/api/worker/activity-tasks/poll")["timeout_seconds"],
22633            1
22634        );
22635        assert!(
22636            server.request_body("/api/worker/workflow-tasks/poll")["poll_request_id"]
22637                .as_str()
22638                .is_some_and(|id| id.starts_with("rust-workflow-poll-"))
22639        );
22640        assert!(
22641            server.request_body("/api/worker/activity-tasks/poll")["poll_request_id"]
22642                .as_str()
22643                .is_some_and(|id| id.starts_with("rust-activity-poll-"))
22644        );
22645    }
22646
22647    #[tokio::test]
22648    async fn query_task_endpoints_send_the_query_feature_protocol() {
22649        let server = MockWorkerServer::start();
22650        let client = Client::builder(server.base_url())
22651            .timeout(Duration::from_secs(2))
22652            .build()
22653            .expect("client");
22654
22655        client
22656            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
22657            .await
22658            .expect("query poll");
22659        client
22660            .complete_query_task(
22661                "query-capture",
22662                "capture-worker",
22663                1,
22664                json!(8),
22665                DEFAULT_CODEC,
22666            )
22667            .await
22668            .expect("query complete");
22669        client
22670            .fail_query_task(
22671                "query-capture",
22672                "capture-worker",
22673                1,
22674                "failed",
22675                "query_rejected",
22676                "QueryFailed",
22677            )
22678            .await
22679            .expect("query fail");
22680
22681        for path in [
22682            "/api/worker/query-tasks/poll",
22683            "/api/worker/query-tasks/query-capture/complete",
22684            "/api/worker/query-tasks/query-capture/fail",
22685        ] {
22686            assert_eq!(
22687                server.worker_protocol_for(path).as_deref(),
22688                Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
22689                "unexpected protocol for {path}"
22690            );
22691        }
22692
22693        assert_eq!(
22694            server.request_body("/api/worker/query-tasks/poll")["timeout_seconds"],
22695            1
22696        );
22697        assert!(
22698            server.request_body("/api/worker/query-tasks/poll")["poll_request_id"]
22699                .as_str()
22700                .is_some_and(|id| id.starts_with("rust-query-poll-"))
22701        );
22702    }
22703
22704    #[tokio::test]
22705    async fn disconnected_client_polls_retry_once_with_the_same_request_id() {
22706        let server = MockWorkerServer::transient_worker_failures();
22707        let client = Client::builder(server.base_url())
22708            .timeout(Duration::from_secs(2))
22709            .build()
22710            .expect("client");
22711
22712        client
22713            .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
22714            .await
22715            .expect("workflow poll retry");
22716        client
22717            .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
22718            .await
22719            .expect("activity poll retry");
22720        client
22721            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
22722            .await
22723            .expect("query poll retry");
22724
22725        for path in [
22726            "/api/worker/workflow-tasks/poll",
22727            "/api/worker/activity-tasks/poll",
22728            "/api/worker/query-tasks/poll",
22729        ] {
22730            let bodies = server.request_bodies(path);
22731            assert_eq!(bodies.len(), 2, "{path} must be retried once");
22732            assert_eq!(
22733                bodies[0]["poll_request_id"], bodies[1]["poll_request_id"],
22734                "{path} must preserve the request binding across retry"
22735            );
22736        }
22737    }
22738
22739    #[tokio::test]
22740    async fn worker_poll_retries_preserve_request_id_across_consecutive_failures() {
22741        let server = MockWorkerServer::consecutive_poll_failures(2);
22742        let client = Client::builder(server.base_url())
22743            .timeout(Duration::from_secs(2))
22744            .build()
22745            .expect("client");
22746        let mut worker = Worker::new(client, "capture")
22747            .worker_id("capture-worker")
22748            .poll_timeout(Duration::from_millis(10))
22749            .retry_policy(WorkerRetryPolicy {
22750                max_retries: 2,
22751                initial_backoff: Duration::from_millis(1),
22752                max_backoff: Duration::from_millis(1),
22753            });
22754        worker.register_workflow(
22755            "capture.workflow",
22756            |_ctx, _input| async move { Ok(Value::Null) },
22757        );
22758        worker.register_activity(
22759            "capture.activity",
22760            |_ctx, _input| async move { Ok(Value::Null) },
22761        );
22762        worker.register_query("capture.workflow", "current", |_ctx, _args| async move {
22763            Ok(Value::Null)
22764        });
22765
22766        assert_eq!(worker.run_once().await.expect("poll retries"), 0);
22767
22768        for path in [
22769            "/api/worker/workflow-tasks/poll",
22770            "/api/worker/activity-tasks/poll",
22771            "/api/worker/query-tasks/poll",
22772        ] {
22773            let bodies = server.request_bodies(path);
22774            assert_eq!(bodies.len(), 3, "{path} must use exactly two retries");
22775            assert!(
22776                bodies
22777                    .iter()
22778                    .all(|body| body["poll_request_id"] == bodies[0]["poll_request_id"]),
22779                "{path} must preserve one request binding across every retry"
22780            );
22781        }
22782    }
22783
22784    #[tokio::test]
22785    async fn query_protocol_rejection_from_older_server_is_typed() {
22786        let server = MockWorkerServer::reject_query_protocol();
22787        let client = Client::builder(server.base_url())
22788            .timeout(Duration::from_secs(2))
22789            .build()
22790            .expect("client");
22791
22792        let error = client
22793            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
22794            .await
22795            .expect_err("server below query protocol floor must reject");
22796        let Error::Protocol(failure) = error else {
22797            panic!("expected typed protocol failure");
22798        };
22799
22800        assert_eq!(failure.status, 400);
22801        assert_eq!(failure.reason, "unsupported_protocol_version");
22802        assert_eq!(failure.supported_version.as_deref(), Some("1.7"));
22803        assert_eq!(
22804            failure.requested_version.as_deref(),
22805            Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
22806        );
22807        assert_eq!(
22808            server
22809                .worker_protocol_for("/api/worker/query-tasks/poll")
22810                .as_deref(),
22811            Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
22812        );
22813    }
22814
22815    #[tokio::test]
22816    async fn run_once_without_query_handlers_keeps_pre_query_server_compatibility() {
22817        let server = MockWorkerServer::reject_query_protocol();
22818        let client = Client::builder(server.base_url())
22819            .timeout(Duration::from_secs(2))
22820            .build()
22821            .expect("client");
22822        let mut worker = Worker::new(client, "rust-workers")
22823            .worker_id("baseline-worker")
22824            .poll_timeout(Duration::from_millis(10));
22825
22826        worker.register_workflow("baseline.workflow", |_ctx, _input| async move {
22827            Ok(Value::Null)
22828        });
22829
22830        assert_eq!(worker.run_once().await.expect("baseline run once"), 0);
22831        assert_eq!(
22832            server
22833                .worker_protocol_for("/api/worker/workflow-tasks/poll")
22834                .as_deref(),
22835            Some(WORKER_PROTOCOL_VERSION)
22836        );
22837        assert_eq!(
22838            server.worker_protocol_for("/api/worker/query-tasks/poll"),
22839            None,
22840            "a worker without query handlers must not use the query-task endpoint"
22841        );
22842    }
22843
22844    #[tokio::test]
22845    async fn completion_time_query_rejection_is_typed_without_stopping_worker() {
22846        let server = MockWorkerServer::reject_query_completion();
22847        let client = Client::builder(server.base_url())
22848            .timeout(Duration::from_secs(2))
22849            .build()
22850            .expect("client");
22851
22852        let error = client
22853            .complete_query_task("query-late", "late-worker", 1, json!(8), DEFAULT_CODEC)
22854            .await
22855            .expect_err("expired completion must be rejected");
22856        let Error::QueryFailed(failure) = error else {
22857            panic!("expected typed query failure");
22858        };
22859        assert_eq!(failure.status, 409);
22860        assert_eq!(failure.reason, "query_task_timed_out");
22861
22862        let mut worker = Worker::new(client, "rust-workers")
22863            .worker_id("late-worker")
22864            .poll_timeout(Duration::from_millis(10));
22865        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
22866        worker.register_query(
22867            "counter",
22868            "current",
22869            |_ctx, _args| async move { Ok(json!(8)) },
22870        );
22871
22872        assert_eq!(worker.run_once().await.expect("late task is handled"), 1);
22873        assert_eq!(
22874            worker
22875                .run_once()
22876                .await
22877                .expect("worker continues after late completion"),
22878            0
22879        );
22880        assert_eq!(
22881            server.request_count("/api/worker/query-tasks/query-late/complete"),
22882            2
22883        );
22884        assert_eq!(
22885            server.request_count("/api/worker/query-tasks/query-late/fail"),
22886            0,
22887            "a server completion rejection must not be reported as an encoding failure"
22888        );
22889    }
22890
22891    #[tokio::test]
22892    async fn normal_shutdown_joins_pollers_and_deregisters_once() {
22893        let server = MockWorkerServer::start();
22894        let client = Client::builder(server.base_url())
22895            .timeout(Duration::from_secs(2))
22896            .build()
22897            .expect("client");
22898        let mut worker = Worker::new(client, "rust-workers")
22899            .worker_id("joined-worker")
22900            .poll_timeout(Duration::from_millis(10));
22901        worker.register_workflow(
22902            "joined.workflow",
22903            |_ctx, _input| async move { Ok(Value::Null) },
22904        );
22905        worker.register_activity(
22906            "joined.activity",
22907            |_ctx, _input| async move { Ok(Value::Null) },
22908        );
22909        worker.register_query("joined.workflow", "state", |_ctx, _input| async move {
22910            Ok(Value::Null)
22911        });
22912
22913        worker
22914            .run_until(tokio::time::sleep(Duration::from_millis(20)))
22915            .await
22916            .expect("normal shutdown");
22917
22918        let deregistration_path = "/api/worker/registrations/mock-worker";
22919        assert_eq!(server.request_count(deregistration_path), 1);
22920        for poll_path in [
22921            "/api/worker/workflow-tasks/poll",
22922            "/api/worker/activity-tasks/poll",
22923            "/api/worker/query-tasks/poll",
22924        ] {
22925            assert!(server.request_count(poll_path) > 0, "missing {poll_path}");
22926        }
22927        assert_eq!(
22928            server.captured_paths().last().map(String::as_str),
22929            Some(deregistration_path),
22930            "deregistration must start only after every poller has joined"
22931        );
22932    }
22933
22934    #[tokio::test]
22935    async fn registration_failure_does_not_deregister() {
22936        let server = MockWorkerServer::rejected_registration();
22937        let client = Client::builder(server.base_url())
22938            .timeout(Duration::from_secs(2))
22939            .build()
22940            .expect("client");
22941        let worker = Worker::new(client, "rust-workers").worker_id("never-registered");
22942
22943        let error = worker
22944            .run_until(async {})
22945            .await
22946            .expect_err("registration must fail");
22947        assert!(matches!(
22948            error,
22949            Error::Http {
22950                status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
22951                ..
22952            }
22953        ));
22954        assert!(server
22955            .captured_paths()
22956            .iter()
22957            .all(|path| !path.starts_with("/api/worker/registrations/")));
22958    }
22959
22960    #[tokio::test]
22961    async fn protocol_116_server_rejects_occurrence_identity_worker_registration() {
22962        let server = MockWorkerServer::rejected_registration_protocol();
22963        let client = Client::builder(server.base_url())
22964            .timeout(Duration::from_secs(2))
22965            .build()
22966            .expect("client");
22967        let worker = Worker::new(client, "rust-workers").worker_id("protocol-117-worker");
22968
22969        let error = worker
22970            .run_until(async {})
22971            .await
22972            .expect_err("a protocol 1.16 server must reject this worker");
22973        let Error::Protocol(failure) = error else {
22974            panic!("expected typed protocol rejection");
22975        };
22976        assert_eq!(failure.reason, "unsupported_protocol_version");
22977        assert_eq!(failure.supported_version.as_deref(), Some("1.16"));
22978        assert_eq!(failure.requested_version.as_deref(), Some("1.17"));
22979        assert_eq!(
22980            server
22981                .worker_protocol_for("/api/worker/register")
22982                .as_deref(),
22983            Some(WORKER_PROTOCOL_VERSION)
22984        );
22985    }
22986
22987    #[tokio::test]
22988    async fn declined_registration_does_not_deregister() {
22989        let server = MockWorkerServer::declined_registration();
22990        let client = Client::builder(server.base_url())
22991            .timeout(Duration::from_secs(2))
22992            .build()
22993            .expect("client");
22994        let worker = Worker::new(client, "rust-workers").worker_id("declined-worker");
22995
22996        let error = worker
22997            .run_until(async {})
22998            .await
22999            .expect_err("declined registration must fail");
23000        assert!(matches!(error, Error::WorkerLoop(_)));
23001        assert!(error.to_string().contains("was not accepted"));
23002        assert!(server
23003            .captured_paths()
23004            .iter()
23005            .all(|path| !path.starts_with("/api/worker/registrations/")));
23006    }
23007
23008    #[tokio::test]
23009    async fn deregistration_http_failure_is_returned_after_normal_shutdown() {
23010        let server = MockWorkerServer::rejected_deregistration();
23011        let client = Client::builder(server.base_url())
23012            .timeout(Duration::from_secs(2))
23013            .build()
23014            .expect("client");
23015        let worker = Worker::new(client, "rust-workers").worker_id("forbidden-cleanup");
23016
23017        let error = worker
23018            .run_until(async {})
23019            .await
23020            .expect_err("deregistration must fail");
23021        assert!(matches!(
23022            error,
23023            Error::Http {
23024                status: reqwest::StatusCode::FORBIDDEN,
23025                ..
23026            }
23027        ));
23028        assert_eq!(
23029            server.request_count("/api/worker/registrations/mock-worker"),
23030            1
23031        );
23032    }
23033
23034    #[tokio::test]
23035    async fn deregistration_protocol_failure_is_returned_after_normal_shutdown() {
23036        let server = MockWorkerServer::rejected_deregistration_protocol();
23037        let client = Client::builder(server.base_url())
23038            .timeout(Duration::from_secs(2))
23039            .build()
23040            .expect("client");
23041        let worker = Worker::new(client, "rust-workers").worker_id("protocol-cleanup");
23042
23043        let error = worker
23044            .run_until(async {})
23045            .await
23046            .expect_err("protocol rejection must fail shutdown");
23047        let Error::Protocol(failure) = error else {
23048            panic!("expected typed protocol failure");
23049        };
23050        assert_eq!(failure.reason, "unsupported_protocol_version");
23051        assert_eq!(
23052            failure.requested_version.as_deref(),
23053            Some(WORKER_PROTOCOL_VERSION)
23054        );
23055        assert_eq!(
23056            server.request_count("/api/worker/registrations/mock-worker"),
23057            1
23058        );
23059    }
23060
23061    #[tokio::test]
23062    async fn primary_poller_error_retains_deregistration_failure_context() {
23063        let server = MockWorkerServer::unauthorized_polls_and_rejected_deregistration();
23064        let client = Client::builder(server.base_url())
23065            .timeout(Duration::from_secs(2))
23066            .build()
23067            .expect("client");
23068        let mut worker = Worker::new(client, "rust-workers")
23069            .worker_id("combined-failure")
23070            .poll_timeout(Duration::from_millis(10));
23071        worker.register_workflow("combined.workflow", |_ctx, _input| async move {
23072            Ok(Value::Null)
23073        });
23074
23075        let error = worker
23076            .run()
23077            .await
23078            .expect_err("worker and cleanup must fail");
23079        let summary = error.to_string();
23080        assert!(summary.contains("authentication_failed"));
23081        assert!(summary.contains("worker cannot deregister"));
23082        let Error::WorkerShutdown {
23083            primary,
23084            deregistration,
23085        } = error
23086        else {
23087            panic!("expected combined worker shutdown error");
23088        };
23089        assert!(matches!(
23090            *primary,
23091            Error::Http {
23092                status: reqwest::StatusCode::UNAUTHORIZED,
23093                ..
23094            }
23095        ));
23096        assert!(matches!(
23097            *deregistration,
23098            Error::Http {
23099                status: reqwest::StatusCode::FORBIDDEN,
23100                ..
23101            }
23102        ));
23103        assert_eq!(
23104            server.request_count("/api/worker/registrations/mock-worker"),
23105            1
23106        );
23107    }
23108
23109    #[tokio::test]
23110    async fn activity_only_worker_can_shutdown_without_workflow_poller() {
23111        let server = MockWorkerServer::start();
23112        let client = Client::builder(server.base_url())
23113            .timeout(Duration::from_secs(2))
23114            .build()
23115            .expect("client");
23116        let mut worker = Worker::new(client, "rust-workers")
23117            .worker_id("activity-only-worker")
23118            .poll_timeout(Duration::from_millis(10));
23119
23120        worker.register_activity(
23121            "activity.only",
23122            |_ctx, _args| async move { Ok(Value::Null) },
23123        );
23124
23125        worker.run_until(async {}).await.expect("run worker");
23126    }
23127
23128    #[tokio::test]
23129    async fn workflow_only_worker_can_shutdown_without_activity_poller() {
23130        let server = MockWorkerServer::start();
23131        let client = Client::builder(server.base_url())
23132            .timeout(Duration::from_secs(2))
23133            .build()
23134            .expect("client");
23135        let mut worker = Worker::new(client, "rust-workers")
23136            .worker_id("workflow-only-worker")
23137            .poll_timeout(Duration::from_millis(10));
23138
23139        worker.register_workflow(
23140            "workflow.only",
23141            |_ctx, _input| async move { Ok(Value::Null) },
23142        );
23143
23144        worker.run_until(async {}).await.expect("run worker");
23145    }
23146
23147    #[tokio::test]
23148    async fn worker_heartbeat_observer_receives_server_acknowledgements() {
23149        let server = MockWorkerServer::start();
23150        let client = Client::builder(server.base_url())
23151            .timeout(Duration::from_secs(2))
23152            .build()
23153            .expect("client");
23154        let observations = Arc::new(Mutex::new(Vec::new()));
23155        let observed = Arc::clone(&observations);
23156        let mut worker = Worker::new(client, "rust-workers")
23157            .worker_id("observed-heartbeat-worker")
23158            .poll_timeout(Duration::from_millis(10))
23159            .on_worker_heartbeat(move |observation| {
23160                observed
23161                    .lock()
23162                    .expect("heartbeat observations")
23163                    .push(observation.clone());
23164            });
23165
23166        worker.register_workflow("workflow.observed", |_ctx, _input| async move {
23167            Ok(Value::Null)
23168        });
23169        let acknowledged = Arc::clone(&observations);
23170        worker
23171            .run_until(async move {
23172                tokio::time::timeout(Duration::from_secs(2), async move {
23173                    loop {
23174                        if !acknowledged
23175                            .lock()
23176                            .expect("heartbeat observations")
23177                            .is_empty()
23178                        {
23179                            break;
23180                        }
23181                        tokio::time::sleep(Duration::from_millis(1)).await;
23182                    }
23183                })
23184                .await
23185                .expect("heartbeat acknowledgement within timeout");
23186            })
23187            .await
23188            .expect("run worker");
23189
23190        let observations = observations.lock().expect("heartbeat observations");
23191        let first = observations.first().expect("heartbeat acknowledgement");
23192        assert_eq!(first.worker_id, "observed-heartbeat-worker");
23193        assert_eq!(first.task_queue, "rust-workers");
23194        assert!(first.acknowledged_at_unix_millis > 0);
23195        assert_eq!(first.acknowledgement, json!({}));
23196    }
23197
23198    #[tokio::test]
23199    async fn delayed_worker_heartbeat_keeps_cadence_and_pollers_live() {
23200        let server = MockWorkerServer::delayed_heartbeat_worker();
23201        let client = Client::builder(server.base_url())
23202            .timeout(Duration::from_secs(3))
23203            .build()
23204            .expect("client");
23205        let observations = Arc::new(Mutex::new(Vec::new()));
23206        let observed = Arc::clone(&observations);
23207        let mut worker = Worker::new(client, "rust-snapshot-workers")
23208            .worker_id("rust-snapshot-worker")
23209            .poll_timeout(Duration::from_millis(10))
23210            .on_worker_heartbeat(move |observation| {
23211                observed
23212                    .lock()
23213                    .expect("heartbeat observations")
23214                    .push(observation.clone());
23215            });
23216
23217        worker.register_workflow("snapshot", |ctx, _input| async move {
23218            ctx.wait_signal("finish").await?;
23219            Ok(json!({"status": "finished"}))
23220        });
23221        worker.register_query("snapshot", "current", |ctx, _args| async move {
23222            Ok(json!(ctx
23223                .signals("increment")
23224                .iter()
23225                .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
23226                .sum::<i64>()))
23227        });
23228        worker.register_activity("cancel-aware", |_ctx, _args| async move {
23229            Ok(json!({"late": "completion"}))
23230        });
23231
23232        worker
23233            .run_until(tokio::time::sleep(Duration::from_millis(3_800)))
23234            .await
23235            .expect("delayed heartbeat must allow a clean worker shutdown");
23236
23237        let observations = observations.lock().expect("heartbeat observations");
23238        assert!(
23239            observations.len() >= 3,
23240            "the immediate heartbeat, delayed acknowledgement, and next cadence heartbeat must complete"
23241        );
23242        assert!(
23243            observations.windows(2).all(|pair| {
23244                pair[1].acknowledged_at_unix_millis
23245                    .saturating_sub(pair[0].acknowledged_at_unix_millis)
23246                    >= 850
23247            }),
23248            "successful acknowledgements must not catch up faster than the advertised one-second cadence: {observations:?}"
23249        );
23250        drop(observations);
23251
23252        let heartbeat_times = server.request_times("/api/worker/heartbeat");
23253        let delayed_request_at = *heartbeat_times
23254            .get(1)
23255            .expect("intentionally delayed heartbeat request");
23256        let delay_window_start = delayed_request_at + Duration::from_millis(100);
23257        let delay_window_end = delayed_request_at + Duration::from_millis(1_400);
23258        for path in [
23259            "/api/worker/workflow-tasks/poll",
23260            "/api/worker/activity-tasks/poll",
23261            "/api/worker/query-tasks/poll",
23262        ] {
23263            assert!(
23264                server
23265                    .request_times(path)
23266                    .iter()
23267                    .any(|received_at| *received_at >= delay_window_start
23268                        && *received_at <= delay_window_end),
23269                "{path} must keep polling while a heartbeat acknowledgement is delayed"
23270            );
23271        }
23272        assert!(
23273            server.request_count("/api/worker/workflow-tasks/snapshot-wait-3/fail") >= 1,
23274            "workflow work must be settled"
23275        );
23276        assert!(
23277            server.request_count("/api/worker/activity-tasks/activity-cancel/complete") >= 1,
23278            "activity work must be settled"
23279        );
23280        assert!(
23281            server.request_count("/api/worker/query-tasks/snapshot-current/complete") >= 1,
23282            "query work must be settled"
23283        );
23284    }
23285
23286    #[tokio::test]
23287    async fn retried_worker_heartbeat_restarts_the_advertised_cadence() {
23288        let server = MockWorkerServer::heartbeat_retry_worker();
23289        let client = Client::builder(server.base_url())
23290            .timeout(Duration::from_secs(2))
23291            .build()
23292            .expect("client");
23293        let observations = Arc::new(Mutex::new(Vec::new()));
23294        let observed = Arc::clone(&observations);
23295        let worker = Worker::new(client, "rust-workers")
23296            .worker_id("heartbeat-retry-worker")
23297            .retry_policy(WorkerRetryPolicy {
23298                max_retries: 1,
23299                initial_backoff: Duration::from_millis(300),
23300                max_backoff: Duration::from_millis(300),
23301            })
23302            .on_worker_heartbeat(move |observation| {
23303                observed
23304                    .lock()
23305                    .expect("heartbeat observations")
23306                    .push(observation.clone());
23307            });
23308
23309        worker
23310            .run_until(tokio::time::sleep(Duration::from_millis(2_700)))
23311            .await
23312            .expect("retryable heartbeat failure must remain bounded and recover");
23313
23314        let observations = observations.lock().expect("heartbeat observations");
23315        assert!(observations.len() >= 3, "heartbeat retry must recover");
23316        assert!(
23317            observations.windows(2).all(|pair| {
23318                pair[1]
23319                    .acknowledged_at_unix_millis
23320                    .saturating_sub(pair[0].acknowledged_at_unix_millis)
23321                    >= 850
23322            }),
23323            "a successful retry must start a fresh advertised cadence: {observations:?}"
23324        );
23325        assert_eq!(
23326            server.request_count("/api/worker/heartbeat"),
23327            observations.len() + 1,
23328            "one retryable failure must add exactly one bounded request"
23329        );
23330    }
23331
23332    #[tokio::test]
23333    async fn query_enabled_worker_ignores_unmatched_signals_then_completes_once() {
23334        let server = MockWorkerServer::waiting_query_worker();
23335        let client = Client::builder(server.base_url())
23336            .timeout(Duration::from_secs(2))
23337            .build()
23338            .expect("client");
23339        let observations = Arc::new(Mutex::new(Vec::new()));
23340        let observed = Arc::clone(&observations);
23341        let mut worker = Worker::new(client, "rust-snapshot-workers")
23342            .worker_id("rust-snapshot-worker")
23343            .poll_timeout(Duration::from_millis(10))
23344            .on_worker_heartbeat(move |observation| {
23345                observed
23346                    .lock()
23347                    .expect("heartbeat observations")
23348                    .push(observation.clone());
23349            });
23350
23351        worker.register_workflow("snapshot", |ctx, _input| async move {
23352            ctx.wait_signal("finish").await?;
23353            Ok(json!({"status": "finished"}))
23354        });
23355        worker.register_query("snapshot", "current", |ctx, _args| async move {
23356            let current = ctx
23357                .signals("increment")
23358                .iter()
23359                .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
23360                .sum::<i64>();
23361            Ok(json!(current))
23362        });
23363        worker.register_update("snapshot", "replace", |_ctx, args| async move { Ok(args) });
23364
23365        worker
23366            .run_until(tokio::time::sleep(Duration::from_millis(3_200)))
23367            .await
23368            .expect("pending workflow and query poller must remain live until shutdown");
23369
23370        assert!(
23371            observations.lock().expect("heartbeat observations").len() >= 4,
23372            "the immediate heartbeat and at least three advertised one-second intervals must be acknowledged"
23373        );
23374        assert!(
23375            server.request_count("/api/worker/workflow-tasks/poll") >= 3,
23376            "workflow polling must continue after empty replay acknowledgements"
23377        );
23378        assert!(
23379            server.request_count("/api/worker/query-tasks/poll") >= 2,
23380            "query polling must continue after serving the current query"
23381        );
23382        assert_eq!(
23383            server.request_body("/api/worker/register")["capabilities"],
23384            json!([
23385                CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY,
23386                DURABLE_SELECTION_CAPABILITY,
23387                MEMO_UPSERTS_CAPABILITY,
23388                TYPED_SEARCH_ATTRIBUTES_CAPABILITY,
23389                QUERY_TASKS_CAPABILITY,
23390                WORKFLOW_UPDATES_CAPABILITY,
23391                MESSAGE_STREAMS_CAPABILITY
23392            ])
23393        );
23394        assert_eq!(
23395            server.request_body("/api/worker/register")["workflow_command_contracts"]["snapshot"],
23396            json!({
23397                "queries": ["current"],
23398                "query_contracts": [],
23399                "signals": [],
23400                "signal_contracts": [],
23401                "updates": ["replace"],
23402                "update_contracts": [],
23403                "update_validators": [],
23404            })
23405        );
23406
23407        let opened = server.request_body("/api/worker/workflow-tasks/snapshot-open/complete");
23408        assert_eq!(
23409            opened["commands"],
23410            json!([{
23411                "type": "open_signal_wait",
23412                "signal_name": "finish",
23413            }])
23414        );
23415
23416        for task_id in ["snapshot-wait-3", "snapshot-wait-5"] {
23417            let fail_path = format!("/api/worker/workflow-tasks/{task_id}/fail");
23418            let completion_path = format!("/api/worker/workflow-tasks/{task_id}/complete");
23419            let failure = server.request_body(&fail_path);
23420            assert_eq!(
23421                failure["failure"]["type"],
23422                WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE
23423            );
23424            assert_eq!(server.request_count(&completion_path), 0);
23425        }
23426
23427        let query_completion =
23428            server.request_body("/api/worker/query-tasks/snapshot-current/complete");
23429        assert_eq!(query_completion["result"], json!(8));
23430
23431        let terminal_path = "/api/worker/workflow-tasks/snapshot-finish/complete";
23432        assert_eq!(
23433            server.request_count(terminal_path),
23434            1,
23435            "the matching signal must settle the workflow exactly once"
23436        );
23437        let terminal = server.request_body(terminal_path);
23438        assert_eq!(terminal["commands"].as_array().map(Vec::len), Some(1));
23439        assert_eq!(terminal["commands"][0]["type"], "complete_workflow");
23440        assert_eq!(
23441            decode_wire_value(&terminal["commands"][0]["result"], DEFAULT_CODEC)
23442                .expect("terminal workflow result"),
23443            json!({"status": "finished"})
23444        );
23445    }
23446
23447    #[tokio::test]
23448    async fn worker_retries_poll_and_heartbeat_transport_failures_independently() {
23449        let server = MockWorkerServer::transient_worker_failures();
23450        let client = Client::builder(server.base_url())
23451            .timeout(Duration::from_secs(2))
23452            .build()
23453            .expect("client");
23454        let mut worker = Worker::new(client, "rust-workers")
23455            .worker_id("retry-worker")
23456            .poll_timeout(Duration::from_millis(10))
23457            .retry_policy(WorkerRetryPolicy {
23458                max_retries: 2,
23459                initial_backoff: Duration::from_millis(1),
23460                max_backoff: Duration::from_millis(1),
23461            });
23462        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
23463        worker.register_activity(
23464            "counter.activity",
23465            |_ctx, _input| async move { Ok(Value::Null) },
23466        );
23467        worker.register_query(
23468            "counter",
23469            "current",
23470            |_ctx, _args| async move { Ok(json!(8)) },
23471        );
23472
23473        worker
23474            .run_until(tokio::time::sleep(Duration::from_millis(75)))
23475            .await
23476            .expect("transient failures must not stop the worker");
23477
23478        for path in [
23479            "/api/worker/heartbeat",
23480            "/api/worker/workflow-tasks/poll",
23481            "/api/worker/activity-tasks/poll",
23482            "/api/worker/query-tasks/poll",
23483        ] {
23484            assert!(
23485                server.request_count(path) >= 2,
23486                "{path} must continue after its transient failure"
23487            );
23488        }
23489    }
23490
23491    #[tokio::test]
23492    async fn worker_bounds_transport_retries() {
23493        let server = MockWorkerServer::unavailable_polls();
23494        let client = Client::builder(server.base_url())
23495            .timeout(Duration::from_secs(2))
23496            .build()
23497            .expect("client");
23498        let mut worker = Worker::new(client, "rust-workers")
23499            .worker_id("bounded-retry-worker")
23500            .poll_timeout(Duration::from_millis(10))
23501            .retry_policy(WorkerRetryPolicy {
23502                max_retries: 2,
23503                initial_backoff: Duration::from_millis(1),
23504                max_backoff: Duration::from_millis(1),
23505            });
23506        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
23507
23508        let error = worker.run().await.expect_err("retry bound must terminate");
23509        assert!(matches!(error, Error::Transport(_)));
23510        assert_eq!(
23511            server.request_count("/api/worker/workflow-tasks/poll"),
23512            3,
23513            "one initial request plus exactly two retries"
23514        );
23515    }
23516
23517    #[tokio::test]
23518    async fn worker_retry_policy_can_disable_poll_retries() {
23519        let server = MockWorkerServer::unavailable_polls();
23520        let client = Client::builder(server.base_url())
23521            .timeout(Duration::from_secs(2))
23522            .build()
23523            .expect("client");
23524        let mut worker = Worker::new(client, "rust-workers")
23525            .worker_id("no-retry-worker")
23526            .poll_timeout(Duration::from_millis(10))
23527            .retry_policy(WorkerRetryPolicy {
23528                max_retries: 0,
23529                initial_backoff: Duration::from_millis(1),
23530                max_backoff: Duration::from_millis(1),
23531            });
23532        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
23533
23534        let error = worker
23535            .run_once()
23536            .await
23537            .expect_err("disabled retries must return the first transport failure");
23538        assert!(matches!(error, Error::Transport(_)));
23539        assert_eq!(
23540            server.request_count("/api/worker/workflow-tasks/poll"),
23541            1,
23542            "max_retries=0 must send only the initial request"
23543        );
23544    }
23545
23546    #[tokio::test]
23547    async fn worker_does_not_retry_authentication_failures() {
23548        let server = MockWorkerServer::unauthorized_polls();
23549        let client = Client::builder(server.base_url())
23550            .timeout(Duration::from_secs(2))
23551            .build()
23552            .expect("client");
23553        let mut worker = Worker::new(client, "rust-workers")
23554            .worker_id("unauthorized-worker")
23555            .poll_timeout(Duration::from_millis(10));
23556        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
23557
23558        let error = worker
23559            .run()
23560            .await
23561            .expect_err("authentication must terminate");
23562        let Error::Http { status, body } = error else {
23563            panic!("expected stable HTTP authentication error");
23564        };
23565        assert_eq!(status, reqwest::StatusCode::UNAUTHORIZED);
23566        assert!(body.contains("authentication_failed"));
23567        assert_eq!(
23568            server.request_count("/api/worker/workflow-tasks/poll"),
23569            1,
23570            "authentication failures must not be retried"
23571        );
23572    }
23573
23574    #[derive(Clone, Debug)]
23575    struct CapturedRequest {
23576        method: String,
23577        path: String,
23578        authorization: Option<String>,
23579        namespace: Option<String>,
23580        worker_protocol: Option<String>,
23581        control_protocol: Option<String>,
23582        body: String,
23583        received_at: Instant,
23584    }
23585
23586    struct MockWorkerServer {
23587        addr: SocketAddr,
23588        stop: Arc<AtomicBool>,
23589        requests: Arc<Mutex<Vec<CapturedRequest>>>,
23590        thread: Option<thread::JoinHandle<()>>,
23591    }
23592
23593    #[derive(Clone, Copy, Default)]
23594    struct MockWorkerBehavior {
23595        reject_query_protocol: bool,
23596        reject_query_completion: bool,
23597        waiting_query_worker: bool,
23598        decline_registration: bool,
23599        complete_named_signal: bool,
23600        poll_failures_per_path: usize,
23601        heartbeat_failures: usize,
23602        heartbeat_failure_request: Option<usize>,
23603        delayed_heartbeat_request: Option<usize>,
23604        heartbeat_response_delay: Duration,
23605        concurrent_requests: bool,
23606        unauthorized_polls: bool,
23607        reject_registration: bool,
23608        reject_registration_protocol: bool,
23609        reject_deregistration: bool,
23610        reject_deregistration_protocol: bool,
23611        cancelled_activity: bool,
23612        draining_polls: bool,
23613        invalid_task_payload_codec: Option<InvalidTaskPayloadCodec>,
23614        workflow_completion_status: Option<&'static str>,
23615        workflow_completion_body: Option<&'static str>,
23616    }
23617
23618    impl MockWorkerServer {
23619        fn start() -> Self {
23620            Self::start_with_behavior(MockWorkerBehavior::default())
23621        }
23622
23623        fn reject_query_protocol() -> Self {
23624            Self::start_with_behavior(MockWorkerBehavior {
23625                reject_query_protocol: true,
23626                ..MockWorkerBehavior::default()
23627            })
23628        }
23629
23630        fn reject_query_completion() -> Self {
23631            Self::start_with_behavior(MockWorkerBehavior {
23632                reject_query_completion: true,
23633                ..MockWorkerBehavior::default()
23634            })
23635        }
23636
23637        fn waiting_query_worker() -> Self {
23638            Self::start_with_behavior(MockWorkerBehavior {
23639                waiting_query_worker: true,
23640                complete_named_signal: true,
23641                ..MockWorkerBehavior::default()
23642            })
23643        }
23644
23645        fn transient_worker_failures() -> Self {
23646            Self::start_with_behavior(MockWorkerBehavior {
23647                poll_failures_per_path: 1,
23648                heartbeat_failures: 1,
23649                ..MockWorkerBehavior::default()
23650            })
23651        }
23652
23653        fn consecutive_poll_failures(count: usize) -> Self {
23654            Self::start_with_behavior(MockWorkerBehavior {
23655                poll_failures_per_path: count,
23656                ..MockWorkerBehavior::default()
23657            })
23658        }
23659
23660        fn delayed_heartbeat_worker() -> Self {
23661            Self::start_with_behavior(MockWorkerBehavior {
23662                waiting_query_worker: true,
23663                delayed_heartbeat_request: Some(2),
23664                heartbeat_response_delay: Duration::from_millis(1_500),
23665                concurrent_requests: true,
23666                cancelled_activity: true,
23667                ..MockWorkerBehavior::default()
23668            })
23669        }
23670
23671        fn heartbeat_retry_worker() -> Self {
23672            Self::start_with_behavior(MockWorkerBehavior {
23673                waiting_query_worker: true,
23674                heartbeat_failure_request: Some(2),
23675                concurrent_requests: true,
23676                ..MockWorkerBehavior::default()
23677            })
23678        }
23679
23680        fn unavailable_polls() -> Self {
23681            Self::start_with_behavior(MockWorkerBehavior {
23682                poll_failures_per_path: usize::MAX,
23683                ..MockWorkerBehavior::default()
23684            })
23685        }
23686
23687        fn unauthorized_polls() -> Self {
23688            Self::start_with_behavior(MockWorkerBehavior {
23689                unauthorized_polls: true,
23690                ..MockWorkerBehavior::default()
23691            })
23692        }
23693
23694        fn rejected_registration() -> Self {
23695            Self::start_with_behavior(MockWorkerBehavior {
23696                reject_registration: true,
23697                ..MockWorkerBehavior::default()
23698            })
23699        }
23700
23701        fn rejected_registration_protocol() -> Self {
23702            Self::start_with_behavior(MockWorkerBehavior {
23703                reject_registration_protocol: true,
23704                ..MockWorkerBehavior::default()
23705            })
23706        }
23707
23708        fn declined_registration() -> Self {
23709            Self::start_with_behavior(MockWorkerBehavior {
23710                decline_registration: true,
23711                ..MockWorkerBehavior::default()
23712            })
23713        }
23714
23715        fn rejected_deregistration() -> Self {
23716            Self::start_with_behavior(MockWorkerBehavior {
23717                reject_deregistration: true,
23718                ..MockWorkerBehavior::default()
23719            })
23720        }
23721
23722        fn rejected_deregistration_protocol() -> Self {
23723            Self::start_with_behavior(MockWorkerBehavior {
23724                reject_deregistration_protocol: true,
23725                ..MockWorkerBehavior::default()
23726            })
23727        }
23728
23729        fn unauthorized_polls_and_rejected_deregistration() -> Self {
23730            Self::start_with_behavior(MockWorkerBehavior {
23731                unauthorized_polls: true,
23732                reject_deregistration: true,
23733                ..MockWorkerBehavior::default()
23734            })
23735        }
23736
23737        fn cancelled_activity() -> Self {
23738            Self::start_with_behavior(MockWorkerBehavior {
23739                cancelled_activity: true,
23740                ..MockWorkerBehavior::default()
23741            })
23742        }
23743
23744        fn draining_polls() -> Self {
23745            Self::start_with_behavior(MockWorkerBehavior {
23746                draining_polls: true,
23747                ..MockWorkerBehavior::default()
23748            })
23749        }
23750
23751        fn invalid_task_payload_codec(codec: InvalidTaskPayloadCodec) -> Self {
23752            Self::start_with_behavior(MockWorkerBehavior {
23753                invalid_task_payload_codec: Some(codec),
23754                ..MockWorkerBehavior::default()
23755            })
23756        }
23757
23758        fn workflow_completion(status: &'static str, body: &'static str) -> Self {
23759            Self::start_with_behavior(MockWorkerBehavior {
23760                workflow_completion_status: Some(status),
23761                workflow_completion_body: Some(body),
23762                ..MockWorkerBehavior::default()
23763            })
23764        }
23765
23766        fn start_with_behavior(behavior: MockWorkerBehavior) -> Self {
23767            let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock server");
23768            listener
23769                .set_nonblocking(true)
23770                .expect("configure mock listener");
23771            let addr = listener.local_addr().expect("mock server address");
23772            let stop = Arc::new(AtomicBool::new(false));
23773            let server_stop = Arc::clone(&stop);
23774            let requests = Arc::new(Mutex::new(Vec::new()));
23775            let server_requests = Arc::clone(&requests);
23776            let thread = thread::spawn(move || {
23777                let mut request_threads = Vec::new();
23778                while !server_stop.load(Ordering::SeqCst) {
23779                    match listener.accept() {
23780                        Ok((mut stream, _)) => {
23781                            if behavior.concurrent_requests {
23782                                let requests = Arc::clone(&server_requests);
23783                                request_threads.push(thread::spawn(move || {
23784                                    handle_mock_worker_request(&mut stream, &requests, behavior)
23785                                }));
23786                            } else {
23787                                handle_mock_worker_request(&mut stream, &server_requests, behavior);
23788                            }
23789                        }
23790                        Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
23791                            let mut index = 0;
23792                            while index < request_threads.len() {
23793                                if request_threads[index].is_finished() {
23794                                    request_threads
23795                                        .swap_remove(index)
23796                                        .join()
23797                                        .expect("join mock request");
23798                                } else {
23799                                    index += 1;
23800                                }
23801                            }
23802                            thread::sleep(Duration::from_millis(5));
23803                        }
23804                        Err(_) => break,
23805                    }
23806                }
23807                for request_thread in request_threads {
23808                    request_thread.join().expect("join mock request");
23809                }
23810            });
23811
23812            Self {
23813                addr,
23814                stop,
23815                requests,
23816                thread: Some(thread),
23817            }
23818        }
23819
23820        fn base_url(&self) -> String {
23821            format!("http://{}", self.addr)
23822        }
23823
23824        fn worker_protocol_for(&self, path: &str) -> Option<String> {
23825            self.requests
23826                .lock()
23827                .expect("captured requests")
23828                .iter()
23829                .find(|request| request.path == path)
23830                .and_then(|request| request.worker_protocol.clone())
23831        }
23832
23833        fn control_protocol_for(&self, path: &str) -> Option<String> {
23834            self.requests
23835                .lock()
23836                .expect("captured requests")
23837                .iter()
23838                .find(|request| request.path == path)
23839                .and_then(|request| request.control_protocol.clone())
23840        }
23841
23842        fn method_for(&self, path: &str) -> Option<String> {
23843            self.requests
23844                .lock()
23845                .expect("captured requests")
23846                .iter()
23847                .find(|request| request.path == path)
23848                .map(|request| request.method.clone())
23849        }
23850
23851        fn authorization_for(&self, path: &str) -> Option<String> {
23852            self.requests
23853                .lock()
23854                .expect("captured requests")
23855                .iter()
23856                .find(|request| request.path == path)
23857                .and_then(|request| request.authorization.clone())
23858        }
23859
23860        fn namespace_for(&self, path: &str) -> Option<String> {
23861            self.requests
23862                .lock()
23863                .expect("captured requests")
23864                .iter()
23865                .find(|request| request.path == path)
23866                .and_then(|request| request.namespace.clone())
23867        }
23868
23869        fn request_count(&self, path: &str) -> usize {
23870            self.requests
23871                .lock()
23872                .expect("captured requests")
23873                .iter()
23874                .filter(|request| request.path == path)
23875                .count()
23876        }
23877
23878        fn captured_paths(&self) -> Vec<String> {
23879            self.requests
23880                .lock()
23881                .expect("captured requests")
23882                .iter()
23883                .map(|request| request.path.clone())
23884                .collect()
23885        }
23886
23887        fn request_times(&self, path: &str) -> Vec<Instant> {
23888            self.requests
23889                .lock()
23890                .expect("captured requests")
23891                .iter()
23892                .filter(|request| request.path == path)
23893                .map(|request| request.received_at)
23894                .collect()
23895        }
23896
23897        fn request_body(&self, path: &str) -> Value {
23898            let requests = self.requests.lock().expect("captured requests");
23899            let body = &requests
23900                .iter()
23901                .find(|request| request.path == path)
23902                .unwrap_or_else(|| panic!("missing request for {path}"))
23903                .body;
23904            serde_json::from_str(body).unwrap_or_else(|error| {
23905                panic!("invalid JSON request body for {path}: {error}: {body:?}")
23906            })
23907        }
23908
23909        fn request_bodies(&self, path: &str) -> Vec<Value> {
23910            self.requests
23911                .lock()
23912                .expect("captured requests")
23913                .iter()
23914                .filter(|request| request.path == path)
23915                .map(|request| {
23916                    serde_json::from_str(&request.body).unwrap_or_else(|error| {
23917                        panic!(
23918                            "invalid JSON request body for {path}: {error}: {:?}",
23919                            request.body
23920                        )
23921                    })
23922                })
23923                .collect()
23924        }
23925    }
23926
23927    impl Drop for MockWorkerServer {
23928        fn drop(&mut self) {
23929            self.stop.store(true, Ordering::SeqCst);
23930            let _ = TcpStream::connect(self.addr);
23931
23932            if let Some(thread) = self.thread.take() {
23933                thread.join().expect("join mock server");
23934            }
23935        }
23936    }
23937
23938    fn handle_mock_worker_request(
23939        stream: &mut TcpStream,
23940        requests: &Arc<Mutex<Vec<CapturedRequest>>>,
23941        behavior: MockWorkerBehavior,
23942    ) {
23943        let _ = stream.set_read_timeout(Some(Duration::from_millis(200)));
23944        let mut buffer = [0_u8; 8192];
23945        let mut request = Vec::new();
23946
23947        loop {
23948            match stream.read(&mut buffer) {
23949                Ok(0) => break,
23950                Ok(read) => {
23951                    request.extend_from_slice(&buffer[..read]);
23952                    if mock_request_is_complete(&request) {
23953                        break;
23954                    }
23955                }
23956                Err(error)
23957                    if matches!(
23958                        error.kind(),
23959                        std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
23960                    ) =>
23961                {
23962                    break;
23963                }
23964                Err(_) => return,
23965            }
23966        }
23967
23968        let request = String::from_utf8_lossy(&request);
23969        let body = request
23970            .split_once("\r\n\r\n")
23971            .map(|(_, body)| body)
23972            .unwrap_or_default();
23973        let path = request
23974            .lines()
23975            .next()
23976            .and_then(|line| line.split_whitespace().nth(1))
23977            .unwrap_or_default();
23978        let method = request
23979            .lines()
23980            .next()
23981            .and_then(|line| line.split_whitespace().next())
23982            .unwrap_or_default();
23983        let authorization = request.lines().find_map(|line| {
23984            let (name, value) = line.split_once(':')?;
23985            name.eq_ignore_ascii_case("Authorization")
23986                .then(|| value.trim().to_string())
23987        });
23988        let namespace = request.lines().find_map(|line| {
23989            let (name, value) = line.split_once(':')?;
23990            name.eq_ignore_ascii_case("X-Namespace")
23991                .then(|| value.trim().to_string())
23992        });
23993        let worker_protocol = request.lines().find_map(|line| {
23994            let (name, value) = line.split_once(':')?;
23995            name.eq_ignore_ascii_case("X-Durable-Workflow-Protocol-Version")
23996                .then(|| value.trim().to_string())
23997        });
23998        let control_protocol = request.lines().find_map(|line| {
23999            let (name, value) = line.split_once(':')?;
24000            name.eq_ignore_ascii_case("X-Durable-Workflow-Control-Plane-Version")
24001                .then(|| value.trim().to_string())
24002        });
24003        let request_number = {
24004            let mut requests = requests.lock().expect("captured requests");
24005            requests.push(CapturedRequest {
24006                method: method.to_string(),
24007                path: path.to_string(),
24008                authorization,
24009                namespace,
24010                worker_protocol: worker_protocol.clone(),
24011                control_protocol,
24012                body: body.to_string(),
24013                received_at: Instant::now(),
24014            });
24015            requests
24016                .iter()
24017                .filter(|request| request.path == path)
24018                .count()
24019        };
24020
24021        if path == "/api/worker/register" {
24022            if behavior.reject_registration_protocol {
24023                write_mock_response(
24024                    stream,
24025                    "400 Bad Request",
24026                    r#"{"reason":"unsupported_protocol_version","message":"condition-wait occurrence identity requires worker protocol 1.17","supported_version":"1.16","requested_version":"1.17"}"#,
24027                );
24028                return;
24029            }
24030            if behavior.reject_registration {
24031                write_mock_response(
24032                    stream,
24033                    "503 Service Unavailable",
24034                    r#"{"reason":"registration_unavailable","message":"registration failed"}"#,
24035                );
24036                return;
24037            }
24038        }
24039
24040        if path.starts_with("/api/worker/registrations/") {
24041            if behavior.reject_deregistration_protocol {
24042                write_mock_response(
24043                    stream,
24044                    "400 Bad Request",
24045                    r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.17","requested_version":"1.19"}"#,
24046                );
24047            } else if behavior.reject_deregistration {
24048                write_mock_response(
24049                    stream,
24050                    "403 Forbidden",
24051                    r#"{"reason":"authorization_failed","message":"worker cannot deregister"}"#,
24052                );
24053            } else {
24054                write_mock_response(
24055                    stream,
24056                    "200 OK",
24057                    r#"{"worker_id":"deregistered-worker","outcome":"deregistered","recovered_workflow_task_count":2}"#,
24058                );
24059            }
24060            return;
24061        }
24062
24063        let is_poll = matches!(
24064            path,
24065            "/api/worker/workflow-tasks/poll"
24066                | "/api/worker/activity-tasks/poll"
24067                | "/api/worker/query-tasks/poll"
24068        );
24069        if is_poll && request_number <= behavior.poll_failures_per_path {
24070            return;
24071        }
24072        if path == "/api/worker/heartbeat" && request_number <= behavior.heartbeat_failures {
24073            return;
24074        }
24075        if path == "/api/worker/heartbeat"
24076            && behavior.heartbeat_failure_request == Some(request_number)
24077        {
24078            return;
24079        }
24080        if path == "/api/worker/heartbeat"
24081            && behavior.delayed_heartbeat_request == Some(request_number)
24082        {
24083            thread::sleep(behavior.heartbeat_response_delay);
24084        }
24085        if behavior.unauthorized_polls && is_poll {
24086            write_mock_response(
24087                stream,
24088                "401 Unauthorized",
24089                r#"{"reason":"authentication_failed","message":"invalid worker token"}"#,
24090            );
24091            return;
24092        }
24093        if behavior.draining_polls && is_poll {
24094            write_mock_response(
24095                stream,
24096                "409 Conflict",
24097                r#"{"task":null,"poll_status":"draining","reason":"worker_draining","worker_status":"draining","drain_intent":"draining"}"#,
24098            );
24099            return;
24100        }
24101
24102        if let Some(codec_case) = behavior.invalid_task_payload_codec {
24103            if is_poll && request_number == 1 {
24104                let mut task = match path {
24105                    "/api/worker/workflow-tasks/poll" => json!({
24106                        "task_id": "codec-workflow",
24107                        "workflow_type": "codec.workflow",
24108                        "payload_codec": DEFAULT_CODEC,
24109                        "workflow_task_attempt": 1,
24110                        "lease_owner": "codec-worker"
24111                    }),
24112                    "/api/worker/activity-tasks/poll" => json!({
24113                        "task_id": "codec-activity",
24114                        "activity_attempt_id": "codec-activity-attempt",
24115                        "activity_type": "codec.activity",
24116                        "payload_codec": DEFAULT_CODEC,
24117                        "attempt_number": 1,
24118                        "lease_owner": "codec-worker"
24119                    }),
24120                    "/api/worker/query-tasks/poll" => json!({
24121                        "query_task_id": "codec-query",
24122                        "query_task_attempt": 1,
24123                        "workflow_type": "codec.workflow",
24124                        "query_name": "known",
24125                        "payload_codec": DEFAULT_CODEC,
24126                        "lease_owner": "codec-worker"
24127                    }),
24128                    _ => unreachable!("is_poll limits task codec probe paths"),
24129                };
24130                codec_case.apply(&mut task);
24131                write_mock_response(stream, "200 OK", &json!({"task": task}).to_string());
24132                return;
24133            }
24134
24135            if matches!(
24136                path,
24137                "/api/worker/workflow-tasks/codec-workflow/fail"
24138                    | "/api/worker/activity-tasks/codec-activity/fail"
24139                    | "/api/worker/query-tasks/codec-query/fail"
24140            ) {
24141                write_mock_response(stream, "200 OK", r#"{"outcome":"failed"}"#);
24142                return;
24143            }
24144        }
24145
24146        if behavior.reject_query_protocol && path.starts_with("/api/worker/query-tasks/") {
24147            let requested_version = worker_protocol.as_deref().unwrap_or("missing");
24148            let body = format!(
24149                r#"{{"reason":"unsupported_protocol_version","message":"Query tasks require worker protocol 1.8 or newer.","supported_version":"1.7","requested_version":"{requested_version}"}}"#
24150            );
24151            write_mock_response(stream, "400 Bad Request", &body);
24152            return;
24153        }
24154
24155        if behavior.reject_query_completion && path == "/api/worker/query-tasks/query-late/complete"
24156        {
24157            write_mock_response(
24158                stream,
24159                "409 Conflict",
24160                r#"{"reason":"query_task_timed_out","message":"query task timed out before completion"}"#,
24161            );
24162            return;
24163        }
24164
24165        if behavior.workflow_completion_status.is_some()
24166            && path == "/api/worker/workflow-tasks/poll"
24167            && request_number == 1
24168        {
24169            write_mock_response(
24170                stream,
24171                "200 OK",
24172                r#"{"task":{"task_id":"workflow-timeout-task","workflow_id":"reused-workflow-id","run_id":"run-selected-timeout","workflow_type":"timeout.workflow","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"workflow_task_attempt":3,"lease_owner":"timeout-worker"}}"#,
24173            );
24174            return;
24175        }
24176
24177        if path == "/api/worker/workflow-tasks/workflow-timeout-task/complete" {
24178            if let (Some(status), Some(body)) = (
24179                behavior.workflow_completion_status,
24180                behavior.workflow_completion_body,
24181            ) {
24182                write_mock_response(stream, status, body);
24183                return;
24184            }
24185        }
24186
24187        if behavior.waiting_query_worker {
24188            if behavior.complete_named_signal
24189                && path == "/api/worker/workflow-tasks/poll"
24190                && request_number == 1
24191            {
24192                let body = json!({
24193                    "task": {
24194                        "task_id": "snapshot-open",
24195                        "workflow_id": "snapshot-1",
24196                        "run_id": "snapshot-run-1",
24197                        "workflow_type": "snapshot",
24198                        "payload_codec": DEFAULT_CODEC,
24199                        "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
24200                            .expect("Avro workflow arguments"),
24201                        "history_events": [],
24202                        "workflow_task_attempt": 1,
24203                        "lease_owner": "rust-snapshot-worker"
24204                    }
24205                })
24206                .to_string();
24207                write_mock_response(stream, "200 OK", &body);
24208                return;
24209            }
24210
24211            let signal_request = request_number - usize::from(behavior.complete_named_signal);
24212            let signal_request_limit = 2 + usize::from(behavior.complete_named_signal);
24213            if path == "/api/worker/workflow-tasks/poll"
24214                && signal_request >= 1
24215                && signal_request <= signal_request_limit
24216            {
24217                let finish = behavior.complete_named_signal && signal_request == 3;
24218                let amounts = if signal_request == 1 {
24219                    vec![3]
24220                } else {
24221                    vec![3, 5]
24222                };
24223                let task_id = if signal_request == 1 {
24224                    "snapshot-wait-3"
24225                } else if finish {
24226                    "snapshot-finish"
24227                } else {
24228                    "snapshot-wait-5"
24229                };
24230                let mut history_events = std::iter::once(json!({
24231                    "event_type": "SignalWaitOpened",
24232                    "payload": {"sequence": 1, "signal_name": "finish"}
24233                }))
24234                .chain(amounts.iter().enumerate().map(|(index, amount)| {
24235                    json!({
24236                        "event_type": "SignalReceived",
24237                        "payload": {
24238                            "signal_id": format!("increment-{amount}"),
24239                            "signal_name": "increment",
24240                            "workflow_sequence": index + 2,
24241                            "payload_codec": DEFAULT_CODEC,
24242                            "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
24243                                .expect("Avro signal envelope")
24244                        }
24245                    })
24246                }))
24247                .collect::<Vec<_>>();
24248                let (resume_id, resume_name, resume_arguments) = if finish {
24249                    history_events.push(json!({
24250                        "event_type": "SignalReceived",
24251                        "payload": {
24252                            "signal_id": "finish",
24253                            "signal_name": "finish",
24254                            "workflow_sequence": 4,
24255                            "payload_codec": DEFAULT_CODEC,
24256                            "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
24257                                .expect("Avro finish signal envelope")
24258                        }
24259                    }));
24260                    (
24261                        "finish".to_string(),
24262                        "finish".to_string(),
24263                        encode_value_envelope(&json!([]), DEFAULT_CODEC)
24264                            .expect("Avro finish resume signal"),
24265                    )
24266                } else {
24267                    let amount = amounts.last().expect("amount");
24268                    (
24269                        format!("increment-{amount}"),
24270                        "increment".to_string(),
24271                        encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
24272                            .expect("Avro increment resume signal"),
24273                    )
24274                };
24275                let body = json!({
24276                    "task": {
24277                        "task_id": task_id,
24278                        "workflow_id": "snapshot-1",
24279                        "run_id": "snapshot-run-1",
24280                        "workflow_type": "snapshot",
24281                        "payload_codec": DEFAULT_CODEC,
24282                        "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
24283                            .expect("Avro workflow arguments"),
24284                        "history_events": history_events,
24285                        "workflow_task_attempt": 1,
24286                        "workflow_signal_id": resume_id,
24287                        "signal_name": resume_name,
24288                        "signal_arguments": resume_arguments,
24289                        "lease_owner": "rust-snapshot-worker"
24290                    }
24291                })
24292                .to_string();
24293                write_mock_response(stream, "200 OK", &body);
24294                return;
24295            }
24296
24297            if path == "/api/worker/query-tasks/poll" && request_number == 1 {
24298                let history_events = [3, 5]
24299                    .into_iter()
24300                    .enumerate()
24301                    .map(|(index, amount)| {
24302                        json!({
24303                            "event_type": "SignalReceived",
24304                            "payload": {
24305                                "signal_id": format!("increment-{amount}"),
24306                                "signal_name": "increment",
24307                                "workflow_sequence": index + 2,
24308                                "payload_codec": DEFAULT_CODEC,
24309                                "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
24310                                    .expect("Avro query signal envelope")
24311                            }
24312                        })
24313                    })
24314                    .collect::<Vec<_>>();
24315                let body = json!({
24316                    "task": {
24317                        "query_task_id": "snapshot-current",
24318                        "query_task_attempt": 1,
24319                        "lease_owner": "rust-snapshot-worker",
24320                        "workflow_id": "snapshot-1",
24321                        "run_id": "snapshot-run-1",
24322                        "workflow_type": "snapshot",
24323                        "query_name": "current",
24324                        "payload_codec": DEFAULT_CODEC,
24325                        "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
24326                            .expect("Avro workflow arguments"),
24327                        "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
24328                            .expect("Avro query arguments"),
24329                        "history_events": history_events,
24330                        "run_status": "waiting"
24331                    }
24332                })
24333                .to_string();
24334                write_mock_response(stream, "200 OK", &body);
24335                return;
24336            }
24337
24338            if path == "/api/worker/workflow-tasks/snapshot-wait-3/fail"
24339                || path == "/api/worker/workflow-tasks/snapshot-wait-5/fail"
24340            {
24341                write_mock_response(
24342                    stream,
24343                    "200 OK",
24344                    r#"{"outcome":"waiting_for_history","recorded":true}"#,
24345                );
24346                return;
24347            }
24348
24349            if path == "/api/worker/workflow-tasks/snapshot-open/complete" {
24350                write_mock_response(stream, "200 OK", r#"{"outcome":"waiting","recorded":true}"#);
24351                return;
24352            }
24353
24354            if path == "/api/worker/workflow-tasks/snapshot-finish/complete" {
24355                write_mock_response(
24356                    stream,
24357                    "200 OK",
24358                    r#"{"outcome":"completed","run_status":"completed","recorded":true}"#,
24359                );
24360                return;
24361            }
24362
24363            if path == "/api/worker/query-tasks/snapshot-current/complete" {
24364                write_mock_response(stream, "200 OK", r#"{"outcome":"completed"}"#);
24365                return;
24366            }
24367        }
24368
24369        if matches!(
24370            path,
24371            "/api/workflows/typed-1/query/inspect" | "/api/workflows/typed-1/update/replace"
24372        ) {
24373            let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
24374                .expect("typed mock result");
24375            let body = json!({
24376                "result": typed_fidelity_probe().into_json().expect("result projection"),
24377                "result_envelope": result,
24378            })
24379            .to_string();
24380            write_mock_response(stream, "200 OK", &body);
24381            return;
24382        }
24383
24384        if path == "/api/workflows/typed-1" {
24385            let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
24386                .expect("typed mock result");
24387            let body = json!({
24388                "workflow_id": "typed-1",
24389                "run_id": "run-typed-1",
24390                "workflow_type": "typed.echo",
24391                "status": "completed",
24392                "output": typed_fidelity_probe().into_json().expect("output projection"),
24393                "output_envelope": result,
24394            })
24395            .to_string();
24396            write_mock_response(stream, "200 OK", &body);
24397            return;
24398        }
24399
24400        let (status, body) = match path {
24401            "/api/health" => ("200 OK", r#"{"status":"ok"}"#),
24402            "/api/workflows" => (
24403                "201 Created",
24404                r#"{"workflow_id":"wf-start-options","run_id":"run-start-options","workflow_type":"rust.timeout"}"#,
24405            ),
24406            "/api/worker/register" if behavior.decline_registration => (
24407                "200 OK",
24408                r#"{"worker_id":"declined-worker","registered":false}"#,
24409            ),
24410            "/api/worker/register" if behavior.waiting_query_worker => (
24411                "200 OK",
24412                r#"{"worker_id":"rust-snapshot-worker","registered":true,"heartbeat_interval_seconds":1}"#,
24413            ),
24414            "/api/worker/register" => (
24415                "200 OK",
24416                r#"{"worker_id":"mock-worker","registered":true,"heartbeat_interval_seconds":3600}"#,
24417            ),
24418            "/api/worker/heartbeat" => ("200 OK", "{}"),
24419            "/api/worker/activity-tasks/poll"
24420                if behavior.cancelled_activity && request_number == 1 =>
24421            {
24422                (
24423                    "200 OK",
24424                    r#"{"task":{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","activity_type":"cancel-aware","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"rust-cancel-worker"}}"#,
24425                )
24426            }
24427            "/api/worker/activity-tasks/poll" | "/api/worker/workflow-tasks/poll" => {
24428                ("200 OK", r#"{"task":null}"#)
24429            }
24430            "/api/worker/query-tasks/poll"
24431                if behavior.reject_query_completion && request_number == 1 =>
24432            {
24433                (
24434                    "200 OK",
24435                    r#"{"task":{"query_task_id":"query-late","query_task_attempt":1,"lease_owner":"late-worker","workflow_id":"counter-late","run_id":"run-late","workflow_type":"counter","query_name":"current","payload_codec":"avro","workflow_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"query_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"run_status":"running"}}"#,
24436                )
24437            }
24438            "/api/worker/query-tasks/poll" => ("200 OK", r#"{"task":null}"#),
24439            "/api/worker/query-tasks/query-capture/complete"
24440            | "/api/worker/query-tasks/query-capture/fail" => ("200 OK", "{}"),
24441            "/api/worker/activity-tasks/activity-cancel/heartbeat" => (
24442                "200 OK",
24443                r#"{"activity_attempt_id":"attempt-cancel","cancel_requested":true,"can_continue":false,"reason":"run_cancelled","run_closed_reason":"cancelled","heartbeat_recorded":false}"#,
24444            ),
24445            "/api/worker/activity-tasks/activity-cancel/complete" => (
24446                "409 Conflict",
24447                r#"{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","reason":"run_cancelled","cancel_requested":true,"can_continue":false,"run_closed_reason":"cancelled"}"#,
24448            ),
24449            "/api/worker/activity-tasks/activity-typed/complete"
24450            | "/api/worker/activity-tasks/activity-typed/fail"
24451            | "/api/workflows/typed-1/signal/changed" => ("200 OK", "{}"),
24452            "/api/workflows/counter-1/query/current" => (
24453                "200 OK",
24454                r#"{"workflow_id":"counter-1","query_name":"current","result":{"count":8},"result_envelope":{"codec":"avro","blob":"wwHioz3/VYAiNw4CCmNvdW50BBAA"}}"#,
24455            ),
24456            "/api/workflows/counter-1/query/missing" => (
24457                "404 Not Found",
24458                r#"{"workflow_id":"counter-1","query_name":"missing","reason":"rejected_unknown_query","message":"unknown query"}"#,
24459            ),
24460            "/api/workflows/wf-lifecycle/cancel" => (
24461                "200 OK",
24462                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","reason":"cleanup requested","command_status":"accepted"}"#,
24463            ),
24464            "/api/workflows/wf-lifecycle/terminate" => (
24465                "200 OK",
24466                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","reason":"forced stop","command_status":"accepted"}"#,
24467            ),
24468            "/api/workflows/wf-lifecycle/runs/run-current/cancel" => (
24469                "200 OK",
24470                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","command_status":"accepted"}"#,
24471            ),
24472            "/api/workflows/wf-lifecycle/runs/run-current/terminate" => (
24473                "200 OK",
24474                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","command_status":"accepted"}"#,
24475            ),
24476            "/api/workflows/wf-lifecycle/runs/run-stale/cancel"
24477            | "/api/workflows/wf-lifecycle/runs/run-stale/terminate" => (
24478                "409 Conflict",
24479                r#"{"workflow_id":"wf-lifecycle","run_id":"run-stale","reason":"historical_run_command_rejected","target_scope":"run","message":"Commands cannot target historical runs."}"#,
24480            ),
24481            "/api/workflows/wf-failed" | "/api/workflows/wf-failed/runs/run-failed" => (
24482                "200 OK",
24483                r#"{"workflow_id":"wf-failed","run_id":"run-failed","status":"failed","closed_reason":"failed","error":"payment failed","failure":{"message":"payment failed","failure_category":"application","exception_type":"PaymentError","exception_class":"billing::PaymentError","non_retryable":true,"exception":{"type":"PaymentError","class":"billing::PaymentError","message":"payment failed"},"failures":[{"id":"failure-17","failure_category":"application"}]}}"#,
24484            ),
24485            "/api/workflows/wf-cancelled" => (
24486                "200 OK",
24487                r#"{"workflow_id":"wf-cancelled","run_id":"run-cancelled","status":"cancelled","closed_reason":"cancelled","reason":"cleanup requested"}"#,
24488            ),
24489            "/api/workflows/wf-terminated" => (
24490                "200 OK",
24491                r#"{"workflow_id":"wf-terminated","run_id":"run-terminated","status":"terminated","closed_reason":"terminated","reason":"forced stop"}"#,
24492            ),
24493            "/api/workflows/wf-timed-out" => (
24494                "200 OK",
24495                r#"{"workflow_id":"wf-timed-out","run_id":"run-timed-out","status":"failed","closed_reason":"timed_out","reason":"run_timeout"}"#,
24496            ),
24497            "/api/workflows/wf-waiting" | "/api/workflows/wf-waiting/runs/run-waiting" => (
24498                "200 OK",
24499                r#"{"workflow_id":"wf-waiting","run_id":"run-waiting","status":"waiting"}"#,
24500            ),
24501            "/api/workflows/wf-selected" => (
24502                "200 OK",
24503                r#"{"workflow_id":"wf-selected","run_id":"run-current","status":"completed","output":"current run output"}"#,
24504            ),
24505            "/api/workflows/wf-selected/runs/run-selected" => (
24506                "200 OK",
24507                r#"{"workflow_id":"wf-selected","run_id":"run-selected","status":"cancelled","closed_reason":"cancelled","reason":"selected run cancelled"}"#,
24508            ),
24509            _ => ("404 Not Found", r#"{"message":"not found"}"#),
24510        };
24511        write_mock_response(stream, status, body);
24512    }
24513
24514    fn mock_request_is_complete(request: &[u8]) -> bool {
24515        let Some(header_end) = request
24516            .windows(4)
24517            .position(|window| window == b"\r\n\r\n")
24518            .map(|position| position + 4)
24519        else {
24520            return false;
24521        };
24522        let headers = String::from_utf8_lossy(&request[..header_end]);
24523        let content_length = headers.lines().find_map(|line| {
24524            let (name, value) = line.split_once(':')?;
24525            name.eq_ignore_ascii_case("content-length")
24526                .then(|| value.trim().parse::<usize>().ok())
24527                .flatten()
24528        });
24529
24530        request.len() >= header_end + content_length.unwrap_or(0)
24531    }
24532
24533    fn write_mock_response(stream: &mut TcpStream, status: &str, body: &str) {
24534        let response = format!(
24535            "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
24536            body.len()
24537        );
24538
24539        let _ = stream.write_all(response.as_bytes());
24540        let _ = stream.flush();
24541    }
24542}