1#![doc = include_str!("../README.md")]
2
3mod runtime_payloads;
4mod runtime_uploads;
5
6use std::{
7 any::{type_name, Any, TypeId},
8 collections::{BTreeMap, HashMap},
9 future::Future,
10 io::{self, Read},
11 pin::Pin,
12 sync::{
13 atomic::{AtomicBool, Ordering},
14 Arc, Mutex, OnceLock,
15 },
16 task::{Context as TaskContext, Poll},
17 time::{Duration, Instant, SystemTime, UNIX_EPOCH},
18};
19
20use apache_avro::{from_avro_datum, to_avro_datum, types::Value as AvroDatum, Schema};
21use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
22use chrono::DateTime;
23use futures_util::{future::OptionFuture, task::noop_waker_ref};
24use serde::{
25 de::DeserializeOwned,
26 ser::{SerializeMap, SerializeSeq},
27 Deserialize, Deserializer, Serialize, Serializer,
28};
29pub use serde_json::{json, Value};
30use sha2::{Digest, Sha256};
31use thiserror::Error;
32pub use uuid::Uuid;
33
34pub const WORKER_PROTOCOL_VERSION: &str = "1.19";
35pub const PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION: &str = "1.18";
37pub const CONTROL_PLANE_VERSION: &str = "2";
38pub const DEFAULT_CODEC: &str = "avro";
39pub const SDK_VERSION: &str = concat!("durable-workflow-rust/", env!("CARGO_PKG_VERSION"));
40pub const CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY: &str =
42 "condition_wait_occurrence_identity";
43pub const MEMO_UPSERTS_CAPABILITY: &str = "memo_upserts";
45pub const QUERY_TASKS_CAPABILITY: &str = "query_tasks";
47pub const TYPED_SEARCH_ATTRIBUTES_CAPABILITY: &str = "typed_search_attributes";
49pub const WORKFLOW_UPDATES_CAPABILITY: &str = "workflow_updates";
51pub const MESSAGE_STREAMS_CAPABILITY: &str = "message_streams";
53pub const DURABLE_SELECTION_CAPABILITY: &str = "durable_selection";
55pub const MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.15";
56pub const MESSAGE_STREAM_SIGNAL: &str = "__durable_workflow_message_stream";
57pub const MESSAGE_STREAM_SCHEMA: &str = "durable-workflow.v2.message-stream.message";
58pub const MESSAGE_STREAM_CURSOR_SCHEMA: &str = "durable-workflow.v2.message-stream.cursor";
59pub const MESSAGE_STREAM_MAX_BATCH: usize = 100;
60pub const QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
62pub const SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
64pub const MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.14";
66pub const TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.16";
68pub const CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.9";
70pub const CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.17";
72pub const DURABLE_SELECTION_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.19";
74
75pub fn worker_protocol_supports_message_streams(version: &str) -> bool {
76 let Some((major, minor)) = version.split_once('.') else {
77 return false;
78 };
79 major == "1" && minor.parse::<u64>().is_ok_and(|minor| minor >= 15)
80}
81
82fn validate_user_signal_name(signal_name: &str) -> Result<()> {
83 if signal_name == MESSAGE_STREAM_SIGNAL {
84 return Err(Error::Codec(format!(
85 "signal name {MESSAGE_STREAM_SIGNAL:?} is reserved by the workflow runtime"
86 )));
87 }
88 Ok(())
89}
90
91const MAX_LONG_POLL_TIMEOUT_SECONDS: u64 = 60;
92const WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE: &str =
93 "Workflow task waiting for scheduled history.";
94const WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE: &str = "WorkflowTaskWaitingForHistory";
95const MISSING_TASK_PAYLOAD_CODEC: &str = "\0missing-task-payload-codec";
96const NULL_TASK_PAYLOAD_CODEC: &str = "\0null-task-payload-codec";
97const NON_STRING_TASK_PAYLOAD_CODEC: &str = "\0non-string-task-payload-codec";
98const MAX_MEMO_ENTRIES: usize = 100;
99const MAX_MEMO_VALUE_SIZE_BYTES: usize = 10_240;
100const MAX_MEMO_TOTAL_SIZE_BYTES: usize = 65_536;
101
102const QUERY_TASK_FINAL_REJECTION_REASONS: &[&str] = &[
103 "lease_expired",
104 "query_task_not_found",
105 "query_task_not_leased",
106 "query_task_timed_out",
107];
108
109pub fn portable_worker_affinity_capability_manifest() -> Value {
111 json!({
112 "local_activities": {
113 "supported": false,
114 "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
115 "reason": "rust_worker_does_not_execute_record_local_activity",
116 },
117 "worker_sessions": {
118 "supported": false,
119 "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
120 "reason": "rust_worker_has_no_typed_session_lifecycle",
121 },
122 "sticky_execution": {
123 "supported": false,
124 "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
125 "reason": "rust_worker_uses_complete_durable_history_replay",
126 },
127 })
128}
129
130pub const AVRO_VALUE_SCHEMA_JSON: &str =
132 include_str!("../schema/durable_workflow.protocol.Value.v1.avsc");
133pub const AVRO_VALUE_SCHEMA_FINGERPRINT_HEX: &str = "e2a33dff55802237";
134pub const AVRO_VALUE_SCHEMA_FINGERPRINT: [u8; 8] = [0xe2, 0xa3, 0x3d, 0xff, 0x55, 0x80, 0x22, 0x37];
135const AVRO_SINGLE_OBJECT_MAGIC: [u8; 2] = [0xc3, 0x01];
136
137static AVRO_VALUE_SCHEMA: OnceLock<std::result::Result<Schema, String>> = OnceLock::new();
138static AVRO_VALUE_ORDERED_MAP_ENCODING_SCHEMA: OnceLock<std::result::Result<Schema, String>> =
139 OnceLock::new();
140
141#[derive(Clone, Copy)]
142enum RequestProtocol {
143 ControlPlane,
144 Worker(&'static str),
145}
146
147pub type Result<T> = std::result::Result<T, Error>;
148
149#[derive(Debug, Error)]
150pub enum Error {
151 #[error("transport error: {0}")]
152 Transport(#[from] reqwest::Error),
153 #[error(
154 "invalid Durable Workflow base URL: omit the SDK-owned /api suffix and pass the Server or Cloud runtime base URL; the SDK appends /api automatically"
155 )]
156 InvalidBaseUrl,
157 #[error("json error: {0}")]
158 Json(#[from] serde_json::Error),
159 #[error("http {status}: {body}")]
160 Http {
161 status: reqwest::StatusCode,
162 body: String,
163 },
164 #[error("codec error: {0}")]
165 Codec(String),
166 #[error(transparent)]
167 QueryFailed(QueryFailure),
168 #[error(transparent)]
169 Protocol(ProtocolFailure),
170 #[error(transparent)]
171 NonDeterministicReplay(ReplayFailure),
172 #[error(transparent)]
173 ChildWorkflowFailed(ChildWorkflowFailure),
174 #[error(transparent)]
175 ActivityFailed(ActivityFailure),
176 #[error(transparent)]
177 ParallelFailed(ParallelFailure),
178 #[error(transparent)]
179 SagaCompensationFailed(SagaCompensationFailure),
180 #[error(transparent)]
181 InvalidParallelGroup(ParallelGroupError),
182 #[error(transparent)]
183 DurableOperationCancelled(DurableOperationCancelled),
184 #[error(transparent)]
185 WorkflowCancellationRequested(WorkflowCancellationRequested),
186 #[error(transparent)]
187 WorkflowCommandRejected(WorkflowCommandRejection),
188 #[error(transparent)]
189 WorkflowFailed(WorkflowTerminalOutcome),
190 #[error(transparent)]
191 WorkflowCancelled(WorkflowTerminalOutcome),
192 #[error(transparent)]
193 WorkflowTerminated(WorkflowTerminalOutcome),
194 #[error(transparent)]
195 WorkflowTimedOut(WorkflowTerminalOutcome),
196 #[error(transparent)]
197 ActivityTaskRejected(ActivityTaskRejection),
198 #[error("workflow handler {0:?} is not registered")]
199 WorkflowNotRegistered(String),
200 #[error("activity handler {0:?} is not registered")]
201 ActivityNotRegistered(String),
202 #[error(
203 "{handler_kind} handler {handler_name:?} {value_kind} type {rust_type} is incompatible with the fixed Avro Value codec: {message}"
204 )]
205 HandlerType {
206 handler_kind: HandlerKind,
207 handler_name: String,
208 value_kind: HandlerValueKind,
209 rust_type: &'static str,
210 message: String,
211 },
212 #[error("workflow future yielded without emitting a durable command")]
213 WorkflowYieldedWithoutCommand,
214 #[error(
215 "workflow_stream_command_identity_missing: workflow stream authoring requires a non-empty server-provided workflow_command_id"
216 )]
217 MissingWorkflowCommandIdentity,
218 #[error("workflow state lock is poisoned")]
219 WorkflowStatePoisoned,
220 #[error("timer duration is too large for the worker protocol")]
221 TimerDurationOverflow,
222 #[error(transparent)]
223 InvalidConditionWaitOptions(#[from] ConditionWaitOptionsError),
224 #[error(transparent)]
225 InvalidSearchAttributeUpdate(#[from] SearchAttributeUpdateError),
226 #[error("operation timed out")]
227 Timeout,
228 #[error(
229 "missing {role}-plane credentials: configure ClientBuilder::{role}_token or ClientBuilder::token; a {opposite_role}-plane token cannot authorize this request"
230 )]
231 MissingRoleCredentials {
232 role: &'static str,
233 opposite_role: &'static str,
234 },
235 #[error("worker loop error: {0}")]
236 WorkerLoop(String),
237 #[error(
238 "workflow command contract for {workflow_type:?} declares update validators, but this Rust SDK cannot execute synchronous pre-accept update validation"
239 )]
240 UnsupportedUpdateValidators { workflow_type: String },
241 #[error("{primary}; worker deregistration also failed: {deregistration}")]
242 WorkerShutdown {
243 primary: Box<Error>,
244 deregistration: Box<Error>,
245 },
246 #[error("invalid child workflow options: {0}")]
247 InvalidChildWorkflowOptions(String),
248 #[error("invalid workflow memo update: {0}")]
249 InvalidMemoUpdate(String),
250 #[error(
251 "workflow_memo_updates_unavailable: the connected runtime did not advertise workflow memo update support"
252 )]
253 WorkflowMemoUpdatesUnavailable,
254 #[error(transparent)]
255 InvalidActivityOptions(ActivityOptionsError),
256 #[error(transparent)]
257 InvalidContinueAsNewOptions(#[from] ContinueAsNewOptionsError),
258 #[doc(hidden)]
259 #[error("workflow requested continue as new")]
260 ContinueAsNew(ContinueAsNewRequest),
261}
262
263#[derive(Clone, Debug, Error, PartialEq, Eq)]
265pub enum ConditionWaitOptionsError {
266 #[error("condition_key must be non-empty")]
267 EmptyKey,
268 #[error("condition_definition_fingerprint must be non-empty")]
269 EmptyPredicateIdentity,
270 #[error("condition timeout is too large for the worker protocol")]
271 TimeoutOverflow,
272}
273
274#[derive(Clone, Debug, PartialEq, Eq)]
281pub struct ConditionWaitOptions {
282 condition_key: String,
283 predicate_identity: String,
284 timeout: Option<Duration>,
285}
286
287impl ConditionWaitOptions {
288 pub fn new(condition_key: impl Into<String>, predicate_identity: impl Into<String>) -> Self {
289 Self {
290 condition_key: condition_key.into(),
291 predicate_identity: predicate_identity.into(),
292 timeout: None,
293 }
294 }
295
296 pub fn timeout(mut self, timeout: Duration) -> Self {
297 self.timeout = Some(timeout);
298 self
299 }
300
301 fn validate(
302 &self,
303 ) -> std::result::Result<ValidatedConditionWaitOptions, ConditionWaitOptionsError> {
304 let condition_key = self.condition_key.trim();
305 if condition_key.is_empty() {
306 return Err(ConditionWaitOptionsError::EmptyKey);
307 }
308 let predicate_identity = self.predicate_identity.trim();
309 if predicate_identity.is_empty() {
310 return Err(ConditionWaitOptionsError::EmptyPredicateIdentity);
311 }
312 let timeout_seconds = self
313 .timeout
314 .map(|timeout| {
315 timeout
316 .as_secs()
317 .checked_add(u64::from(timeout.subsec_nanos() > 0))
318 .ok_or(ConditionWaitOptionsError::TimeoutOverflow)
319 })
320 .transpose()?;
321
322 Ok(ValidatedConditionWaitOptions {
323 condition_key: condition_key.to_string(),
324 predicate_identity: predicate_identity.to_string(),
325 timeout_seconds,
326 })
327 }
328}
329
330#[derive(Clone, Debug, PartialEq, Eq)]
331struct ValidatedConditionWaitOptions {
332 condition_key: String,
333 predicate_identity: String,
334 timeout_seconds: Option<u64>,
335}
336
337const CONDITION_WAIT_OCCURRENCE_PREFIX: &str = "rust:condition-wait:";
338
339#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
341#[serde(rename_all = "snake_case")]
342pub enum ConditionWaitResult {
343 Satisfied,
344 TimedOut,
345}
346
347impl ConditionWaitResult {
348 pub fn is_satisfied(self) -> bool {
349 self == Self::Satisfied
350 }
351
352 pub fn is_timed_out(self) -> bool {
353 self == Self::TimedOut
354 }
355}
356
357#[doc(hidden)]
359pub fn __condition_definition_fingerprint(source: &str) -> String {
360 let mut digest = Sha256::new();
361 digest.update(b"durable-workflow-rust.wait-condition.v1\0");
362 digest.update(source.as_bytes());
363 format!("sha256:{:x}", digest.finalize())
364}
365
366#[macro_export]
372macro_rules! wait_condition {
373 ($ctx:expr, $key:expr, timeout: $timeout:expr, $predicate:expr $(,)?) => {{
374 $ctx.wait_condition(
375 $crate::ConditionWaitOptions::new(
376 $key,
377 $crate::__condition_definition_fingerprint(concat!(
378 module_path!(),
379 "\0",
380 stringify!($predicate)
381 )),
382 )
383 .timeout($timeout),
384 $predicate,
385 )
386 }};
387 ($ctx:expr, $key:expr, $predicate:expr $(,)?) => {{
388 $ctx.wait_condition(
389 $crate::ConditionWaitOptions::new(
390 $key,
391 $crate::__condition_definition_fingerprint(concat!(
392 module_path!(),
393 "\0",
394 stringify!($predicate)
395 )),
396 ),
397 $predicate,
398 )
399 }};
400}
401
402const MAX_SEARCH_ATTRIBUTES_PER_UPDATE: usize = 100;
403const MAX_SEARCH_ATTRIBUTE_KEY_LENGTH: usize = 64;
404const MAX_SEARCH_ATTRIBUTE_STRING_LENGTH: usize = 2_048;
405const MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH: usize = 255;
406const MAX_SEARCH_ATTRIBUTE_UPDATE_BYTES: usize = 65_536;
407
408#[derive(Clone, Debug, Error, PartialEq, Eq)]
410pub enum SearchAttributeUpdateError {
411 #[error("search-attribute update requires at least one attribute")]
412 Empty,
413 #[error("search attribute key {0:?} must be 1-64 URL-safe ASCII characters")]
414 InvalidKey(String),
415 #[error("search-attribute update exceeds the limit of 100 attributes")]
416 TooManyAttributes,
417 #[error("search attribute {key:?} {kind} value exceeds {limit} bytes")]
418 ValueTooLong {
419 key: String,
420 kind: &'static str,
421 limit: usize,
422 },
423 #[error(
424 "search attribute {0:?} must not contain an empty string value; use delete() to remove it"
425 )]
426 EmptyString(String),
427 #[error("search attribute {0:?} has a non-finite float value")]
428 NonFiniteFloat(String),
429 #[error("search attribute {0:?} must use an RFC 3339 datetime with an explicit timezone")]
430 InvalidDateTime(String),
431 #[error("search-attribute update exceeds the 65536-byte protocol limit")]
432 PayloadTooLarge,
433}
434
435#[derive(Clone, Debug, PartialEq)]
437pub enum SearchAttributeValue {
438 String(String),
439 Keyword(String),
440 KeywordList(Vec<String>),
441 Int(i64),
442 Float(f64),
443 Bool(bool),
444 DateTime(String),
445 Delete,
446}
447
448impl SearchAttributeValue {
449 fn type_name(&self) -> Option<&'static str> {
450 match self {
451 Self::String(_) => Some("string"),
452 Self::Keyword(_) => Some("keyword"),
453 Self::KeywordList(_) => Some("keyword_list"),
454 Self::Int(_) => Some("int"),
455 Self::Float(_) => Some("float"),
456 Self::Bool(_) => Some("bool"),
457 Self::DateTime(_) => Some("datetime"),
458 Self::Delete => None,
459 }
460 }
461
462 fn normalized(self, key: &str) -> std::result::Result<Self, SearchAttributeUpdateError> {
463 let normalize_string = |value: String, kind: &'static str, limit: usize| {
464 let value = value.trim().to_string();
465 if value.is_empty() {
466 return Err(SearchAttributeUpdateError::EmptyString(key.to_string()));
467 }
468 if value.len() > limit {
469 return Err(SearchAttributeUpdateError::ValueTooLong {
470 key: key.to_string(),
471 kind,
472 limit,
473 });
474 }
475 Ok(value)
476 };
477
478 match self {
479 Self::String(value) => Ok(Self::String(normalize_string(
480 value,
481 "string",
482 MAX_SEARCH_ATTRIBUTE_STRING_LENGTH,
483 )?)),
484 Self::Keyword(value) => Ok(Self::Keyword(normalize_string(
485 value,
486 "keyword",
487 MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH,
488 )?)),
489 Self::KeywordList(values) => {
490 let values = values
491 .into_iter()
492 .map(|value| {
493 let value = value.trim().to_string();
494 if value.len() > MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH {
495 return Err(SearchAttributeUpdateError::ValueTooLong {
496 key: key.to_string(),
497 kind: "keyword-list entry",
498 limit: MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH,
499 });
500 }
501 Ok(value)
502 })
503 .collect::<std::result::Result<Vec<_>, _>>()?;
504 Ok(Self::KeywordList(values))
505 }
506 Self::Float(value) if !value.is_finite() => {
507 Err(SearchAttributeUpdateError::NonFiniteFloat(key.to_string()))
508 }
509 Self::DateTime(value) => {
510 let value =
511 normalize_string(value, "datetime", MAX_SEARCH_ATTRIBUTE_STRING_LENGTH)?;
512 if DateTime::parse_from_rfc3339(&value).is_err() {
513 return Err(SearchAttributeUpdateError::InvalidDateTime(key.to_string()));
514 }
515 Ok(Self::DateTime(value))
516 }
517 value => Ok(value),
518 }
519 }
520
521 fn into_json(self) -> Value {
522 match self {
523 Self::String(value) | Self::Keyword(value) | Self::DateTime(value) => {
524 Value::String(value)
525 }
526 Self::KeywordList(values) => {
527 Value::Array(values.into_iter().map(Value::String).collect())
528 }
529 Self::Int(value) => json!(value),
530 Self::Float(value) => json!(value),
531 Self::Bool(value) => json!(value),
532 Self::Delete => Value::Null,
533 }
534 }
535}
536
537#[derive(Clone, Debug, Default, PartialEq)]
539pub struct SearchAttributeUpdate {
540 attributes: BTreeMap<String, SearchAttributeValue>,
541}
542
543impl SearchAttributeUpdate {
544 pub fn new() -> Self {
545 Self::default()
546 }
547
548 pub fn set(
549 mut self,
550 key: impl Into<String>,
551 value: SearchAttributeValue,
552 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
553 let key = key.into();
554 validate_search_attribute_key(&key)?;
555 if !self.attributes.contains_key(&key)
556 && self.attributes.len() >= MAX_SEARCH_ATTRIBUTES_PER_UPDATE
557 {
558 return Err(SearchAttributeUpdateError::TooManyAttributes);
559 }
560 self.attributes.insert(key.clone(), value.normalized(&key)?);
561 self.validate_size()?;
562 Ok(self)
563 }
564
565 pub fn string(
566 self,
567 key: impl Into<String>,
568 value: impl Into<String>,
569 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
570 self.set(key, SearchAttributeValue::String(value.into()))
571 }
572
573 pub fn keyword(
574 self,
575 key: impl Into<String>,
576 value: impl Into<String>,
577 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
578 self.set(key, SearchAttributeValue::Keyword(value.into()))
579 }
580
581 pub fn keyword_list<I, V>(
582 self,
583 key: impl Into<String>,
584 values: I,
585 ) -> std::result::Result<Self, SearchAttributeUpdateError>
586 where
587 I: IntoIterator<Item = V>,
588 V: Into<String>,
589 {
590 self.set(
591 key,
592 SearchAttributeValue::KeywordList(values.into_iter().map(Into::into).collect()),
593 )
594 }
595
596 pub fn int(
597 self,
598 key: impl Into<String>,
599 value: i64,
600 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
601 self.set(key, SearchAttributeValue::Int(value))
602 }
603
604 pub fn float(
605 self,
606 key: impl Into<String>,
607 value: f64,
608 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
609 self.set(key, SearchAttributeValue::Float(value))
610 }
611
612 pub fn bool(
613 self,
614 key: impl Into<String>,
615 value: bool,
616 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
617 self.set(key, SearchAttributeValue::Bool(value))
618 }
619
620 pub fn datetime(
621 self,
622 key: impl Into<String>,
623 value: impl Into<String>,
624 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
625 self.set(key, SearchAttributeValue::DateTime(value.into()))
626 }
627
628 pub fn delete(
629 self,
630 key: impl Into<String>,
631 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
632 self.set(key, SearchAttributeValue::Delete)
633 }
634
635 fn validate_size(&self) -> std::result::Result<(), SearchAttributeUpdateError> {
636 let (attributes, _) = self.clone().into_wire_parts();
637 if serde_json::to_vec(&attributes)
638 .map(|payload| payload.len() > MAX_SEARCH_ATTRIBUTE_UPDATE_BYTES)
639 .unwrap_or(true)
640 {
641 return Err(SearchAttributeUpdateError::PayloadTooLarge);
642 }
643 Ok(())
644 }
645
646 fn into_wire_parts(self) -> (Value, BTreeMap<String, String>) {
647 let mut attributes = serde_json::Map::new();
648 let mut attribute_types = BTreeMap::new();
649 for (key, value) in self.attributes {
650 if let Some(type_name) = value.type_name() {
651 attribute_types.insert(key.clone(), type_name.to_string());
652 }
653 attributes.insert(key, value.into_json());
654 }
655 (Value::Object(attributes), attribute_types)
656 }
657
658 fn validate(&self) -> std::result::Result<(), SearchAttributeUpdateError> {
659 if self.attributes.is_empty() {
660 return Err(SearchAttributeUpdateError::Empty);
661 }
662 self.validate_size()
663 }
664}
665
666fn validate_search_attribute_key(key: &str) -> std::result::Result<(), SearchAttributeUpdateError> {
667 let valid = !key.is_empty()
668 && key.len() <= MAX_SEARCH_ATTRIBUTE_KEY_LENGTH
669 && key
670 .bytes()
671 .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-' | b':'));
672 if valid {
673 Ok(())
674 } else {
675 Err(SearchAttributeUpdateError::InvalidKey(key.to_string()))
676 }
677}
678
679#[derive(Clone, Copy, Debug, PartialEq, Eq)]
681pub enum HandlerKind {
682 Workflow,
683 Activity,
684}
685
686impl std::fmt::Display for HandlerKind {
687 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
688 formatter.write_str(match self {
689 Self::Workflow => "workflow",
690 Self::Activity => "activity",
691 })
692 }
693}
694
695#[derive(Clone, Copy, Debug, PartialEq, Eq)]
697pub enum HandlerValueKind {
698 Input,
699 Result,
700}
701
702impl std::fmt::Display for HandlerValueKind {
703 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
704 formatter.write_str(match self {
705 Self::Input => "input",
706 Self::Result => "result",
707 })
708 }
709}
710
711#[derive(Clone, Copy, Debug, PartialEq, Eq)]
713pub enum WorkflowCommandKind {
714 Cancel,
715 Terminate,
716}
717
718impl WorkflowCommandKind {
719 fn as_str(self) -> &'static str {
720 match self {
721 Self::Cancel => "cancel",
722 Self::Terminate => "terminate",
723 }
724 }
725}
726
727#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
729pub struct WorkflowCommandOptions {
730 #[serde(skip_serializing_if = "Option::is_none")]
731 pub reason: Option<String>,
732 #[serde(skip_serializing_if = "Option::is_none")]
733 pub request_id: Option<String>,
734}
735
736#[derive(Clone, Debug, PartialEq, Eq)]
743pub struct WorkflowStartOptions {
744 pub execution_timeout_seconds: u64,
745 pub run_timeout_seconds: u64,
746}
747
748impl Default for WorkflowStartOptions {
749 fn default() -> Self {
750 Self {
751 execution_timeout_seconds: 3600,
752 run_timeout_seconds: 600,
753 }
754 }
755}
756
757impl WorkflowStartOptions {
758 pub fn new() -> Self {
759 Self::default()
760 }
761
762 pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
763 self.execution_timeout_seconds = seconds;
764 self
765 }
766
767 pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
768 self.run_timeout_seconds = seconds;
769 self
770 }
771
772 fn validate(&self) -> Result<()> {
773 if self.execution_timeout_seconds == 0 {
774 return Err(Error::Codec(
775 "execution_timeout_seconds must be at least 1".to_string(),
776 ));
777 }
778 if self.run_timeout_seconds == 0 {
779 return Err(Error::Codec(
780 "run_timeout_seconds must be at least 1".to_string(),
781 ));
782 }
783 if self.run_timeout_seconds > self.execution_timeout_seconds {
784 return Err(Error::Codec(
785 "run_timeout_seconds cannot exceed execution_timeout_seconds".to_string(),
786 ));
787 }
788
789 Ok(())
790 }
791}
792
793#[derive(Clone, Debug, Default, PartialEq, Eq)]
798pub struct ContinueAsNewOptions {
799 pub workflow_type: Option<String>,
800 pub task_queue: Option<String>,
801}
802
803impl ContinueAsNewOptions {
804 pub fn new() -> Self {
805 Self::default()
806 }
807
808 pub fn workflow_type(mut self, workflow_type: impl Into<String>) -> Self {
809 self.workflow_type = Some(workflow_type.into());
810 self
811 }
812
813 pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
814 self.task_queue = Some(task_queue.into());
815 self
816 }
817
818 fn validate(&self) -> std::result::Result<(), ContinueAsNewOptionsError> {
819 for (field, value) in [
820 ("workflow_type", self.workflow_type.as_deref()),
821 ("task_queue", self.task_queue.as_deref()),
822 ] {
823 if value.is_some_and(|value| value.trim().is_empty()) {
824 return Err(ContinueAsNewOptionsError {
825 field,
826 message: format!("{field} must not be empty"),
827 });
828 }
829 }
830 Ok(())
831 }
832}
833
834#[derive(Clone, Debug, Error, PartialEq, Eq)]
836#[error("invalid continue-as-new option {field}: {message}")]
837pub struct ContinueAsNewOptionsError {
838 pub field: &'static str,
839 pub message: String,
840}
841
842#[derive(Clone, Debug, Default, PartialEq, Eq)]
844pub struct WorkflowHistoryBudget {
845 pub event_count: u64,
846 pub size_bytes: Option<u64>,
847 pub continue_as_new_recommended: bool,
848 pub pressure: Option<String>,
849}
850
851#[doc(hidden)]
852#[derive(Clone, Debug)]
853pub struct ContinueAsNewRequest {
854 arguments: AvroValue,
855 options: ContinueAsNewOptions,
856}
857
858impl WorkflowCommandOptions {
859 pub fn new() -> Self {
860 Self::default()
861 }
862
863 pub fn reason(mut self, reason: impl Into<String>) -> Self {
864 self.reason = Some(reason.into());
865 self
866 }
867
868 pub fn request_id(mut self, request_id: impl Into<String>) -> Self {
869 self.request_id = Some(request_id.into());
870 self
871 }
872}
873
874#[derive(Clone, Debug, PartialEq)]
876pub struct WorkflowCommandResult {
877 pub command: WorkflowCommandKind,
878 pub workflow_id: String,
879 pub run_id: Option<String>,
880 pub outcome: Option<String>,
881 pub reason: Option<String>,
882 pub command_status: Option<String>,
883 pub raw: Value,
884}
885
886#[derive(Clone, Debug, PartialEq)]
888pub struct WorkflowRedriveResult {
889 pub workflow_id: String,
890 pub source_run_id: String,
891 pub run_id: String,
892 pub resume_step_sequence: u64,
893 pub raw: Value,
894}
895
896#[derive(Clone, Debug, Error)]
898#[error("workflow {command:?} rejected ({reason}, HTTP {status}): {message}")]
899pub struct WorkflowCommandRejection {
900 pub command: WorkflowCommandKind,
901 pub status: u16,
902 pub reason: String,
903 pub message: String,
904 pub workflow_id: String,
905 pub run_id: Option<String>,
906 pub target_scope: Option<String>,
907 pub body: Value,
908}
909
910#[derive(Clone, Copy, Debug, PartialEq, Eq)]
912pub enum WorkflowTerminalKind {
913 Failed,
914 Cancelled,
915 Terminated,
916 TimedOut,
917}
918
919#[derive(Clone, Debug, Error)]
925#[error("workflow {workflow_id} run {run_id:?} ended as {kind:?} ({reason})")]
926pub struct WorkflowTerminalOutcome {
927 pub kind: WorkflowTerminalKind,
928 pub workflow_id: String,
929 pub run_id: Option<String>,
930 pub reason: String,
931 pub failure_category: Option<String>,
932 pub failure_id: Option<String>,
933 pub exception_type: Option<String>,
934 pub exception_class: Option<String>,
935 pub non_retryable: Option<bool>,
936 pub message: Option<String>,
937 pub exception: Option<Value>,
938 pub raw: Value,
939}
940
941#[derive(Clone, Debug, Error)]
943#[error("activity task {operation} rejected ({reason}, HTTP {status})")]
944pub struct ActivityTaskRejection {
945 pub operation: String,
946 pub status: u16,
947 pub reason: String,
948 pub task_id: String,
949 pub activity_attempt_id: String,
950 pub cancel_requested: bool,
951 pub can_continue: Option<bool>,
952 pub run_closed_reason: Option<String>,
953 pub body: Value,
954}
955
956#[derive(Clone, Copy, Debug, PartialEq, Eq)]
958pub enum ActivityOptionsErrorKind {
959 EmptyTaskQueue,
960 EmptyRetryPolicy,
961 InvalidMaxAttempts,
962 BackoffWithoutRetryBudget,
963 TooManyBackoffIntervals,
964 InvalidBackoffCoefficient,
965 BackoffGenerationTooLarge,
966 BackoffOverflow,
967 EmptyNonRetryableErrorType,
968 TimeoutNotPositive,
969 TimeoutOverflow,
970 TimeoutOrder,
971}
972
973#[derive(Clone, Debug, Error, PartialEq, Eq)]
975#[error("invalid activity options ({kind:?}, {field:?}): {message}")]
976pub struct ActivityOptionsError {
977 pub kind: ActivityOptionsErrorKind,
978 pub field: Option<&'static str>,
979 pub message: String,
980}
981
982impl ActivityOptionsError {
983 fn new(
984 kind: ActivityOptionsErrorKind,
985 field: Option<&'static str>,
986 message: impl Into<String>,
987 ) -> Self {
988 Self {
989 kind,
990 field,
991 message: message.into(),
992 }
993 }
994}
995
996#[derive(Clone, Copy, Debug, PartialEq, Eq)]
998pub enum ActivityFailureKind {
999 Failed,
1000 Cancelled,
1001 TimedOut,
1002}
1003
1004#[derive(Clone, Debug, Error, PartialEq)]
1009#[error("activity failed ({reason}): {message}")]
1010pub struct ActivityFailure {
1011 pub kind: ActivityFailureKind,
1012 pub reason: String,
1013 pub message: String,
1014 pub activity_execution_id: Option<String>,
1015 pub activity_attempt_id: Option<String>,
1016 pub activity_type: Option<String>,
1017 pub activity_class: Option<String>,
1018 pub attempt_number: Option<u64>,
1019 pub failure_id: Option<String>,
1020 pub failure_category: Option<String>,
1021 pub timeout_kind: Option<String>,
1022 pub non_retryable: bool,
1023 pub exception_type: Option<String>,
1024 pub exception_class: Option<String>,
1025 pub code: Option<Value>,
1026 pub exception: Option<Value>,
1027}
1028
1029#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1031pub enum ChildWorkflowFailureKind {
1032 Failed,
1033 Cancelled,
1034 Terminated,
1035}
1036
1037#[derive(Clone, Debug, Error)]
1043#[error("child workflow failed ({reason}): {message}")]
1044pub struct ChildWorkflowFailure {
1045 pub kind: ChildWorkflowFailureKind,
1046 pub reason: String,
1047 pub message: String,
1048 pub parent_workflow_id: Option<String>,
1049 pub parent_workflow_run_id: Option<String>,
1050 pub child_workflow_id: Option<String>,
1051 pub child_workflow_run_id: Option<String>,
1052 pub child_workflow_type: Option<String>,
1053 pub failure_id: Option<String>,
1054 pub failure_category: Option<String>,
1055 pub exception_type: Option<String>,
1056 pub exception_class: Option<String>,
1057 pub non_retryable: bool,
1058 pub code: Option<Value>,
1059 pub exception: Option<Value>,
1060}
1061
1062#[derive(Clone, Debug, PartialEq, Eq)]
1064pub struct WorkflowIdentity {
1065 pub workflow_id: Option<String>,
1066 pub run_id: Option<String>,
1067}
1068
1069#[derive(Clone, Debug, PartialEq)]
1071pub struct ChildWorkflowResult {
1072 pub parent: WorkflowIdentity,
1073 pub child: WorkflowIdentity,
1074 pub child_workflow_type: Option<String>,
1075 pub result: Value,
1076}
1077
1078#[derive(Clone, Debug, PartialEq)]
1080pub struct ChildWorkflowAvroResult {
1081 pub parent: WorkflowIdentity,
1082 pub child: WorkflowIdentity,
1083 pub child_workflow_type: Option<String>,
1084 pub result: AvroValue,
1085}
1086
1087#[derive(Clone, Debug, Deserialize, Hash, PartialEq, Eq, Serialize)]
1089#[serde(untagged)]
1090pub enum SelectionKey {
1091 Index(usize),
1092 Name(String),
1093}
1094
1095impl From<usize> for SelectionKey {
1096 fn from(value: usize) -> Self {
1097 Self::Index(value)
1098 }
1099}
1100
1101impl From<String> for SelectionKey {
1102 fn from(value: String) -> Self {
1103 Self::Name(value)
1104 }
1105}
1106
1107impl From<&str> for SelectionKey {
1108 fn from(value: &str) -> Self {
1109 Self::Name(value.to_string())
1110 }
1111}
1112
1113#[derive(Clone, Debug, Error, PartialEq, Eq)]
1115#[error("selected {operation_kind} operation {operation_identity} was explicitly cancelled")]
1116pub struct DurableOperationCancelled {
1117 pub selection_group_id: String,
1118 pub member_key: SelectionKey,
1119 pub member_index: usize,
1120 pub operation_kind: String,
1121 pub operation_identity: String,
1122}
1123
1124#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
1130pub struct ParallelGroupMetadata {
1131 pub parallel_group_id: String,
1132 pub parallel_group_kind: String,
1133 pub parallel_group_base_sequence: u64,
1134 pub parallel_group_size: usize,
1135 pub parallel_group_index: usize,
1136 #[serde(default, skip_serializing_if = "Option::is_none")]
1137 pub parallel_group_mode: Option<String>,
1138 #[serde(default, skip_serializing_if = "Option::is_none")]
1139 pub selection_member_key: Option<SelectionKey>,
1140 #[serde(default, skip_serializing_if = "Option::is_none")]
1141 pub selection_member_index: Option<usize>,
1142 #[serde(default, skip_serializing_if = "Option::is_none")]
1143 pub selection_member_base_sequence: Option<u64>,
1144 #[serde(default, skip_serializing_if = "Option::is_none")]
1145 pub selection_member_size: Option<usize>,
1146 #[serde(default, skip_serializing_if = "Option::is_none")]
1147 pub selection_member_kind: Option<String>,
1148}
1149
1150#[derive(Clone, Debug, PartialEq)]
1152pub enum ParallelResult {
1153 Activity(Value),
1154 ChildWorkflow(ChildWorkflowResult),
1155 Timer,
1156 Signal(Vec<Value>),
1157 Condition(ConditionWaitResult),
1158 Group(Vec<ParallelResult>),
1159}
1160
1161#[derive(Clone, Debug, PartialEq)]
1163pub enum ParallelAvroResult {
1164 Activity(AvroValue),
1165 ChildWorkflow(ChildWorkflowAvroResult),
1166 Timer,
1167 Signal(Vec<AvroValue>),
1168 Condition(ConditionWaitResult),
1169 Group(Vec<ParallelAvroResult>),
1170}
1171
1172impl ParallelAvroResult {
1173 fn into_json_result(self) -> Result<ParallelResult> {
1174 match self {
1175 Self::Activity(value) => Ok(ParallelResult::Activity(value.into_json()?)),
1176 Self::ChildWorkflow(result) => Ok(ParallelResult::ChildWorkflow(ChildWorkflowResult {
1177 parent: result.parent,
1178 child: result.child,
1179 child_workflow_type: result.child_workflow_type,
1180 result: result.result.into_json()?,
1181 })),
1182 Self::Timer => Ok(ParallelResult::Timer),
1183 Self::Signal(values) => Ok(ParallelResult::Signal(
1184 values
1185 .into_iter()
1186 .map(AvroValue::into_json)
1187 .collect::<Result<Vec<_>>>()?,
1188 )),
1189 Self::Condition(result) => Ok(ParallelResult::Condition(result)),
1190 Self::Group(results) => Ok(ParallelResult::Group(
1191 results
1192 .into_iter()
1193 .map(Self::into_json_result)
1194 .collect::<Result<Vec<_>>>()?,
1195 )),
1196 }
1197 }
1198}
1199
1200#[derive(Clone, Debug, PartialEq)]
1202pub struct ParallelCompletion {
1203 pub member_path: Vec<usize>,
1204 pub result: ParallelResult,
1205}
1206
1207#[derive(Debug, Error)]
1215#[error("parallel group {group_id} member {member_path:?} failed: {cause}")]
1216pub struct ParallelFailure {
1217 pub group_id: String,
1218 pub member_path: Vec<usize>,
1219 pub group_path: Vec<ParallelGroupMetadata>,
1220 pub completed: Vec<ParallelCompletion>,
1221 #[source]
1222 pub cause: Box<Error>,
1223}
1224
1225#[derive(Clone, Debug, Error, PartialEq, Eq)]
1227#[error("invalid deterministic parallel group ({reason}): {message}")]
1228pub struct ParallelGroupError {
1229 pub reason: &'static str,
1230 pub member_path: Vec<usize>,
1231 pub message: String,
1232}
1233
1234#[derive(Clone, Debug, Error, PartialEq, Eq)]
1236#[error("workflow cancellation was requested")]
1237pub struct WorkflowCancellationRequested;
1238
1239#[derive(Debug, Error)]
1241#[error(
1242 "saga forward execution failed; compensation activity {compensation_activity_type} (registration {compensation_registration_order}) also failed: {compensation_failure}"
1243)]
1244pub struct SagaCompensationFailure {
1245 pub initiating_failure: Box<Error>,
1246 pub compensation_failure: Box<Error>,
1247 pub compensation_activity_type: String,
1248 pub compensation_registration_order: usize,
1249}
1250
1251#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
1253pub enum ParentClosePolicy {
1254 #[default]
1255 Abandon,
1256 RequestCancel,
1257 Terminate,
1258}
1259
1260impl ParentClosePolicy {
1261 fn as_str(self) -> &'static str {
1262 match self {
1263 Self::Abandon => "abandon",
1264 Self::RequestCancel => "request_cancel",
1265 Self::Terminate => "terminate",
1266 }
1267 }
1268}
1269
1270#[derive(Clone, Debug, Default, PartialEq, Eq)]
1272pub struct ChildWorkflowRetryPolicy {
1273 pub max_attempts: Option<u32>,
1274 pub backoff_seconds: Vec<u64>,
1275 pub non_retryable_error_types: Vec<String>,
1276}
1277
1278#[derive(Clone, Debug, PartialEq, Eq)]
1282pub struct ChildWorkflowOptions {
1283 pub task_queue: String,
1284 pub parent_close_policy: ParentClosePolicy,
1285 pub retry_policy: Option<ChildWorkflowRetryPolicy>,
1286 pub execution_timeout_seconds: Option<u64>,
1287 pub run_timeout_seconds: Option<u64>,
1288}
1289
1290impl ChildWorkflowOptions {
1291 pub fn new(task_queue: impl Into<String>) -> Self {
1292 Self {
1293 task_queue: task_queue.into(),
1294 parent_close_policy: ParentClosePolicy::Abandon,
1295 retry_policy: None,
1296 execution_timeout_seconds: None,
1297 run_timeout_seconds: None,
1298 }
1299 }
1300
1301 pub fn parent_close_policy(mut self, policy: ParentClosePolicy) -> Self {
1302 self.parent_close_policy = policy;
1303 self
1304 }
1305
1306 pub fn retry_policy(mut self, policy: ChildWorkflowRetryPolicy) -> Self {
1307 self.retry_policy = Some(policy);
1308 self
1309 }
1310
1311 pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
1312 self.execution_timeout_seconds = Some(seconds);
1313 self
1314 }
1315
1316 pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
1317 self.run_timeout_seconds = Some(seconds);
1318 self
1319 }
1320}
1321
1322#[derive(Clone, Debug, PartialEq, Eq)]
1324pub enum ActivityBackoff {
1325 Explicit(Vec<Duration>),
1328 Exponential {
1330 initial_interval: Duration,
1331 coefficient: u32,
1332 maximum_interval: Option<Duration>,
1333 },
1334}
1335
1336#[derive(Clone, Debug, Default, PartialEq, Eq)]
1338pub struct ActivityRetryPolicy {
1339 pub max_attempts: Option<u32>,
1340 pub backoff: Option<ActivityBackoff>,
1341 pub non_retryable_error_types: Vec<String>,
1342}
1343
1344impl ActivityRetryPolicy {
1345 pub fn new(max_attempts: u32) -> Self {
1347 Self {
1348 max_attempts: Some(max_attempts),
1349 ..Self::default()
1350 }
1351 }
1352
1353 pub fn backoff_intervals(mut self, intervals: impl IntoIterator<Item = Duration>) -> Self {
1354 self.backoff = Some(ActivityBackoff::Explicit(intervals.into_iter().collect()));
1355 self
1356 }
1357
1358 pub fn exponential_backoff(
1359 mut self,
1360 initial_interval: Duration,
1361 coefficient: u32,
1362 maximum_interval: Option<Duration>,
1363 ) -> Self {
1364 self.backoff = Some(ActivityBackoff::Exponential {
1365 initial_interval,
1366 coefficient,
1367 maximum_interval,
1368 });
1369 self
1370 }
1371
1372 pub fn non_retryable_error_type(mut self, error_type: impl Into<String>) -> Self {
1373 self.non_retryable_error_types.push(error_type.into());
1374 self
1375 }
1376
1377 pub fn non_retryable_error_types(
1378 mut self,
1379 error_types: impl IntoIterator<Item = impl Into<String>>,
1380 ) -> Self {
1381 self.non_retryable_error_types
1382 .extend(error_types.into_iter().map(Into::into));
1383 self
1384 }
1385}
1386
1387#[derive(Clone, Debug, Default, PartialEq, Eq)]
1392pub struct ActivityOptions {
1393 pub task_queue: Option<String>,
1394 pub retry_policy: Option<ActivityRetryPolicy>,
1395 pub start_to_close_timeout: Option<Duration>,
1396 pub schedule_to_start_timeout: Option<Duration>,
1397 pub schedule_to_close_timeout: Option<Duration>,
1398 pub heartbeat_timeout: Option<Duration>,
1399}
1400
1401impl ActivityOptions {
1402 pub fn new() -> Self {
1403 Self::default()
1404 }
1405
1406 pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
1407 self.task_queue = Some(task_queue.into());
1408 self
1409 }
1410
1411 pub fn retry_policy(mut self, policy: ActivityRetryPolicy) -> Self {
1412 self.retry_policy = Some(policy);
1413 self
1414 }
1415
1416 pub fn start_to_close_timeout(mut self, timeout: Duration) -> Self {
1417 self.start_to_close_timeout = Some(timeout);
1418 self
1419 }
1420
1421 pub fn schedule_to_start_timeout(mut self, timeout: Duration) -> Self {
1422 self.schedule_to_start_timeout = Some(timeout);
1423 self
1424 }
1425
1426 pub fn schedule_to_close_timeout(mut self, timeout: Duration) -> Self {
1427 self.schedule_to_close_timeout = Some(timeout);
1428 self
1429 }
1430
1431 pub fn heartbeat_timeout(mut self, timeout: Duration) -> Self {
1432 self.heartbeat_timeout = Some(timeout);
1433 self
1434 }
1435
1436 fn validate(&self) -> std::result::Result<ValidatedActivityOptions, ActivityOptionsError> {
1437 if self
1438 .task_queue
1439 .as_deref()
1440 .is_some_and(|queue| queue.trim().is_empty())
1441 {
1442 return Err(ActivityOptionsError::new(
1443 ActivityOptionsErrorKind::EmptyTaskQueue,
1444 Some("task_queue"),
1445 "task_queue must not be empty",
1446 ));
1447 }
1448
1449 for (field, value) in [
1450 ("start_to_close_timeout", self.start_to_close_timeout),
1451 ("schedule_to_start_timeout", self.schedule_to_start_timeout),
1452 ("schedule_to_close_timeout", self.schedule_to_close_timeout),
1453 ("heartbeat_timeout", self.heartbeat_timeout),
1454 ] {
1455 if value.is_some_and(|value| value.is_zero()) {
1456 return Err(ActivityOptionsError::new(
1457 ActivityOptionsErrorKind::TimeoutNotPositive,
1458 Some(field),
1459 format!("{field} must be positive"),
1460 ));
1461 }
1462 }
1463
1464 validate_timeout_order(
1465 "heartbeat_timeout",
1466 self.heartbeat_timeout,
1467 "start_to_close_timeout",
1468 self.start_to_close_timeout,
1469 )?;
1470 validate_timeout_order(
1471 "start_to_close_timeout",
1472 self.start_to_close_timeout,
1473 "schedule_to_close_timeout",
1474 self.schedule_to_close_timeout,
1475 )?;
1476 validate_timeout_order(
1477 "schedule_to_start_timeout",
1478 self.schedule_to_start_timeout,
1479 "schedule_to_close_timeout",
1480 self.schedule_to_close_timeout,
1481 )?;
1482
1483 Ok(ValidatedActivityOptions {
1484 task_queue: self.task_queue.clone(),
1485 retry_policy: self
1486 .retry_policy
1487 .as_ref()
1488 .map(validate_activity_retry_policy)
1489 .transpose()?,
1490 start_to_close_timeout: timeout_seconds(
1491 "start_to_close_timeout",
1492 self.start_to_close_timeout,
1493 )?,
1494 schedule_to_start_timeout: timeout_seconds(
1495 "schedule_to_start_timeout",
1496 self.schedule_to_start_timeout,
1497 )?,
1498 schedule_to_close_timeout: timeout_seconds(
1499 "schedule_to_close_timeout",
1500 self.schedule_to_close_timeout,
1501 )?,
1502 heartbeat_timeout: timeout_seconds("heartbeat_timeout", self.heartbeat_timeout)?,
1503 })
1504 }
1505}
1506
1507pub enum ParallelOperation {
1513 Activity {
1514 activity_type: String,
1515 options: ActivityOptions,
1516 arguments: Result<AvroValue>,
1517 },
1518 ChildWorkflow {
1519 workflow_type: String,
1520 options: ChildWorkflowOptions,
1521 arguments: Result<AvroValue>,
1522 },
1523 Timer(Duration),
1524 Signal(String),
1525 Condition {
1526 options: ConditionWaitOptions,
1527 predicate: Box<dyn Fn() -> Result<bool> + Send + 'static>,
1528 },
1529 Group(Vec<ParallelOperation>),
1530}
1531
1532impl ParallelOperation {
1533 pub fn activity<T: Serialize>(activity_type: impl Into<String>, args: T) -> Self {
1534 Self::activity_with_options(activity_type, ActivityOptions::new(), args)
1535 }
1536
1537 pub fn activity_with_options<T: Serialize>(
1538 activity_type: impl Into<String>,
1539 options: ActivityOptions,
1540 args: T,
1541 ) -> Self {
1542 Self::Activity {
1543 activity_type: activity_type.into(),
1544 options,
1545 arguments: AvroValue::from_serialize(&args),
1546 }
1547 }
1548
1549 pub fn child_workflow<T: Serialize>(
1550 workflow_type: impl Into<String>,
1551 options: ChildWorkflowOptions,
1552 args: T,
1553 ) -> Self {
1554 Self::ChildWorkflow {
1555 workflow_type: workflow_type.into(),
1556 options,
1557 arguments: AvroValue::from_serialize(&args),
1558 }
1559 }
1560
1561 pub fn timer(duration: Duration) -> Self {
1562 Self::Timer(duration)
1563 }
1564
1565 pub fn signal(signal_name: impl Into<String>) -> Self {
1566 Self::Signal(signal_name.into())
1567 }
1568
1569 pub fn condition<F>(options: ConditionWaitOptions, predicate: F) -> Self
1570 where
1571 F: Fn() -> Result<bool> + Send + 'static,
1572 {
1573 Self::Condition {
1574 options,
1575 predicate: Box::new(predicate),
1576 }
1577 }
1578
1579 pub fn group(operations: Vec<ParallelOperation>) -> Self {
1580 Self::Group(operations)
1581 }
1582}
1583
1584#[derive(Clone, Debug)]
1585struct ValidatedActivityOptions {
1586 task_queue: Option<String>,
1587 retry_policy: Option<Value>,
1588 start_to_close_timeout: Option<u64>,
1589 schedule_to_start_timeout: Option<u64>,
1590 schedule_to_close_timeout: Option<u64>,
1591 heartbeat_timeout: Option<u64>,
1592}
1593
1594fn validate_timeout_order(
1595 smaller_name: &'static str,
1596 smaller: Option<Duration>,
1597 larger_name: &'static str,
1598 larger: Option<Duration>,
1599) -> std::result::Result<(), ActivityOptionsError> {
1600 if matches!((smaller, larger), (Some(smaller), Some(larger)) if smaller > larger) {
1601 return Err(ActivityOptionsError::new(
1602 ActivityOptionsErrorKind::TimeoutOrder,
1603 Some(smaller_name),
1604 format!("{smaller_name} must be <= {larger_name}"),
1605 ));
1606 }
1607 Ok(())
1608}
1609
1610fn timeout_seconds(
1611 field: &'static str,
1612 value: Option<Duration>,
1613) -> std::result::Result<Option<u64>, ActivityOptionsError> {
1614 value
1615 .map(|value| {
1616 activity_protocol_seconds(value).ok_or_else(|| {
1617 ActivityOptionsError::new(
1618 ActivityOptionsErrorKind::TimeoutOverflow,
1619 Some(field),
1620 format!("{field} is too large for the worker protocol"),
1621 )
1622 })
1623 })
1624 .transpose()
1625}
1626
1627fn duration_seconds_ceil(value: Duration) -> Option<u64> {
1628 value
1629 .as_secs()
1630 .checked_add(u64::from(value.subsec_nanos() > 0))
1631}
1632
1633fn activity_protocol_seconds(value: Duration) -> Option<u64> {
1634 duration_seconds_ceil(value).filter(|seconds| *seconds <= i64::MAX as u64)
1635}
1636
1637fn validate_activity_retry_policy(
1638 policy: &ActivityRetryPolicy,
1639) -> std::result::Result<Value, ActivityOptionsError> {
1640 if policy.max_attempts.is_none()
1641 && policy.backoff.is_none()
1642 && policy.non_retryable_error_types.is_empty()
1643 {
1644 return Err(ActivityOptionsError::new(
1645 ActivityOptionsErrorKind::EmptyRetryPolicy,
1646 Some("retry_policy"),
1647 "retry_policy must configure at least one field",
1648 ));
1649 }
1650 if policy.max_attempts == Some(0) {
1651 return Err(ActivityOptionsError::new(
1652 ActivityOptionsErrorKind::InvalidMaxAttempts,
1653 Some("retry_policy.max_attempts"),
1654 "max_attempts must be >= 1",
1655 ));
1656 }
1657 if policy
1658 .non_retryable_error_types
1659 .iter()
1660 .any(|error_type| error_type.trim().is_empty())
1661 {
1662 return Err(ActivityOptionsError::new(
1663 ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
1664 Some("retry_policy.non_retryable_error_types"),
1665 "non_retryable_error_types must not contain empty values",
1666 ));
1667 }
1668
1669 let backoff_seconds = match &policy.backoff {
1670 None => None,
1671 Some(backoff) => {
1672 let max_attempts = policy.max_attempts.ok_or_else(|| {
1673 ActivityOptionsError::new(
1674 ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
1675 Some("retry_policy.backoff"),
1676 "backoff requires max_attempts",
1677 )
1678 })?;
1679 let retry_count = max_attempts.saturating_sub(1) as usize;
1680 let intervals = match backoff {
1681 ActivityBackoff::Explicit(intervals) => {
1682 if intervals.len() > retry_count {
1683 return Err(ActivityOptionsError::new(
1684 ActivityOptionsErrorKind::TooManyBackoffIntervals,
1685 Some("retry_policy.backoff"),
1686 "backoff interval count must not exceed max_attempts - 1",
1687 ));
1688 }
1689 intervals.clone()
1690 }
1691 ActivityBackoff::Exponential {
1692 initial_interval,
1693 coefficient,
1694 maximum_interval,
1695 } => {
1696 if *coefficient < 1 {
1697 return Err(ActivityOptionsError::new(
1698 ActivityOptionsErrorKind::InvalidBackoffCoefficient,
1699 Some("retry_policy.backoff.coefficient"),
1700 "backoff coefficient must be >= 1",
1701 ));
1702 }
1703 if retry_count > 10_000 {
1704 return Err(ActivityOptionsError::new(
1705 ActivityOptionsErrorKind::BackoffGenerationTooLarge,
1706 Some("retry_policy.max_attempts"),
1707 "generated backoff supports at most 10000 retry intervals",
1708 ));
1709 }
1710 let mut current = *initial_interval;
1711 let mut intervals = Vec::with_capacity(retry_count);
1712 for _ in 0..retry_count {
1713 let interval = maximum_interval
1714 .map(|maximum| current.min(maximum))
1715 .unwrap_or(current);
1716 intervals.push(interval);
1717 if maximum_interval.is_some_and(|maximum| interval == maximum) {
1718 break;
1719 }
1720 current = current.checked_mul(*coefficient).ok_or_else(|| {
1721 ActivityOptionsError::new(
1722 ActivityOptionsErrorKind::BackoffOverflow,
1723 Some("retry_policy.backoff"),
1724 "generated backoff interval overflowed",
1725 )
1726 })?;
1727 }
1728 intervals
1729 }
1730 };
1731 Some(
1732 intervals
1733 .into_iter()
1734 .map(|interval| {
1735 activity_protocol_seconds(interval).ok_or_else(|| {
1736 ActivityOptionsError::new(
1737 ActivityOptionsErrorKind::BackoffOverflow,
1738 Some("retry_policy.backoff"),
1739 "backoff interval is too large for the worker protocol",
1740 )
1741 })
1742 })
1743 .collect::<std::result::Result<Vec<_>, _>>()?,
1744 )
1745 }
1746 };
1747
1748 let mut encoded = serde_json::Map::new();
1749 if let Some(max_attempts) = policy.max_attempts {
1750 encoded.insert("max_attempts".to_string(), json!(max_attempts));
1751 }
1752 if let Some(backoff_seconds) = backoff_seconds {
1753 encoded.insert("backoff_seconds".to_string(), json!(backoff_seconds));
1754 }
1755 if !policy.non_retryable_error_types.is_empty() {
1756 let mut canonical_error_types = Vec::new();
1757 for error_type in policy
1758 .non_retryable_error_types
1759 .iter()
1760 .map(|error_type| error_type.trim())
1761 {
1762 if !canonical_error_types.contains(&error_type) {
1763 canonical_error_types.push(error_type);
1764 }
1765 }
1766 encoded.insert(
1767 "non_retryable_error_types".to_string(),
1768 json!(canonical_error_types),
1769 );
1770 }
1771 Ok(Value::Object(encoded))
1772}
1773
1774#[derive(Clone, Debug, Error)]
1777#[error("non-deterministic workflow replay ({reason}) at sequence {sequence:?}: {message}")]
1778pub struct ReplayFailure {
1779 pub reason: String,
1780 pub sequence: Option<u64>,
1781 pub expected: Option<String>,
1782 pub actual: Option<String>,
1783 pub message: String,
1784}
1785
1786impl ReplayFailure {
1787 fn new(
1788 reason: impl Into<String>,
1789 sequence: Option<u64>,
1790 expected: Option<String>,
1791 actual: Option<String>,
1792 message: impl Into<String>,
1793 ) -> Self {
1794 Self {
1795 reason: reason.into(),
1796 sequence,
1797 expected,
1798 actual,
1799 message: message.into(),
1800 }
1801 }
1802}
1803
1804#[derive(Clone, Debug, Error)]
1806#[error("query failed ({reason}, HTTP {status}): {message}")]
1807pub struct QueryFailure {
1808 pub status: u16,
1809 pub reason: String,
1810 pub message: String,
1811 pub body: Value,
1812}
1813
1814#[derive(Clone, Debug, Error)]
1816#[error("protocol rejected ({reason}, HTTP {status}): {message}")]
1817pub struct ProtocolFailure {
1818 pub status: u16,
1819 pub reason: String,
1820 pub message: String,
1821 pub supported_version: Option<String>,
1822 pub requested_version: Option<String>,
1823 pub body: Value,
1824}
1825
1826#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
1827pub struct PayloadEnvelope {
1828 pub codec: String,
1829 pub blob: String,
1830}
1831
1832impl PayloadEnvelope {
1833 pub fn avro<T: Serialize>(value: &T) -> Result<Self> {
1834 encode_payload(value, DEFAULT_CODEC)
1835 }
1836
1837 pub fn avro_value(value: &AvroValue) -> Result<Self> {
1840 encode_avro_value(value)
1841 }
1842}
1843
1844#[derive(Clone, Debug)]
1846pub enum AvroValue {
1847 Null,
1848 Boolean(bool),
1849 Long(i64),
1850 Double(f64),
1851 Bytes(Vec<u8>),
1852 String(String),
1853 Array(Vec<AvroValue>),
1854 Map(BTreeMap<String, AvroValue>),
1855}
1856
1857impl PartialEq for AvroValue {
1858 fn eq(&self, other: &Self) -> bool {
1859 match (self, other) {
1860 (Self::Null, Self::Null) => true,
1861 (Self::Boolean(left), Self::Boolean(right)) => left == right,
1862 (Self::Long(left), Self::Long(right)) => left == right,
1863 (Self::Double(left), Self::Double(right)) => left.to_bits() == right.to_bits(),
1864 (Self::Bytes(left), Self::Bytes(right)) => left == right,
1865 (Self::String(left), Self::String(right)) => left == right,
1866 (Self::Array(left), Self::Array(right)) => left == right,
1867 (Self::Map(left), Self::Map(right)) => left == right,
1868 _ => false,
1869 }
1870 }
1871}
1872
1873impl AvroValue {
1874 fn from_serialize<T: Serialize>(value: &T) -> Result<Self> {
1875 Self::from_serde_value(
1876 serde_value::to_value(value).map_err(|error| {
1877 Error::Codec(format!("could not adapt value for Avro: {error}"))
1878 })?,
1879 )
1880 }
1881
1882 fn from_serde_value(value: serde_value::Value) -> Result<Self> {
1883 use serde_value::Value as SerdeValue;
1884
1885 match value {
1886 SerdeValue::Unit => Ok(Self::Null),
1887 SerdeValue::Bool(value) => Ok(Self::Boolean(value)),
1888 SerdeValue::I8(value) => Ok(Self::Long(i64::from(value))),
1889 SerdeValue::I16(value) => Ok(Self::Long(i64::from(value))),
1890 SerdeValue::I32(value) => Ok(Self::Long(i64::from(value))),
1891 SerdeValue::I64(value) => Ok(Self::Long(value)),
1892 SerdeValue::U8(value) => Ok(Self::Long(i64::from(value))),
1893 SerdeValue::U16(value) => Ok(Self::Long(i64::from(value))),
1894 SerdeValue::U32(value) => Ok(Self::Long(i64::from(value))),
1895 SerdeValue::U64(value) => i64::try_from(value).map(Self::Long).map_err(|_| {
1896 Error::Codec(
1897 "integer_overflow: Avro Value long must be within signed 64-bit range"
1898 .to_string(),
1899 )
1900 }),
1901 SerdeValue::F32(value) => Self::finite_double(f64::from(value)),
1902 SerdeValue::F64(value) => Self::finite_double(value),
1903 SerdeValue::Char(value) => Ok(Self::String(value.to_string())),
1904 SerdeValue::String(value) => Ok(Self::String(value)),
1905 SerdeValue::Bytes(value) => Ok(Self::Bytes(value)),
1906 SerdeValue::Option(None) => Ok(Self::Null),
1907 SerdeValue::Option(Some(value)) | SerdeValue::Newtype(value) => {
1908 Self::from_serde_value(*value)
1909 }
1910 SerdeValue::Seq(values) => values
1911 .into_iter()
1912 .map(Self::from_serde_value)
1913 .collect::<Result<Vec<_>>>()
1914 .map(Self::Array),
1915 SerdeValue::Map(values) => values
1916 .into_iter()
1917 .map(|(key, value)| {
1918 let SerdeValue::String(key) = key else {
1919 return Err(Error::Codec(
1920 "invalid_map_key: Avro Value map keys must be strings".to_string(),
1921 ));
1922 };
1923
1924 Ok((key, Self::from_serde_value(value)?))
1925 })
1926 .collect::<Result<BTreeMap<_, _>>>()
1927 .map(Self::Map),
1928 }
1929 }
1930
1931 fn finite_double(value: f64) -> Result<Self> {
1932 if !value.is_finite() {
1933 return Err(Error::Codec(
1934 "non_finite_float: Avro Value doubles must be finite".to_string(),
1935 ));
1936 }
1937
1938 Ok(Self::Double(value))
1939 }
1940
1941 fn into_json(self) -> Result<Value> {
1942 match self {
1943 Self::Null => Ok(Value::Null),
1944 Self::Boolean(value) => Ok(Value::Bool(value)),
1945 Self::Long(value) => Ok(Value::Number(value.into())),
1946 Self::Double(value) => serde_json::Number::from_f64(value)
1947 .map(Value::Number)
1948 .ok_or_else(|| {
1949 Error::Codec(
1950 "non_finite_float: decoded Avro Value double is not finite".to_string(),
1951 )
1952 }),
1953 Self::Bytes(value) => Ok(json!({
1954 "$type": "bytes",
1955 "base64": BASE64.encode(value),
1956 })),
1957 Self::String(value) => Ok(Value::String(value)),
1958 Self::Array(values) => values
1959 .into_iter()
1960 .map(Self::into_json)
1961 .collect::<Result<Vec<_>>>()
1962 .map(Value::Array),
1963 Self::Map(values) => values
1964 .into_iter()
1965 .map(|(key, value)| Ok((key, value.into_json()?)))
1966 .collect::<Result<serde_json::Map<_, _>>>()
1967 .map(Value::Object),
1968 }
1969 }
1970
1971 fn into_serde_value(self) -> serde_value::Value {
1972 use serde_value::Value as SerdeValue;
1973
1974 match self {
1975 Self::Null => SerdeValue::Unit,
1976 Self::Boolean(value) => SerdeValue::Bool(value),
1977 Self::Long(value) => SerdeValue::I64(value),
1978 Self::Double(value) => SerdeValue::F64(value),
1979 Self::Bytes(value) => SerdeValue::Bytes(value),
1980 Self::String(value) => SerdeValue::String(value),
1981 Self::Array(values) => {
1982 SerdeValue::Seq(values.into_iter().map(Self::into_serde_value).collect())
1983 }
1984 Self::Map(values) => SerdeValue::Map(
1985 values
1986 .into_iter()
1987 .map(|(key, value)| (SerdeValue::String(key), value.into_serde_value()))
1988 .collect(),
1989 ),
1990 }
1991 }
1992
1993 pub fn deserialize<T: DeserializeOwned>(self) -> Result<T> {
1994 self.into_serde_value().deserialize_into().map_err(|error| {
1995 Error::Codec(format!(
1996 "avro_value_type_mismatch: could not adapt decoded value: {error}"
1997 ))
1998 })
1999 }
2000}
2001
2002impl Serialize for AvroValue {
2003 fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
2004 where
2005 S: Serializer,
2006 {
2007 match self {
2008 Self::Null => serializer.serialize_unit(),
2009 Self::Boolean(value) => serializer.serialize_bool(*value),
2010 Self::Long(value) => serializer.serialize_i64(*value),
2011 Self::Double(value) => serializer.serialize_f64(*value),
2012 Self::Bytes(value) => serializer.serialize_bytes(value),
2013 Self::String(value) => serializer.serialize_str(value),
2014 Self::Array(values) => {
2015 let mut sequence = serializer.serialize_seq(Some(values.len()))?;
2016 for value in values {
2017 sequence.serialize_element(value)?;
2018 }
2019 sequence.end()
2020 }
2021 Self::Map(values) => {
2022 let mut map = serializer.serialize_map(Some(values.len()))?;
2023 for (key, value) in values {
2024 map.serialize_entry(key, value)?;
2025 }
2026 map.end()
2027 }
2028 }
2029 }
2030}
2031
2032pub fn encode_avro_value(value: &AvroValue) -> Result<PayloadEnvelope> {
2033 let datum = avro_value_to_datum(value)?;
2034 let datum = to_avro_datum(avro_value_ordered_map_encoding_schema()?, datum)
2035 .map_err(|err| Error::Codec(format!("avro_value_encode_failed: {err}")))?;
2036 let mut bytes = Vec::with_capacity(datum.len() + 10);
2037 bytes.extend_from_slice(&AVRO_SINGLE_OBJECT_MAGIC);
2038 bytes.extend_from_slice(&AVRO_VALUE_SCHEMA_FINGERPRINT);
2039 bytes.extend_from_slice(&datum);
2040 Ok(PayloadEnvelope {
2041 codec: DEFAULT_CODEC.to_string(),
2042 blob: BASE64.encode(bytes),
2043 })
2044}
2045
2046pub fn decode_avro_value(envelope: &PayloadEnvelope) -> Result<AvroValue> {
2047 if envelope.codec != DEFAULT_CODEC {
2048 return Err(unsupported_payload_codec(&envelope.codec));
2049 }
2050 decode_avro_value_blob(&envelope.blob)
2051}
2052
2053pub fn encode_payload<T: Serialize>(value: &T, codec: &str) -> Result<PayloadEnvelope> {
2054 let blob = match codec {
2055 DEFAULT_CODEC => encode_avro_value(&AvroValue::from_serialize(value)?)?.blob,
2056 other => return Err(unsupported_payload_codec(other)),
2057 };
2058
2059 Ok(PayloadEnvelope {
2060 codec: codec.to_string(),
2061 blob,
2062 })
2063}
2064
2065pub fn decode_payload<T: DeserializeOwned>(envelope: &PayloadEnvelope) -> Result<T> {
2066 match envelope.codec.as_str() {
2067 DEFAULT_CODEC => decode_avro_value(envelope)?.deserialize(),
2068 other => Err(unsupported_payload_codec(other)),
2069 }
2070}
2071
2072fn handler_type_error<T>(
2073 handler_kind: HandlerKind,
2074 handler_name: &str,
2075 value_kind: HandlerValueKind,
2076 message: impl Into<String>,
2077) -> Error {
2078 Error::HandlerType {
2079 handler_kind,
2080 handler_name: handler_name.to_string(),
2081 value_kind,
2082 rust_type: type_name::<T>(),
2083 message: message.into(),
2084 }
2085}
2086
2087fn decode_handler_input<T: DeserializeOwned>(
2088 arguments: AvroValue,
2089 handler_kind: HandlerKind,
2090 handler_name: &str,
2091) -> Result<T> {
2092 let argument = match arguments {
2093 AvroValue::Array(mut arguments) if arguments.len() == 1 => {
2094 arguments.pop().expect("one typed handler argument")
2095 }
2096 AvroValue::Array(arguments) if arguments.is_empty() => AvroValue::Null,
2097 AvroValue::Array(arguments) => {
2098 return Err(handler_type_error::<T>(
2099 handler_kind,
2100 handler_name,
2101 HandlerValueKind::Input,
2102 format!(
2103 "typed handlers accept one request value, but the task carried {} arguments",
2104 arguments.len()
2105 ),
2106 ));
2107 }
2108 argument => argument,
2109 };
2110
2111 argument.deserialize().map_err(|error| {
2112 handler_type_error::<T>(
2113 handler_kind,
2114 handler_name,
2115 HandlerValueKind::Input,
2116 error.to_string(),
2117 )
2118 })
2119}
2120
2121fn encode_handler_result<T: Serialize>(
2122 result: &T,
2123 handler_kind: HandlerKind,
2124 handler_name: &str,
2125) -> Result<AvroValue> {
2126 AvroValue::from_serialize(result).map_err(|error| {
2127 handler_type_error::<T>(
2128 handler_kind,
2129 handler_name,
2130 HandlerValueKind::Result,
2131 error.to_string(),
2132 )
2133 })
2134}
2135
2136fn decode_handler_result<T: DeserializeOwned>(
2137 result: AvroValue,
2138 handler_kind: HandlerKind,
2139 handler_name: &str,
2140) -> Result<T> {
2141 result.deserialize().map_err(|error| {
2142 handler_type_error::<T>(
2143 handler_kind,
2144 handler_name,
2145 HandlerValueKind::Result,
2146 error.to_string(),
2147 )
2148 })
2149}
2150
2151#[cfg(test)]
2152fn encode_value_envelope(value: &Value, codec: &str) -> Result<Value> {
2153 Ok(serde_json::to_value(encode_payload(value, codec)?)?)
2154}
2155
2156fn decode_wire_value(value: &Value, fallback_codec: &str) -> Result<Value> {
2157 validate_payload_codec(fallback_codec)?;
2158
2159 if value.is_null() {
2160 return Ok(Value::Null);
2161 }
2162
2163 if let Some((codec, blob)) = payload_envelope_parts(value)? {
2164 return decode_blob(blob, codec);
2165 }
2166
2167 if let Some(blob) = value.as_str() {
2168 return decode_blob(blob, fallback_codec);
2169 }
2170
2171 Err(untagged_payload_value())
2172}
2173
2174fn encode_typed_envelope(value: &AvroValue, codec: &str) -> Result<Value> {
2175 let envelope = match codec {
2176 DEFAULT_CODEC => encode_avro_value(value)?,
2177 other => return Err(unsupported_payload_codec(other)),
2178 };
2179 Ok(serde_json::to_value(envelope)?)
2180}
2181
2182fn decode_wire_avro_value(value: &Value, fallback_codec: &str) -> Result<AvroValue> {
2183 validate_payload_codec(fallback_codec)?;
2184
2185 if value.is_null() {
2186 return Ok(AvroValue::Null);
2187 }
2188
2189 if let Some((codec, blob)) = payload_envelope_parts(value)? {
2190 validate_payload_codec(codec)?;
2191 return decode_avro_value_blob(blob);
2192 }
2193
2194 if let Some(blob) = value.as_str() {
2195 return match fallback_codec {
2196 DEFAULT_CODEC => decode_avro_value_blob(blob),
2197 other => Err(unsupported_payload_codec(other)),
2198 };
2199 }
2200
2201 Err(untagged_payload_value())
2202}
2203
2204fn normalize_avro_arguments(value: AvroValue) -> AvroValue {
2205 match value {
2206 AvroValue::Null => AvroValue::Array(Vec::new()),
2207 AvroValue::Array(_) => value,
2208 other => AvroValue::Array(vec![other]),
2209 }
2210}
2211
2212fn decode_blob(blob: &str, codec: &str) -> Result<Value> {
2213 match codec {
2214 DEFAULT_CODEC => decode_avro_value_blob(blob)?.into_json(),
2215 other => Err(unsupported_payload_codec(other)),
2216 }
2217}
2218
2219fn validate_payload_codec(codec: &str) -> Result<()> {
2220 match codec {
2221 DEFAULT_CODEC => Ok(()),
2222 MISSING_TASK_PAYLOAD_CODEC => {
2223 Err(invalid_task_payload_codec("task payload_codec is missing"))
2224 }
2225 NULL_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec("task payload_codec is null")),
2226 NON_STRING_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec(
2227 "task payload_codec must be a string",
2228 )),
2229 other => Err(unsupported_payload_codec(other)),
2230 }
2231}
2232
2233fn invalid_task_payload_codec(reason: &str) -> Error {
2234 Error::Codec(format!(
2235 "unsupported_payload_codec: {reason}; Durable Workflow 2.0 requires an explicit string payload_codec=\"avro\" before worker task execution"
2236 ))
2237}
2238
2239fn payload_envelope_parts(value: &Value) -> Result<Option<(&str, &str)>> {
2240 let Some(object) = value.as_object() else {
2241 return Ok(None);
2242 };
2243 if !object.contains_key("codec") && !object.contains_key("blob") {
2244 return Ok(None);
2245 }
2246
2247 let codec = object
2248 .get("codec")
2249 .and_then(Value::as_str)
2250 .ok_or_else(invalid_payload_envelope)?;
2251 validate_payload_codec(codec)?;
2252 let blob = object
2253 .get("blob")
2254 .and_then(Value::as_str)
2255 .ok_or_else(invalid_payload_envelope)?;
2256 Ok(Some((codec, blob)))
2257}
2258
2259fn invalid_payload_envelope() -> Error {
2260 Error::Codec(
2261 "invalid_payload_envelope: durable payloads must use an object with string codec=\"avro\" and blob fields"
2262 .to_string(),
2263 )
2264}
2265
2266fn validate_workflow_task_commands(commands: &[Value]) -> Result<()> {
2267 for command in commands {
2268 let Some(command) = command.as_object() else {
2269 continue;
2270 };
2271 let Some(command_type) = command.get("type").and_then(Value::as_str) else {
2272 continue;
2273 };
2274 let Some(payload_field) = workflow_command_payload_field(command_type) else {
2275 continue;
2276 };
2277
2278 if let Some(codec) = command.get("payload_codec") {
2279 let codec = codec.as_str().ok_or_else(invalid_payload_envelope)?;
2280 validate_payload_codec(codec)?;
2281 }
2282
2283 let payload = command
2284 .get(payload_field)
2285 .ok_or_else(invalid_payload_envelope)?;
2286 if runtime_payloads::Reference::parse(payload)?.is_none() {
2287 validate_outbound_payload_envelope(payload)?;
2288 }
2289 }
2290 Ok(())
2291}
2292
2293fn workflow_completion_protocol_version(commands: &[Value]) -> &'static str {
2294 if commands.iter().any(|command| {
2295 command.get("type").and_then(Value::as_str) == Some("open_condition_wait")
2296 && command
2297 .get("condition_wait_occurrence_id")
2298 .and_then(Value::as_str)
2299 .is_some_and(|occurrence_id| !occurrence_id.is_empty())
2300 }) {
2301 CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
2302 } else if commands.iter().any(|command| {
2303 command.get("type").and_then(Value::as_str) == Some("upsert_search_attributes")
2304 && command.get("attribute_types").is_some()
2305 }) {
2306 TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION
2307 } else if commands
2308 .iter()
2309 .any(|command| command.get("type").and_then(Value::as_str) == Some("upsert_memo"))
2310 {
2311 MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION
2312 } else if commands
2313 .iter()
2314 .any(|command| command.get("type").and_then(Value::as_str) == Some("open_condition_wait"))
2315 {
2316 CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION
2317 } else if commands.iter().any(|command| {
2318 command.get("type").and_then(Value::as_str) == Some("upsert_search_attributes")
2319 }) {
2320 SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION
2321 } else {
2322 WORKER_PROTOCOL_VERSION
2323 }
2324}
2325
2326fn workflow_completion_protocol_version_with_message_streams(
2327 commands: &[Value],
2328 has_message_stream_metadata: bool,
2329) -> &'static str {
2330 let command_protocol = workflow_completion_protocol_version(commands);
2331 if has_message_stream_metadata && !worker_protocol_supports_message_streams(command_protocol) {
2332 MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION
2333 } else {
2334 command_protocol
2335 }
2336}
2337
2338fn workflow_command_payload_field(command_type: &str) -> Option<&'static str> {
2339 match command_type {
2340 "complete_workflow" | "complete_update" | "record_side_effect" => Some("result"),
2341 "schedule_activity" | "start_child_workflow" | "continue_as_new" => Some("arguments"),
2342 "start_service_operation" => Some("request_payload"),
2343 "upsert_memo" => Some("entries"),
2344 _ => None,
2345 }
2346}
2347
2348fn validate_outbound_payload_envelope(value: &Value) -> Result<()> {
2349 let Some((codec, blob)) = payload_envelope_parts(value)? else {
2350 return Err(untagged_payload_value());
2351 };
2352 validate_payload_codec(codec)?;
2353 decode_avro_value_blob(blob)?;
2354 Ok(())
2355}
2356
2357fn unsupported_payload_codec(codec: &str) -> Error {
2358 Error::Codec(format!(
2359 "unsupported_payload_codec: workflow payload codec {codec:?} is not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
2360 ))
2361}
2362
2363fn untagged_payload_value() -> Error {
2364 Error::Codec(
2365 "unsupported_payload_codec: untagged durable payload values are not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
2366 .to_string(),
2367 )
2368}
2369
2370fn decode_avro_value_blob(blob: &str) -> Result<AvroValue> {
2371 let bytes = BASE64.decode(blob).map_err(|err| {
2372 Error::Codec(format!(
2373 "invalid_payload_framing: expected strict base64 Avro single-object bytes: {err}"
2374 ))
2375 })?;
2376
2377 if serde_json::from_slice::<Value>(&bytes).is_ok() {
2378 return Err(unsupported_payload_codec("json"));
2379 }
2380
2381 if bytes.len() < 10 || bytes[..2] != AVRO_SINGLE_OBJECT_MAGIC {
2382 return Err(Error::Codec(
2383 "invalid_payload_framing: expected Avro single-object magic c301".to_string(),
2384 ));
2385 }
2386
2387 let fingerprint: [u8; 8] = bytes[2..10]
2388 .try_into()
2389 .map_err(|_| Error::Codec("invalid Avro fingerprint length".to_string()))?;
2390 if fingerprint != AVRO_VALUE_SCHEMA_FINGERPRINT {
2391 return Err(Error::Codec(format!(
2392 "unsupported_payload_schema: unknown CRC-64-AVRO fingerprint {}",
2393 fingerprint
2394 .iter()
2395 .map(|byte| format!("{byte:02x}"))
2396 .collect::<String>()
2397 )));
2398 }
2399
2400 let mut datum_reader = StrictAvroDatumReader::new(&bytes[10..]);
2401 let datum = from_avro_datum(avro_value_schema()?, &mut datum_reader, None);
2405 if datum_reader.truncated {
2406 return Err(Error::Codec(
2407 "invalid_payload_framing: truncated Avro Value datum".to_string(),
2408 ));
2409 }
2410 let datum = datum.map_err(|err| {
2411 Error::Codec(format!(
2412 "invalid_payload_framing: malformed Avro Value datum: {err}"
2413 ))
2414 })?;
2415 if datum_reader.remaining() != 0 {
2416 return Err(Error::Codec(format!(
2417 "invalid_payload_framing: {} trailing bytes after Avro Value datum",
2418 datum_reader.remaining()
2419 )));
2420 }
2421 avro_value_from_datum(datum)
2422}
2423
2424struct StrictAvroDatumReader<'a> {
2425 bytes: &'a [u8],
2426 offset: usize,
2427 truncated: bool,
2428}
2429
2430impl<'a> StrictAvroDatumReader<'a> {
2431 fn new(bytes: &'a [u8]) -> Self {
2432 Self {
2433 bytes,
2434 offset: 0,
2435 truncated: false,
2436 }
2437 }
2438
2439 fn remaining(&self) -> usize {
2440 self.bytes.len() - self.offset
2441 }
2442}
2443
2444impl Read for StrictAvroDatumReader<'_> {
2445 fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
2446 let count = buffer.len().min(self.remaining());
2447 buffer[..count].copy_from_slice(&self.bytes[self.offset..self.offset + count]);
2448 self.offset += count;
2449 if count < buffer.len() {
2450 self.truncated = true;
2451 }
2452
2453 Ok(count)
2454 }
2455}
2456
2457fn avro_value_to_datum(value: &AvroValue) -> Result<AvroDatum> {
2458 let branch = match value {
2459 AvroValue::Null => AvroDatum::Union(0, Box::new(AvroDatum::Null)),
2460 AvroValue::Boolean(value) => AvroDatum::Union(
2461 1,
2462 Box::new(AvroDatum::Record(vec![(
2463 "boolean".to_string(),
2464 AvroDatum::Boolean(*value),
2465 )])),
2466 ),
2467 AvroValue::Long(value) => AvroDatum::Union(
2468 2,
2469 Box::new(AvroDatum::Record(vec![(
2470 "long".to_string(),
2471 AvroDatum::Long(*value),
2472 )])),
2473 ),
2474 AvroValue::Double(value) => {
2475 if !value.is_finite() {
2476 return Err(Error::Codec(
2477 "non_finite_float: Avro Value doubles must be finite".to_string(),
2478 ));
2479 }
2480 AvroDatum::Union(
2481 3,
2482 Box::new(AvroDatum::Record(vec![(
2483 "double".to_string(),
2484 AvroDatum::Double(*value),
2485 )])),
2486 )
2487 }
2488 AvroValue::Bytes(value) => AvroDatum::Union(
2489 4,
2490 Box::new(AvroDatum::Record(vec![(
2491 "bytes".to_string(),
2492 AvroDatum::Bytes(value.clone()),
2493 )])),
2494 ),
2495 AvroValue::String(value) => AvroDatum::Union(
2496 5,
2497 Box::new(AvroDatum::Record(vec![(
2498 "string".to_string(),
2499 AvroDatum::String(value.clone()),
2500 )])),
2501 ),
2502 AvroValue::Array(values) => AvroDatum::Union(
2503 6,
2504 Box::new(AvroDatum::Record(vec![(
2505 "items".to_string(),
2506 AvroDatum::Array(
2507 values
2508 .iter()
2509 .map(avro_value_to_datum)
2510 .collect::<Result<Vec<_>>>()?,
2511 ),
2512 )])),
2513 ),
2514 AvroValue::Map(values) => AvroDatum::Union(
2515 7,
2516 Box::new(AvroDatum::Record(vec![(
2517 "entries".to_string(),
2518 AvroDatum::Array(
2519 values
2520 .iter()
2521 .map(|(key, value)| {
2522 Ok(AvroDatum::Record(vec![
2523 ("key".to_string(), AvroDatum::String(key.clone())),
2524 ("value".to_string(), avro_value_to_datum(value)?),
2525 ]))
2526 })
2527 .collect::<Result<Vec<_>>>()?,
2528 ),
2529 )])),
2530 ),
2531 };
2532 Ok(AvroDatum::Record(vec![("value".to_string(), branch)]))
2533}
2534
2535fn avro_value_from_datum(datum: AvroDatum) -> Result<AvroValue> {
2536 let AvroDatum::Record(mut outer) = datum else {
2537 return Err(Error::Codec(
2538 "invalid_payload_framing: datum is not a Value record".to_string(),
2539 ));
2540 };
2541 let (_, branch) = outer
2542 .pop()
2543 .filter(|(name, _)| name == "value")
2544 .ok_or_else(|| Error::Codec("invalid_payload_framing: Value field missing".to_string()))?;
2545 let AvroDatum::Union(_, branch) = branch else {
2546 return Err(Error::Codec(
2547 "invalid_payload_framing: invalid Value union".to_string(),
2548 ));
2549 };
2550 match *branch {
2551 AvroDatum::Null => Ok(AvroValue::Null),
2552 AvroDatum::Record(mut fields) => {
2553 let (name, value) = fields.pop().ok_or_else(|| {
2554 Error::Codec("invalid_payload_framing: empty Value branch".to_string())
2555 })?;
2556 match (name.as_str(), value) {
2557 ("boolean", AvroDatum::Boolean(value)) => Ok(AvroValue::Boolean(value)),
2558 ("long", AvroDatum::Long(value)) => Ok(AvroValue::Long(value)),
2559 ("double", AvroDatum::Double(value)) if value.is_finite() => {
2560 Ok(AvroValue::Double(value))
2561 }
2562 ("bytes", AvroDatum::Bytes(value)) => Ok(AvroValue::Bytes(value)),
2563 ("string", AvroDatum::String(value)) => Ok(AvroValue::String(value)),
2564 ("items", AvroDatum::Array(values)) => values
2565 .into_iter()
2566 .map(avro_value_from_datum)
2567 .collect::<Result<Vec<_>>>()
2568 .map(AvroValue::Array),
2569 ("entries", AvroDatum::Map(values)) => values
2570 .into_iter()
2571 .map(|(key, value)| Ok((key, avro_value_from_datum(value)?)))
2572 .collect::<Result<BTreeMap<_, _>>>()
2573 .map(AvroValue::Map),
2574 _ => Err(Error::Codec(
2575 "invalid_payload_framing: unknown Value branch".to_string(),
2576 )),
2577 }
2578 }
2579 _ => Err(Error::Codec(
2580 "invalid_payload_framing: invalid Value branch".to_string(),
2581 )),
2582 }
2583}
2584
2585fn avro_value_schema() -> Result<&'static Schema> {
2586 match AVRO_VALUE_SCHEMA.get_or_init(|| {
2587 Schema::parse_str(AVRO_VALUE_SCHEMA_JSON)
2588 .map_err(|err| format!("could not parse Avro Value schema: {err}"))
2589 }) {
2590 Ok(schema) => Ok(schema),
2591 Err(message) => Err(Error::Codec(message.clone())),
2592 }
2593}
2594
2595fn avro_value_ordered_map_encoding_schema() -> Result<&'static Schema> {
2596 match AVRO_VALUE_ORDERED_MAP_ENCODING_SCHEMA.get_or_init(|| {
2597 let mut schema: Value = serde_json::from_str(AVRO_VALUE_SCHEMA_JSON)
2602 .map_err(|err| format!("could not read packaged Avro Value schema: {err}"))?;
2603 let entries_schema = schema
2604 .pointer_mut("/fields/0/type/7/fields/0/type")
2605 .ok_or_else(|| "packaged Avro Value map schema is missing".to_string())?;
2606 if *entries_schema != json!({"type": "map", "values": "Value"}) {
2607 return Err("packaged Avro Value map schema changed unexpectedly".to_string());
2608 }
2609 *entries_schema = json!({
2610 "type": "array",
2611 "items": {
2612 "type": "record",
2613 "name": "MapEntry",
2614 "fields": [
2615 {"name": "key", "type": "string"},
2616 {"name": "value", "type": "Value"}
2617 ]
2618 }
2619 });
2620 Schema::parse_str(&schema.to_string())
2621 .map_err(|err| format!("could not parse ordered-map Avro Value schema: {err}"))
2622 }) {
2623 Ok(schema) => Ok(schema),
2624 Err(message) => Err(Error::Codec(message.clone())),
2625 }
2626}
2627
2628#[derive(Clone, Debug)]
2629pub struct Client {
2630 http: reqwest::Client,
2631 base_url: String,
2632 token: Option<String>,
2633 control_token: Option<String>,
2634 worker_token: Option<String>,
2635 namespace: String,
2636 max_external_payload_bytes: usize,
2637 worker_storage_admission: Option<WorkerStorageAdmission>,
2638 runtime_upload_policy: Arc<Mutex<runtime_uploads::PolicyCache>>,
2639}
2640
2641impl Client {
2642 pub fn new(base_url: impl Into<String>) -> Result<Self> {
2643 Self::builder(base_url).build()
2644 }
2645
2646 pub fn builder(base_url: impl Into<String>) -> ClientBuilder {
2647 ClientBuilder {
2648 base_url: base_url.into(),
2649 token: None,
2650 control_token: None,
2651 worker_token: None,
2652 namespace: "default".to_string(),
2653 timeout: Duration::from_secs(60),
2654 max_external_payload_bytes: 64 * 1024 * 1024,
2655 }
2656 }
2657
2658 pub async fn health(&self) -> Result<Value> {
2659 self.request_json(
2660 reqwest::Method::GET,
2661 "/health",
2662 RequestProtocol::ControlPlane,
2663 Option::<&Value>::None,
2664 )
2665 .await
2666 }
2667
2668 pub async fn cluster_info(&self) -> Result<Value> {
2669 self.request_json(
2670 reqwest::Method::GET,
2671 "/cluster/info",
2672 RequestProtocol::ControlPlane,
2673 Option::<&Value>::None,
2674 )
2675 .await
2676 }
2677
2678 pub async fn start_workflow<T: Serialize>(
2679 &self,
2680 workflow_type: &str,
2681 task_queue: &str,
2682 workflow_id: &str,
2683 input: T,
2684 ) -> Result<WorkflowHandle> {
2685 self.start_workflow_with_options(
2686 workflow_type,
2687 task_queue,
2688 workflow_id,
2689 WorkflowStartOptions::default(),
2690 input,
2691 )
2692 .await
2693 }
2694
2695 pub async fn start_workflow_with_options<T: Serialize>(
2698 &self,
2699 workflow_type: &str,
2700 task_queue: &str,
2701 workflow_id: &str,
2702 options: WorkflowStartOptions,
2703 input: T,
2704 ) -> Result<WorkflowHandle> {
2705 options.validate()?;
2706 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2707 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2708 let body = json!({
2709 "workflow_id": workflow_id,
2710 "workflow_type": workflow_type,
2711 "task_queue": task_queue,
2712 "input": input_envelope,
2713 "execution_timeout_seconds": options.execution_timeout_seconds,
2714 "run_timeout_seconds": options.run_timeout_seconds
2715 });
2716
2717 let data: Value = self
2718 .request_json(
2719 reqwest::Method::POST,
2720 "/workflows",
2721 RequestProtocol::ControlPlane,
2722 Some(&body),
2723 )
2724 .await?;
2725
2726 Ok(WorkflowHandle {
2727 client: self.clone(),
2728 workflow_id: data
2729 .get("workflow_id")
2730 .and_then(Value::as_str)
2731 .unwrap_or(workflow_id)
2732 .to_string(),
2733 run_id: data
2734 .get("run_id")
2735 .and_then(Value::as_str)
2736 .map(str::to_string),
2737 workflow_type: data
2738 .get("workflow_type")
2739 .and_then(Value::as_str)
2740 .unwrap_or(workflow_type)
2741 .to_string(),
2742 })
2743 }
2744
2745 pub async fn signal_workflow<T: Serialize>(
2746 &self,
2747 workflow_id: &str,
2748 signal_name: &str,
2749 input: T,
2750 ) -> Result<Value> {
2751 self.signal_workflow_target(workflow_id, None, signal_name, input)
2752 .await
2753 }
2754
2755 pub async fn append_message_stream<T: Serialize>(
2757 &self,
2758 workflow_id: &str,
2759 stream_name: &str,
2760 message_id: &str,
2761 input: T,
2762 ) -> Result<Value> {
2763 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2764 let body = json!({
2765 "message_id": message_id,
2766 "input": encode_typed_envelope(&input, DEFAULT_CODEC)?
2767 });
2768 self.request_json(
2769 reqwest::Method::POST,
2770 &format!("/workflows/{workflow_id}/message-streams/{stream_name}/messages"),
2771 RequestProtocol::ControlPlane,
2772 Some(&body),
2773 )
2774 .await
2775 }
2776
2777 pub async fn signal_workflow_run<T: Serialize>(
2779 &self,
2780 workflow_id: &str,
2781 run_id: &str,
2782 signal_name: &str,
2783 input: T,
2784 ) -> Result<Value> {
2785 self.signal_workflow_target(workflow_id, Some(run_id), signal_name, input)
2786 .await
2787 }
2788
2789 async fn signal_workflow_target<T: Serialize>(
2790 &self,
2791 workflow_id: &str,
2792 run_id: Option<&str>,
2793 signal_name: &str,
2794 input: T,
2795 ) -> Result<Value> {
2796 validate_user_signal_name(signal_name)?;
2797 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2798 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2799 let body = json!({
2800 "input": input_envelope
2801 });
2802 let path = match run_id {
2803 Some(run_id) => {
2804 format!("/workflows/{workflow_id}/runs/{run_id}/signal/{signal_name}")
2805 }
2806 None => format!("/workflows/{workflow_id}/signal/{signal_name}"),
2807 };
2808 self.request_json(
2809 reqwest::Method::POST,
2810 &path,
2811 RequestProtocol::ControlPlane,
2812 Some(&body),
2813 )
2814 .await
2815 }
2816
2817 pub async fn cancel_workflow(
2819 &self,
2820 workflow_id: &str,
2821 options: WorkflowCommandOptions,
2822 ) -> Result<WorkflowCommandResult> {
2823 self.workflow_command(workflow_id, None, WorkflowCommandKind::Cancel, options)
2824 .await
2825 }
2826
2827 pub async fn cancel_workflow_run(
2829 &self,
2830 workflow_id: &str,
2831 run_id: &str,
2832 options: WorkflowCommandOptions,
2833 ) -> Result<WorkflowCommandResult> {
2834 self.workflow_command(
2835 workflow_id,
2836 Some(run_id),
2837 WorkflowCommandKind::Cancel,
2838 options,
2839 )
2840 .await
2841 }
2842
2843 pub async fn terminate_workflow(
2845 &self,
2846 workflow_id: &str,
2847 options: WorkflowCommandOptions,
2848 ) -> Result<WorkflowCommandResult> {
2849 self.workflow_command(workflow_id, None, WorkflowCommandKind::Terminate, options)
2850 .await
2851 }
2852
2853 pub async fn terminate_workflow_run(
2855 &self,
2856 workflow_id: &str,
2857 run_id: &str,
2858 options: WorkflowCommandOptions,
2859 ) -> Result<WorkflowCommandResult> {
2860 self.workflow_command(
2861 workflow_id,
2862 Some(run_id),
2863 WorkflowCommandKind::Terminate,
2864 options,
2865 )
2866 .await
2867 }
2868
2869 pub async fn redrive_workflow_run(
2871 &self,
2872 workflow_id: &str,
2873 failed_run_id: &str,
2874 request_id: Option<&str>,
2875 ) -> Result<WorkflowRedriveResult> {
2876 let body = request_id
2877 .map(|id| json!({"request_id": id}))
2878 .unwrap_or_else(|| json!({}));
2879 let data: Value = self
2880 .request_json(
2881 reqwest::Method::POST,
2882 &format!("/workflows/{workflow_id}/runs/{failed_run_id}/redrive"),
2883 RequestProtocol::ControlPlane,
2884 Some(&body),
2885 )
2886 .await?;
2887 if data.get("command_status").and_then(Value::as_str) != Some("accepted")
2888 || data.get("outcome").and_then(Value::as_str) != Some("redriven")
2889 {
2890 return Err(Error::Codec(
2891 "redrive response was not accepted".to_string(),
2892 ));
2893 }
2894 if data.get("workflow_id").and_then(Value::as_str) != Some(workflow_id)
2895 || data.get("continued_from_run_id").and_then(Value::as_str) != Some(failed_run_id)
2896 {
2897 return Err(Error::Codec(
2898 "redrive response does not match the requested source run".to_string(),
2899 ));
2900 }
2901 let run_id = data
2902 .get("run_id")
2903 .and_then(Value::as_str)
2904 .filter(|id| !id.is_empty() && *id != failed_run_id)
2905 .ok_or_else(|| {
2906 Error::Codec("redrive response is missing a distinct successor run_id".to_string())
2907 })?;
2908 let resume_step_sequence = data
2909 .get("resume_step_sequence")
2910 .and_then(Value::as_u64)
2911 .filter(|sequence| *sequence > 0)
2912 .ok_or_else(|| {
2913 Error::Codec("redrive response is missing resume_step_sequence".to_string())
2914 })?;
2915 Ok(WorkflowRedriveResult {
2916 workflow_id: workflow_id.to_string(),
2917 source_run_id: failed_run_id.to_string(),
2918 run_id: run_id.to_string(),
2919 resume_step_sequence,
2920 raw: data,
2921 })
2922 }
2923
2924 async fn workflow_command(
2925 &self,
2926 workflow_id: &str,
2927 run_id: Option<&str>,
2928 command: WorkflowCommandKind,
2929 options: WorkflowCommandOptions,
2930 ) -> Result<WorkflowCommandResult> {
2931 let path = match run_id {
2932 Some(run_id) => format!(
2933 "/workflows/{workflow_id}/runs/{run_id}/{}",
2934 command.as_str()
2935 ),
2936 None => format!("/workflows/{workflow_id}/{}", command.as_str()),
2937 };
2938 let data = match self
2939 .request_json(
2940 reqwest::Method::POST,
2941 &path,
2942 RequestProtocol::ControlPlane,
2943 Some(&options),
2944 )
2945 .await
2946 {
2947 Ok(data) => data,
2948 Err(Error::Http { status, body }) => {
2949 return Err(Error::WorkflowCommandRejected(workflow_command_rejection(
2950 command,
2951 status,
2952 body,
2953 workflow_id,
2954 run_id,
2955 )));
2956 }
2957 Err(error) => return Err(error),
2958 };
2959
2960 Ok(workflow_command_result(command, data, workflow_id, run_id))
2961 }
2962
2963 pub async fn query_workflow<T: Serialize>(
2969 &self,
2970 workflow_id: &str,
2971 query_name: &str,
2972 input: T,
2973 ) -> Result<Value> {
2974 self.query_workflow_target(workflow_id, None, query_name, input)
2975 .await
2976 }
2977
2978 pub async fn query_workflow_run<T: Serialize>(
2980 &self,
2981 workflow_id: &str,
2982 run_id: &str,
2983 query_name: &str,
2984 input: T,
2985 ) -> Result<Value> {
2986 self.query_workflow_target(workflow_id, Some(run_id), query_name, input)
2987 .await
2988 }
2989
2990 pub async fn query_workflow_avro_value<T: Serialize>(
2992 &self,
2993 workflow_id: &str,
2994 query_name: &str,
2995 input: T,
2996 ) -> Result<AvroValue> {
2997 self.query_workflow_avro_value_target(workflow_id, None, query_name, input)
2998 .await
2999 }
3000
3001 pub async fn query_workflow_run_avro_value<T: Serialize>(
3003 &self,
3004 workflow_id: &str,
3005 run_id: &str,
3006 query_name: &str,
3007 input: T,
3008 ) -> Result<AvroValue> {
3009 self.query_workflow_avro_value_target(workflow_id, Some(run_id), query_name, input)
3010 .await
3011 }
3012
3013 async fn query_workflow_avro_value_target<T: Serialize>(
3014 &self,
3015 workflow_id: &str,
3016 run_id: Option<&str>,
3017 query_name: &str,
3018 input: T,
3019 ) -> Result<AvroValue> {
3020 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
3021 let body = json!({"input": encode_typed_envelope(&input, DEFAULT_CODEC)?});
3022 let path = match run_id {
3023 Some(run_id) => {
3024 format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
3025 }
3026 None => format!("/workflows/{workflow_id}/query/{query_name}"),
3027 };
3028 let response: Value = match self
3029 .request_json(
3030 reqwest::Method::POST,
3031 &path,
3032 RequestProtocol::ControlPlane,
3033 Some(&body),
3034 )
3035 .await
3036 {
3037 Ok(response) => response,
3038 Err(Error::Http { status, body }) => {
3039 return Err(Error::QueryFailed(query_failure(status, body)));
3040 }
3041 Err(error) => return Err(error),
3042 };
3043
3044 let envelope = response
3045 .get("result_envelope")
3046 .filter(|envelope| !envelope.is_null())
3047 .ok_or_else(|| {
3048 Error::Codec(
3049 "missing_payload_envelope: typed query result requires result_envelope"
3050 .to_string(),
3051 )
3052 })?;
3053 decode_wire_avro_value(envelope, DEFAULT_CODEC)
3054 }
3055
3056 async fn query_workflow_target<T: Serialize>(
3057 &self,
3058 workflow_id: &str,
3059 run_id: Option<&str>,
3060 query_name: &str,
3061 input: T,
3062 ) -> Result<Value> {
3063 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
3064 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
3065 let body = json!({
3066 "input": input_envelope
3067 });
3068 let path = match run_id {
3069 Some(run_id) => {
3070 format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
3071 }
3072 None => format!("/workflows/{workflow_id}/query/{query_name}"),
3073 };
3074 let response: Value = match self
3075 .request_json(
3076 reqwest::Method::POST,
3077 &path,
3078 RequestProtocol::ControlPlane,
3079 Some(&body),
3080 )
3081 .await
3082 {
3083 Ok(response) => response,
3084 Err(Error::Http { status, body }) => {
3085 return Err(Error::QueryFailed(query_failure(status, body)));
3086 }
3087 Err(error) => return Err(error),
3088 };
3089
3090 if let Some(envelope) = response
3091 .get("result_envelope")
3092 .filter(|envelope| !envelope.is_null())
3093 {
3094 return decode_wire_value(envelope, DEFAULT_CODEC);
3095 }
3096
3097 Ok(response.get("result").cloned().unwrap_or(Value::Null))
3098 }
3099
3100 pub async fn update_workflow<T: Serialize>(
3102 &self,
3103 workflow_id: &str,
3104 update_name: &str,
3105 input: T,
3106 request_id: Option<&str>,
3107 ) -> Result<Value> {
3108 let response = self
3109 .update_workflow_response(workflow_id, update_name, input, request_id)
3110 .await?;
3111 if let Some(envelope) = response
3112 .get("result_envelope")
3113 .filter(|envelope| !envelope.is_null())
3114 {
3115 return decode_wire_value(envelope, DEFAULT_CODEC);
3116 }
3117 Ok(response.get("result").cloned().unwrap_or(response))
3118 }
3119
3120 pub async fn update_workflow_avro_value<T: Serialize>(
3122 &self,
3123 workflow_id: &str,
3124 update_name: &str,
3125 input: T,
3126 request_id: Option<&str>,
3127 ) -> Result<AvroValue> {
3128 let response = self
3129 .update_workflow_response(workflow_id, update_name, input, request_id)
3130 .await?;
3131 let envelope = response
3132 .get("result_envelope")
3133 .filter(|envelope| !envelope.is_null())
3134 .ok_or_else(|| {
3135 Error::Codec(
3136 "missing_payload_envelope: typed update result requires result_envelope"
3137 .to_string(),
3138 )
3139 })?;
3140 decode_wire_avro_value(envelope, DEFAULT_CODEC)
3141 }
3142
3143 async fn update_workflow_response<T: Serialize>(
3144 &self,
3145 workflow_id: &str,
3146 update_name: &str,
3147 input: T,
3148 request_id: Option<&str>,
3149 ) -> Result<Value> {
3150 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
3151 let mut body = json!({
3152 "input": encode_typed_envelope(&input, DEFAULT_CODEC)?,
3153 "wait_for": "completed",
3154 });
3155 if let Some(request_id) = request_id {
3156 body["request_id"] = json!(request_id);
3157 }
3158 self.request_json(
3159 reqwest::Method::POST,
3160 &format!("/workflows/{workflow_id}/update/{update_name}"),
3161 RequestProtocol::ControlPlane,
3162 Some(&body),
3163 )
3164 .await
3165 }
3166
3167 pub async fn describe_workflow(&self, workflow_id: &str) -> Result<WorkflowDescription> {
3168 let path = format!("/workflows/{workflow_id}");
3169 let mut data: WorkflowDescription = self
3170 .request_json(
3171 reqwest::Method::GET,
3172 &path,
3173 RequestProtocol::ControlPlane,
3174 Option::<&Value>::None,
3175 )
3176 .await?;
3177 data.decode_payloads()?;
3178 Ok(data)
3179 }
3180
3181 pub async fn describe_workflow_run(
3183 &self,
3184 workflow_id: &str,
3185 run_id: &str,
3186 ) -> Result<WorkflowDescription> {
3187 let path = format!("/workflows/{workflow_id}/runs/{run_id}");
3188 let mut data: WorkflowDescription = self
3189 .request_json(
3190 reqwest::Method::GET,
3191 &path,
3192 RequestProtocol::ControlPlane,
3193 Option::<&Value>::None,
3194 )
3195 .await?;
3196 data.decode_payloads()?;
3197 Ok(data)
3198 }
3199
3200 fn workflow_stream_path(workflow_id: &str, run_id: &str, stream_name: Option<&str>) -> String {
3201 let mut path = format!(
3202 "/workflows/{}/runs/{}/streams",
3203 percent_encode_path_segment(workflow_id),
3204 percent_encode_path_segment(run_id),
3205 );
3206 if let Some(stream_name) = stream_name {
3207 path.push('/');
3208 path.push_str(&percent_encode_path_segment(stream_name));
3209 }
3210 path
3211 }
3212
3213 pub async fn list_workflow_streams(
3215 &self,
3216 workflow_id: &str,
3217 run_id: &str,
3218 ) -> Result<Vec<WorkflowStreamDescription>> {
3219 let response: WorkflowStreamListResponse = self
3220 .request_json(
3221 reqwest::Method::GET,
3222 &Self::workflow_stream_path(workflow_id, run_id, None),
3223 RequestProtocol::ControlPlane,
3224 Option::<&Value>::None,
3225 )
3226 .await?;
3227 Ok(response.streams)
3228 }
3229
3230 pub async fn describe_workflow_stream(
3232 &self,
3233 workflow_id: &str,
3234 run_id: &str,
3235 stream_name: &str,
3236 ) -> Result<WorkflowStreamDescription> {
3237 let response: WorkflowStreamDescriptionResponse = self
3238 .request_json(
3239 reqwest::Method::GET,
3240 &Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3241 RequestProtocol::ControlPlane,
3242 Option::<&Value>::None,
3243 )
3244 .await?;
3245 Ok(response.stream)
3246 }
3247
3248 pub async fn subscribe_workflow_stream(
3255 &self,
3256 workflow_id: &str,
3257 run_id: &str,
3258 stream_name: &str,
3259 from_offset: u64,
3260 max_items: usize,
3261 wait: Duration,
3262 ) -> Result<WorkflowStreamPage> {
3263 let max_items = max_items.clamp(1, 500);
3264 let wait_seconds = wait.as_secs().min(MAX_LONG_POLL_TIMEOUT_SECONDS);
3265 let path = format!(
3266 "{}/items?from={from_offset}&max_items={max_items}&wait_seconds={wait_seconds}",
3267 Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3268 );
3269 let response: WorkflowStreamPageResponse = self
3270 .request_json_with_timeout(
3271 reqwest::Method::GET,
3272 &path,
3273 RequestProtocol::ControlPlane,
3274 Option::<&Value>::None,
3275 Duration::from_secs(wait_seconds.saturating_add(5).max(5)),
3276 )
3277 .await?;
3278
3279 let items = response
3280 .items
3281 .into_iter()
3282 .map(|raw| {
3283 let offset = raw.get("offset").and_then(Value::as_u64).unwrap_or(0);
3284 let envelope = raw.get("payload").cloned();
3285 let payload = envelope
3286 .as_ref()
3287 .filter(|value| value.get("blob").is_some())
3288 .map(|value| decode_wire_avro_value(value, DEFAULT_CODEC))
3289 .transpose()?
3290 .map(AvroValue::into_json)
3291 .transpose()?;
3292 Ok(WorkflowStreamItem {
3293 offset,
3294 payload,
3295 payload_envelope: envelope,
3296 payload_reference: raw
3297 .get("payload_reference")
3298 .and_then(Value::as_str)
3299 .map(str::to_string),
3300 payload_codec: raw
3301 .get("payload_codec")
3302 .and_then(Value::as_str)
3303 .map(str::to_string),
3304 idempotency_key: raw
3305 .get("idempotency_key")
3306 .and_then(Value::as_str)
3307 .map(str::to_string),
3308 item_type: raw
3309 .get("item_type")
3310 .and_then(Value::as_str)
3311 .map(str::to_string),
3312 content_type: raw
3313 .get("content_type")
3314 .and_then(Value::as_str)
3315 .map(str::to_string),
3316 origin: raw
3317 .get("origin")
3318 .and_then(Value::as_str)
3319 .map(str::to_string),
3320 origin_reference: raw
3321 .get("origin_reference")
3322 .and_then(Value::as_str)
3323 .map(str::to_string),
3324 emitted_at: raw
3325 .get("emitted_at")
3326 .and_then(Value::as_str)
3327 .map(str::to_string),
3328 raw,
3329 })
3330 })
3331 .collect::<Result<Vec<_>>>()?;
3332 Ok(WorkflowStreamPage {
3333 stream: response.stream,
3334 items,
3335 next_offset: response.next_offset,
3336 terminal: response.terminal,
3337 })
3338 }
3339
3340 pub async fn append_workflow_stream(
3342 &self,
3343 workflow_id: &str,
3344 run_id: &str,
3345 stream_name: &str,
3346 items: &[WorkflowStreamAppendItem],
3347 max_pending_items: Option<u64>,
3348 ) -> Result<WorkflowStreamAppendResult> {
3349 if items.is_empty() {
3350 return Err(Error::Codec(
3351 "workflow_stream_items_empty: append requires at least one item".to_string(),
3352 ));
3353 }
3354 let mut body = json!({
3355 "items": items
3356 .iter()
3357 .map(|item| item.wire_value(None))
3358 .collect::<Vec<_>>(),
3359 });
3360 if let Some(max_pending_items) = max_pending_items {
3361 if max_pending_items == 0 {
3362 return Err(Error::Codec(
3363 "workflow_stream_pending_limit_invalid: max_pending_items must be positive"
3364 .to_string(),
3365 ));
3366 }
3367 body["max_pending_items"] = json!(max_pending_items);
3368 }
3369 let response: WorkflowStreamAppendResponse = self
3370 .request_json(
3371 reqwest::Method::POST,
3372 &format!(
3373 "{}/items",
3374 Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3375 ),
3376 RequestProtocol::ControlPlane,
3377 Some(&body),
3378 )
3379 .await?;
3380 Ok(WorkflowStreamAppendResult {
3381 stream: response.stream,
3382 accepted_offsets: response.accepted_offsets,
3383 accepted: response.accepted,
3384 deduped: response.deduped,
3385 })
3386 }
3387
3388 pub async fn close_workflow_stream(
3390 &self,
3391 workflow_id: &str,
3392 run_id: &str,
3393 stream_name: &str,
3394 error_reason: Option<&str>,
3395 retention_seconds: Option<u64>,
3396 ) -> Result<WorkflowStreamDescription> {
3397 let mut body = json!({});
3398 if let Some(error_reason) = error_reason {
3399 body["error_reason"] = json!(error_reason);
3400 }
3401 if let Some(retention_seconds) = retention_seconds {
3402 if retention_seconds == 0 {
3403 return Err(Error::Codec(
3404 "workflow_stream_retention_invalid: retention_seconds must be positive"
3405 .to_string(),
3406 ));
3407 }
3408 body["retention_seconds"] = json!(retention_seconds);
3409 }
3410 let response: WorkflowStreamDescriptionResponse = self
3411 .request_json(
3412 reqwest::Method::POST,
3413 &format!(
3414 "{}/close",
3415 Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3416 ),
3417 RequestProtocol::ControlPlane,
3418 Some(&body),
3419 )
3420 .await?;
3421 Ok(response.stream)
3422 }
3423
3424 pub async fn register_worker(
3425 &self,
3426 worker_id: &str,
3427 task_queue: &str,
3428 supported_workflow_types: Vec<String>,
3429 supported_activity_types: Vec<String>,
3430 max_concurrent_workflow_tasks: usize,
3431 max_concurrent_activity_tasks: usize,
3432 ) -> Result<RegisterWorkerResponse> {
3433 self.register_worker_with_capabilities(
3434 worker_id,
3435 task_queue,
3436 supported_workflow_types,
3437 supported_activity_types,
3438 max_concurrent_workflow_tasks,
3439 max_concurrent_activity_tasks,
3440 Vec::new(),
3441 )
3442 .await
3443 }
3444
3445 pub async fn register_worker_with_capabilities(
3447 &self,
3448 worker_id: &str,
3449 task_queue: &str,
3450 supported_workflow_types: Vec<String>,
3451 supported_activity_types: Vec<String>,
3452 max_concurrent_workflow_tasks: usize,
3453 max_concurrent_activity_tasks: usize,
3454 capabilities: Vec<String>,
3455 ) -> Result<RegisterWorkerResponse> {
3456 self.register_worker_with_command_contracts(
3457 worker_id,
3458 task_queue,
3459 supported_workflow_types,
3460 supported_activity_types,
3461 max_concurrent_workflow_tasks,
3462 max_concurrent_activity_tasks,
3463 capabilities,
3464 Value::Object(serde_json::Map::new()),
3465 )
3466 .await
3467 }
3468
3469 #[allow(clippy::too_many_arguments)]
3476 pub async fn register_worker_with_command_contracts(
3477 &self,
3478 worker_id: &str,
3479 task_queue: &str,
3480 supported_workflow_types: Vec<String>,
3481 supported_activity_types: Vec<String>,
3482 max_concurrent_workflow_tasks: usize,
3483 max_concurrent_activity_tasks: usize,
3484 capabilities: Vec<String>,
3485 workflow_command_contracts: Value,
3486 ) -> Result<RegisterWorkerResponse> {
3487 self.register_worker_with_definition_fingerprints(
3488 worker_id,
3489 task_queue,
3490 supported_workflow_types,
3491 supported_activity_types,
3492 max_concurrent_workflow_tasks,
3493 max_concurrent_activity_tasks,
3494 capabilities,
3495 workflow_command_contracts,
3496 None,
3497 )
3498 .await
3499 }
3500
3501 #[allow(clippy::too_many_arguments)]
3502 async fn register_worker_with_definition_fingerprints(
3503 &self,
3504 worker_id: &str,
3505 task_queue: &str,
3506 supported_workflow_types: Vec<String>,
3507 supported_activity_types: Vec<String>,
3508 max_concurrent_workflow_tasks: usize,
3509 max_concurrent_activity_tasks: usize,
3510 capabilities: Vec<String>,
3511 workflow_command_contracts: Value,
3512 workflow_definition_fingerprints: Option<BTreeMap<String, String>>,
3513 ) -> Result<RegisterWorkerResponse> {
3514 if let Some(contracts) = workflow_command_contracts.as_object() {
3515 for (workflow_type, contract) in contracts {
3516 let Some(update_validators) = contract.get("update_validators") else {
3517 continue;
3518 };
3519 if !update_validators
3520 .as_array()
3521 .is_some_and(|validators| validators.is_empty())
3522 {
3523 return Err(Error::UnsupportedUpdateValidators {
3524 workflow_type: workflow_type.clone(),
3525 });
3526 }
3527 }
3528 }
3529
3530 let mut body = json!({
3531 "worker_id": worker_id,
3532 "task_queue": task_queue,
3533 "runtime": "rust",
3534 "sdk_version": SDK_VERSION,
3535 "supported_workflow_types": supported_workflow_types,
3536 "supported_activity_types": supported_activity_types,
3537 "capabilities": capabilities,
3538 "capability_manifest": portable_worker_affinity_capability_manifest(),
3539 "max_concurrent_workflow_tasks": max_concurrent_workflow_tasks,
3540 "max_concurrent_activity_tasks": max_concurrent_activity_tasks
3541 });
3542 if workflow_command_contracts
3543 .as_object()
3544 .is_some_and(|contracts| !contracts.is_empty())
3545 {
3546 body["workflow_command_contracts"] = workflow_command_contracts;
3547 }
3548 if let Some(fingerprints) = workflow_definition_fingerprints {
3549 body["workflow_definition_fingerprints"] = json!(fingerprints);
3550 }
3551
3552 self.request_json(
3553 reqwest::Method::POST,
3554 "/worker/register",
3555 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3556 Some(&body),
3557 )
3558 .await
3559 }
3560
3561 pub async fn deregister_worker_registration(
3567 &self,
3568 worker_id: &str,
3569 ) -> Result<WorkerDeregistrationEnvelope> {
3570 let path = format!(
3571 "/worker/registrations/{}",
3572 percent_encode_path_segment(worker_id)
3573 );
3574 self.request_json(
3575 reqwest::Method::DELETE,
3576 &path,
3577 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3578 Option::<&Value>::None,
3579 )
3580 .await
3581 }
3582
3583 pub async fn poll_query_task(
3585 &self,
3586 worker_id: &str,
3587 task_queue: &str,
3588 timeout: Duration,
3589 ) -> Result<Option<QueryTask>> {
3590 Ok(self
3591 .poll_query_task_response(worker_id, task_queue, timeout)
3592 .await?
3593 .task)
3594 }
3595
3596 pub async fn poll_query_task_response(
3598 &self,
3599 worker_id: &str,
3600 task_queue: &str,
3601 timeout: Duration,
3602 ) -> Result<PollQueryTaskResponse> {
3603 let poll_request_id = unique_request_id("rust-query-poll");
3604 self.poll_query_task_response_with_request_id(
3605 worker_id,
3606 task_queue,
3607 timeout,
3608 &poll_request_id,
3609 1,
3610 )
3611 .await
3612 }
3613
3614 async fn poll_query_task_response_with_request_id(
3615 &self,
3616 worker_id: &str,
3617 task_queue: &str,
3618 timeout: Duration,
3619 poll_request_id: &str,
3620 transport_retries: usize,
3621 ) -> Result<PollQueryTaskResponse> {
3622 let timeout_seconds = long_poll_timeout_seconds(timeout);
3623 let body = json!({
3624 "worker_id": worker_id,
3625 "task_queue": task_queue,
3626 "poll_request_id": poll_request_id,
3627 "timeout_seconds": timeout_seconds,
3628 });
3629 self.poll_request_json(
3630 "/worker/query-tasks/poll",
3631 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3632 &body,
3633 timeout + Duration::from_secs(5),
3634 transport_retries,
3635 )
3636 .await
3637 }
3638
3639 pub async fn complete_query_task<T: Serialize>(
3641 &self,
3642 query_task_id: &str,
3643 lease_owner: &str,
3644 query_task_attempt: u64,
3645 result: T,
3646 codec: &str,
3647 ) -> Result<Value> {
3648 let typed_result = AvroValue::from_serialize(&result)?;
3649 let result_envelope = encode_typed_envelope(&typed_result, codec)?;
3650 self.complete_query_task_with_envelope(
3651 query_task_id,
3652 lease_owner,
3653 query_task_attempt,
3654 typed_result.into_json()?,
3655 result_envelope,
3656 )
3657 .await
3658 }
3659
3660 async fn complete_query_task_with_envelope(
3661 &self,
3662 query_task_id: &str,
3663 lease_owner: &str,
3664 query_task_attempt: u64,
3665 result: Value,
3666 result_envelope: Value,
3667 ) -> Result<Value> {
3668 let body = json!({
3669 "lease_owner": lease_owner,
3670 "query_task_attempt": query_task_attempt,
3671 "result": result,
3672 "result_envelope": result_envelope,
3673 });
3674 let path = format!("/worker/query-tasks/{query_task_id}/complete");
3675 let response = self
3676 .request_json(
3677 reqwest::Method::POST,
3678 &path,
3679 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3680 Some(&body),
3681 )
3682 .await;
3683 query_task_response(response)
3684 }
3685
3686 pub async fn fail_query_task(
3688 &self,
3689 query_task_id: &str,
3690 lease_owner: &str,
3691 query_task_attempt: u64,
3692 message: impl Into<String>,
3693 reason: impl Into<String>,
3694 failure_type: impl Into<String>,
3695 ) -> Result<Value> {
3696 let body = json!({
3697 "lease_owner": lease_owner,
3698 "query_task_attempt": query_task_attempt,
3699 "failure": {
3700 "message": message.into(),
3701 "reason": reason.into(),
3702 "type": failure_type.into(),
3703 }
3704 });
3705 let path = format!("/worker/query-tasks/{query_task_id}/fail");
3706 let response = self
3707 .request_json(
3708 reqwest::Method::POST,
3709 &path,
3710 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3711 Some(&body),
3712 )
3713 .await;
3714 query_task_response(response)
3715 }
3716
3717 pub async fn heartbeat_worker(
3718 &self,
3719 worker_id: &str,
3720 workflow_available: usize,
3721 activity_available: usize,
3722 ) -> Result<Value> {
3723 let body = json!({
3724 "worker_id": worker_id,
3725 "task_slots": {
3726 "workflow_available": workflow_available,
3727 "activity_available": activity_available
3728 },
3729 "process_metrics": {
3730 "process_id": std::process::id(),
3731 "process_uptime_seconds": 0
3732 }
3733 });
3734
3735 self.request_json(
3736 reqwest::Method::POST,
3737 "/worker/heartbeat",
3738 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3739 Some(&body),
3740 )
3741 .await
3742 }
3743
3744 pub async fn poll_workflow_task(
3745 &self,
3746 worker_id: &str,
3747 task_queue: &str,
3748 timeout: Duration,
3749 ) -> Result<Option<WorkflowTask>> {
3750 Ok(self
3751 .poll_workflow_task_response(worker_id, task_queue, timeout)
3752 .await?
3753 .task)
3754 }
3755
3756 pub async fn poll_workflow_task_response(
3757 &self,
3758 worker_id: &str,
3759 task_queue: &str,
3760 timeout: Duration,
3761 ) -> Result<PollWorkflowTaskResponse> {
3762 let poll_request_id = unique_request_id("rust-workflow-poll");
3763 self.poll_workflow_task_response_with_request_id(
3764 worker_id,
3765 task_queue,
3766 timeout,
3767 &poll_request_id,
3768 1,
3769 )
3770 .await
3771 }
3772
3773 async fn poll_workflow_task_response_with_request_id(
3774 &self,
3775 worker_id: &str,
3776 task_queue: &str,
3777 timeout: Duration,
3778 poll_request_id: &str,
3779 transport_retries: usize,
3780 ) -> Result<PollWorkflowTaskResponse> {
3781 let body = json!({
3782 "worker_id": worker_id,
3783 "task_queue": task_queue,
3784 "poll_request_id": poll_request_id,
3785 "timeout_seconds": long_poll_timeout_seconds(timeout),
3786 });
3787 let mut data: PollWorkflowTaskResponse = self
3788 .poll_request_json(
3789 "/worker/workflow-tasks/poll",
3790 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3791 &body,
3792 timeout + Duration::from_secs(5),
3793 transport_retries,
3794 )
3795 .await?;
3796
3797 if let Some(task) = data.task.as_mut() {
3798 self.fetch_remaining_workflow_history(worker_id, task)
3799 .await?;
3800 }
3801
3802 Ok(data)
3803 }
3804
3805 async fn fetch_remaining_workflow_history(
3806 &self,
3807 worker_id: &str,
3808 task: &mut WorkflowTask,
3809 ) -> Result<()> {
3810 let mut next_token = task.next_history_page_token.clone();
3811
3812 while let Some(token) = next_token.take().filter(|token| !token.is_empty()) {
3813 let lease_owner = task
3814 .lease_owner
3815 .clone()
3816 .unwrap_or_else(|| worker_id.to_string());
3817 let page = self
3818 .workflow_task_history_page(
3819 &task.task_id,
3820 &lease_owner,
3821 task.workflow_task_attempt,
3822 &token,
3823 )
3824 .await?;
3825
3826 task.append_history_page(page);
3827
3828 if task.next_history_page_token.as_deref() == Some(token.as_str()) {
3829 return Err(Error::Codec(
3830 "workflow history pagination returned the same page token".to_string(),
3831 ));
3832 }
3833
3834 next_token = task.next_history_page_token.clone();
3835 }
3836
3837 Ok(())
3838 }
3839
3840 async fn workflow_task_history_page(
3841 &self,
3842 task_id: &str,
3843 lease_owner: &str,
3844 workflow_task_attempt: u64,
3845 next_history_page_token: &str,
3846 ) -> Result<WorkflowTaskHistoryPage> {
3847 let body = json!({
3848 "lease_owner": lease_owner,
3849 "workflow_task_attempt": workflow_task_attempt,
3850 "next_history_page_token": next_history_page_token
3851 });
3852 let path = format!("/worker/workflow-tasks/{task_id}/history");
3853
3854 self.request_json(
3855 reqwest::Method::POST,
3856 &path,
3857 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3858 Some(&body),
3859 )
3860 .await
3861 }
3862
3863 pub async fn complete_workflow_task(
3864 &self,
3865 task_id: &str,
3866 lease_owner: &str,
3867 workflow_task_attempt: u64,
3868 commands: Vec<Value>,
3869 ) -> Result<Value> {
3870 self.complete_workflow_task_with_message_streams(
3871 task_id,
3872 lease_owner,
3873 workflow_task_attempt,
3874 commands,
3875 Vec::new(),
3876 Vec::new(),
3877 )
3878 .await
3879 }
3880
3881 async fn complete_workflow_task_with_message_streams(
3882 &self,
3883 task_id: &str,
3884 lease_owner: &str,
3885 workflow_task_attempt: u64,
3886 commands: Vec<Value>,
3887 message_stream_cursors: Vec<Value>,
3888 message_stream_waits: Vec<Value>,
3889 ) -> Result<Value> {
3890 validate_workflow_task_commands(&commands)?;
3891 let has_message_stream_metadata =
3892 !message_stream_cursors.is_empty() || !message_stream_waits.is_empty();
3893 if has_message_stream_metadata
3894 && !worker_protocol_supports_message_streams(WORKER_PROTOCOL_VERSION)
3895 {
3896 return Err(Error::Codec(
3897 "message_streams_unavailable: message stream completion metadata requires worker protocol 1.15 or newer"
3898 .to_string(),
3899 ));
3900 }
3901 let protocol_version = workflow_completion_protocol_version_with_message_streams(
3902 &commands,
3903 has_message_stream_metadata,
3904 );
3905 let mut body = json!({
3906 "lease_owner": lease_owner,
3907 "workflow_task_attempt": workflow_task_attempt,
3908 "commands": commands
3909 });
3910 if !message_stream_cursors.is_empty() {
3911 body["message_stream_cursors"] = Value::Array(message_stream_cursors);
3912 }
3913 if !message_stream_waits.is_empty() {
3914 body["message_stream_waits"] = Value::Array(message_stream_waits);
3915 }
3916 let path = format!("/worker/workflow-tasks/{task_id}/complete");
3917 self.request_json(
3918 reqwest::Method::POST,
3919 &path,
3920 RequestProtocol::Worker(protocol_version),
3921 Some(&body),
3922 )
3923 .await
3924 }
3925
3926 pub async fn fail_workflow_task(
3927 &self,
3928 task_id: &str,
3929 lease_owner: &str,
3930 workflow_task_attempt: u64,
3931 message: impl Into<String>,
3932 ) -> Result<Value> {
3933 self.fail_workflow_task_with_type(
3934 task_id,
3935 lease_owner,
3936 workflow_task_attempt,
3937 message,
3938 "RustWorkflowTaskFailure",
3939 )
3940 .await
3941 }
3942
3943 async fn fail_workflow_task_with_type(
3944 &self,
3945 task_id: &str,
3946 lease_owner: &str,
3947 workflow_task_attempt: u64,
3948 message: impl Into<String>,
3949 failure_type: &str,
3950 ) -> Result<Value> {
3951 let body = json!({
3952 "lease_owner": lease_owner,
3953 "workflow_task_attempt": workflow_task_attempt,
3954 "failure": {
3955 "message": message.into(),
3956 "type": failure_type
3957 }
3958 });
3959 let path = format!("/worker/workflow-tasks/{task_id}/fail");
3960 self.request_json(
3961 reqwest::Method::POST,
3962 &path,
3963 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3964 Some(&body),
3965 )
3966 .await
3967 }
3968
3969 pub async fn poll_activity_task(
3970 &self,
3971 worker_id: &str,
3972 task_queue: &str,
3973 timeout: Duration,
3974 ) -> Result<Option<ActivityTask>> {
3975 Ok(self
3976 .poll_activity_task_response(worker_id, task_queue, timeout)
3977 .await?
3978 .task)
3979 }
3980
3981 pub async fn poll_activity_task_response(
3983 &self,
3984 worker_id: &str,
3985 task_queue: &str,
3986 timeout: Duration,
3987 ) -> Result<PollActivityTaskResponse> {
3988 let poll_request_id = unique_request_id("rust-activity-poll");
3989 self.poll_activity_task_response_with_request_id(
3990 worker_id,
3991 task_queue,
3992 timeout,
3993 &poll_request_id,
3994 1,
3995 )
3996 .await
3997 }
3998
3999 async fn poll_activity_task_response_with_request_id(
4000 &self,
4001 worker_id: &str,
4002 task_queue: &str,
4003 timeout: Duration,
4004 poll_request_id: &str,
4005 transport_retries: usize,
4006 ) -> Result<PollActivityTaskResponse> {
4007 let body = json!({
4008 "worker_id": worker_id,
4009 "task_queue": task_queue,
4010 "poll_request_id": poll_request_id,
4011 "timeout_seconds": long_poll_timeout_seconds(timeout),
4012 });
4013 let data: PollActivityTaskResponse = self
4014 .poll_request_json(
4015 "/worker/activity-tasks/poll",
4016 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
4017 &body,
4018 timeout + Duration::from_secs(5),
4019 transport_retries,
4020 )
4021 .await?;
4022 Ok(data)
4023 }
4024
4025 pub async fn complete_activity_task<T: Serialize>(
4026 &self,
4027 task_id: &str,
4028 activity_attempt_id: &str,
4029 lease_owner: &str,
4030 result: T,
4031 codec: &str,
4032 ) -> Result<Value> {
4033 let result = encode_typed_envelope(&AvroValue::from_serialize(&result)?, codec)?;
4034 let body = json!({
4035 "activity_attempt_id": activity_attempt_id,
4036 "lease_owner": lease_owner,
4037 "result": result
4038 });
4039 let path = format!("/worker/activity-tasks/{task_id}/complete");
4040 activity_task_response(
4041 self.request_json(
4042 reqwest::Method::POST,
4043 &path,
4044 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
4045 Some(&body),
4046 )
4047 .await,
4048 "complete",
4049 task_id,
4050 activity_attempt_id,
4051 )
4052 }
4053
4054 pub async fn fail_activity_task(
4055 &self,
4056 task_id: &str,
4057 activity_attempt_id: &str,
4058 lease_owner: &str,
4059 message: impl Into<String>,
4060 non_retryable: bool,
4061 ) -> Result<Value> {
4062 let body = json!({
4063 "activity_attempt_id": activity_attempt_id,
4064 "lease_owner": lease_owner,
4065 "failure": {
4066 "message": message.into(),
4067 "type": "RustActivityFailure",
4068 "non_retryable": non_retryable
4069 }
4070 });
4071 let path = format!("/worker/activity-tasks/{task_id}/fail");
4072 activity_task_response(
4073 self.request_json(
4074 reqwest::Method::POST,
4075 &path,
4076 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
4077 Some(&body),
4078 )
4079 .await,
4080 "fail",
4081 task_id,
4082 activity_attempt_id,
4083 )
4084 }
4085
4086 pub async fn heartbeat_activity_task<T: Serialize>(
4087 &self,
4088 task_id: &str,
4089 activity_attempt_id: &str,
4090 lease_owner: &str,
4091 details: T,
4092 ) -> Result<ActivityHeartbeatResponse> {
4093 let details = encode_typed_envelope(&AvroValue::from_serialize(&details)?, DEFAULT_CODEC)?;
4094 let body = json!({
4095 "activity_attempt_id": activity_attempt_id,
4096 "lease_owner": lease_owner,
4097 "details": details
4098 });
4099 let path = format!("/worker/activity-tasks/{task_id}/heartbeat");
4100 activity_task_response(
4101 self.request_json(
4102 reqwest::Method::POST,
4103 &path,
4104 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
4105 Some(&body),
4106 )
4107 .await,
4108 "heartbeat",
4109 task_id,
4110 activity_attempt_id,
4111 )
4112 }
4113
4114 async fn request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
4115 &self,
4116 method: reqwest::Method,
4117 path: &str,
4118 protocol: RequestProtocol,
4119 body: Option<&B>,
4120 ) -> Result<T> {
4121 self.request_json_with_timeout(method, path, protocol, body, Duration::from_secs(60))
4122 .await
4123 }
4124
4125 async fn request_json_with_timeout<T: DeserializeOwned, B: Serialize + ?Sized>(
4126 &self,
4127 method: reqwest::Method,
4128 path: &str,
4129 protocol: RequestProtocol,
4130 body: Option<&B>,
4131 timeout: Duration,
4132 ) -> Result<T> {
4133 let auth_token = self.auth_token(protocol)?;
4134 let mut request = self
4135 .http
4136 .request(method.clone(), format!("{}/api{}", self.base_url, path))
4137 .timeout(timeout)
4138 .header(reqwest::header::ACCEPT, "application/json")
4139 .header(reqwest::header::CONTENT_TYPE, "application/json")
4140 .header("X-Namespace", &self.namespace);
4141
4142 match protocol {
4143 RequestProtocol::Worker(version) => {
4144 request = request.header("X-Durable-Workflow-Protocol-Version", version);
4145 }
4146 RequestProtocol::ControlPlane => {
4147 request = request.header(
4148 "X-Durable-Workflow-Control-Plane-Version",
4149 CONTROL_PLANE_VERSION,
4150 );
4151 }
4152 }
4153
4154 if let Some(token) = auth_token {
4155 request = request.bearer_auth(token);
4156 }
4157
4158 if let Some(body) = body {
4159 let mut body = serde_json::to_value(body)?;
4160 if matches!(
4161 method,
4162 reqwest::Method::POST | reqwest::Method::PUT | reqwest::Method::PATCH
4163 ) {
4164 self.externalize_runtime_payloads(&mut body, path, protocol)
4165 .await?;
4166 }
4167 request = request.json(&body);
4168 }
4169
4170 let request = request.build()?;
4171 let poll_request_id = path.ends_with("/poll").then(|| {
4172 request
4173 .body()
4174 .and_then(reqwest::Body::as_bytes)
4175 .and_then(|body| serde_json::from_slice::<Value>(body).ok())
4176 .and_then(|body| body.get("poll_request_id")?.as_str().map(str::to_owned))
4177 .unwrap_or_default()
4178 });
4179 let mut storage_retries = 0_usize;
4180
4181 loop {
4182 let response = self
4184 .http
4185 .execute(request.try_clone().ok_or_else(|| {
4186 Error::WorkerLoop("worker request body cannot be retried".to_string())
4187 })?)
4188 .await?;
4189 let status = response.status();
4190 let bytes = response.bytes().await?;
4191
4192 if !status.is_success() {
4193 let body = String::from_utf8_lossy(&bytes).to_string();
4194 if let Some(protocol) = protocol_failure(status, &body) {
4195 return Err(Error::Protocol(protocol));
4196 }
4197 let error = Error::Http { status, body };
4198 if self
4199 .wait_for_storage_admission(
4200 &error,
4201 protocol,
4202 Some(path),
4203 poll_request_id.as_deref(),
4204 &mut storage_retries,
4205 )
4206 .await
4207 {
4208 continue;
4209 }
4210 return Err(error);
4211 }
4212
4213 if bytes.is_empty() {
4214 return Ok(serde_json::from_value(Value::Null)?);
4215 }
4216
4217 let mut value: Value = serde_json::from_slice(&bytes)?;
4218 self.resolve_runtime_payloads(&mut value, path, protocol)
4219 .await?;
4220 return Ok(serde_json::from_value(value)?);
4221 }
4222 }
4223
4224 async fn wait_for_storage_admission(
4225 &self,
4226 error: &Error,
4227 protocol: RequestProtocol,
4228 path: Option<&str>,
4229 poll_request_id: Option<&str>,
4230 retries: &mut usize,
4231 ) -> bool {
4232 let Some(admission) = self
4233 .worker_storage_admission
4234 .as_ref()
4235 .filter(|_| matches!(protocol, RequestProtocol::Worker(_)))
4236 else {
4237 return false;
4238 };
4239 let Some(advertised_delay) = worker_storage_admission_retry_after(error, poll_request_id)
4240 .or_else(|| {
4241 path.and_then(|path| {
4242 worker_backend_unavailable_retry_after(error, path, poll_request_id)
4243 })
4244 })
4245 else {
4246 return false;
4247 };
4248 *retries = retries.saturating_add(1);
4249 let delay = worker_retry_delay(admission.policy, *retries)
4250 .max(advertised_delay)
4251 .min(admission.policy.max_backoff.max(Duration::from_millis(1)));
4252 let deadline = tokio::time::Instant::now() + delay;
4253 loop {
4254 if admission.stop.load(Ordering::SeqCst) {
4255 return false;
4256 }
4257 let remaining = deadline.saturating_duration_since(tokio::time::Instant::now());
4258 if remaining.is_zero() {
4259 return true;
4260 }
4261 tokio::time::sleep(remaining.min(Duration::from_millis(100))).await;
4262 }
4263 }
4264
4265 async fn poll_request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
4266 &self,
4267 path: &str,
4268 protocol: RequestProtocol,
4269 body: &B,
4270 timeout: Duration,
4271 max_retries: usize,
4272 ) -> Result<T> {
4273 let mut retries = 0;
4274
4275 loop {
4276 let response = self
4277 .request_json_with_timeout(
4278 reqwest::Method::POST,
4279 path,
4280 protocol,
4281 Some(body),
4282 timeout,
4283 )
4284 .await;
4285
4286 match response {
4287 Err(Error::Transport(_)) if retries < max_retries => retries += 1,
4288 response => return worker_poll_response(response),
4289 }
4290 }
4291 }
4292
4293 fn auth_token(&self, protocol: RequestProtocol) -> Result<Option<&str>> {
4294 match protocol {
4295 RequestProtocol::Worker(_) => {
4296 if let Some(token) = self.worker_token.as_deref().or(self.token.as_deref()) {
4297 return Ok(Some(token));
4298 }
4299 if self.control_token.is_some() {
4300 return Err(Error::MissingRoleCredentials {
4301 role: "worker",
4302 opposite_role: "control",
4303 });
4304 }
4305 Ok(None)
4306 }
4307 RequestProtocol::ControlPlane => {
4308 if let Some(token) = self.control_token.as_deref().or(self.token.as_deref()) {
4309 return Ok(Some(token));
4310 }
4311 if self.worker_token.is_some() {
4312 return Err(Error::MissingRoleCredentials {
4313 role: "control",
4314 opposite_role: "worker",
4315 });
4316 }
4317 Ok(None)
4318 }
4319 }
4320 }
4321}
4322
4323fn query_failure(status: reqwest::StatusCode, raw_body: String) -> QueryFailure {
4324 let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
4325 let reason = body
4326 .get("reason")
4327 .and_then(Value::as_str)
4328 .unwrap_or("query_rejected")
4329 .to_string();
4330 let message = body
4331 .get("message")
4332 .or_else(|| body.get("error"))
4333 .and_then(Value::as_str)
4334 .unwrap_or("workflow query was rejected")
4335 .to_string();
4336
4337 QueryFailure {
4338 status: status.as_u16(),
4339 reason,
4340 message,
4341 body,
4342 }
4343}
4344
4345fn workflow_command_result(
4346 command: WorkflowCommandKind,
4347 data: Value,
4348 workflow_id: &str,
4349 run_id: Option<&str>,
4350) -> WorkflowCommandResult {
4351 WorkflowCommandResult {
4352 command,
4353 workflow_id: data
4354 .get("workflow_id")
4355 .and_then(Value::as_str)
4356 .unwrap_or(workflow_id)
4357 .to_string(),
4358 run_id: data
4359 .get("run_id")
4360 .and_then(Value::as_str)
4361 .or(run_id)
4362 .map(str::to_string),
4363 outcome: data
4364 .get("outcome")
4365 .and_then(Value::as_str)
4366 .map(str::to_string),
4367 reason: data
4368 .get("reason")
4369 .and_then(Value::as_str)
4370 .map(str::to_string),
4371 command_status: data
4372 .get("command_status")
4373 .and_then(Value::as_str)
4374 .map(str::to_string),
4375 raw: data,
4376 }
4377}
4378
4379fn workflow_command_rejection(
4380 command: WorkflowCommandKind,
4381 status: reqwest::StatusCode,
4382 raw_body: String,
4383 workflow_id: &str,
4384 run_id: Option<&str>,
4385) -> WorkflowCommandRejection {
4386 let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
4387 WorkflowCommandRejection {
4388 command,
4389 status: status.as_u16(),
4390 reason: body
4391 .get("reason")
4392 .and_then(Value::as_str)
4393 .unwrap_or("workflow_command_rejected")
4394 .to_string(),
4395 message: body
4396 .get("message")
4397 .or_else(|| body.get("error"))
4398 .and_then(Value::as_str)
4399 .unwrap_or("workflow lifecycle command was rejected")
4400 .to_string(),
4401 workflow_id: body
4402 .get("workflow_id")
4403 .and_then(Value::as_str)
4404 .unwrap_or(workflow_id)
4405 .to_string(),
4406 run_id: body
4407 .get("run_id")
4408 .and_then(Value::as_str)
4409 .or(run_id)
4410 .map(str::to_string),
4411 target_scope: body
4412 .get("target_scope")
4413 .and_then(Value::as_str)
4414 .map(str::to_string),
4415 body,
4416 }
4417}
4418
4419fn query_task_response(response: Result<Value>) -> Result<Value> {
4420 match response {
4421 Err(Error::Http { status, body }) => Err(Error::QueryFailed(query_failure(status, body))),
4422 response => response,
4423 }
4424}
4425
4426fn worker_poll_response<T: DeserializeOwned>(response: Result<T>) -> Result<T> {
4427 match response {
4428 Err(Error::Http { status, body })
4429 if status == reqwest::StatusCode::CONFLICT && worker_poll_body_is_stop(&body) =>
4430 {
4431 Ok(serde_json::from_str(&body)?)
4432 }
4433 response => response,
4434 }
4435}
4436
4437fn worker_poll_body_is_stop(body: &str) -> bool {
4438 serde_json::from_str::<Value>(body)
4439 .ok()
4440 .is_some_and(|body| {
4441 worker_poll_is_stop(
4442 body.get("poll_status").and_then(Value::as_str),
4443 body.get("reason").and_then(Value::as_str),
4444 )
4445 })
4446}
4447
4448fn worker_poll_is_stop(poll_status: Option<&str>, reason: Option<&str>) -> bool {
4449 matches!(poll_status, Some("draining" | "stopped"))
4450 || matches!(reason, Some("worker_draining" | "worker_stopped"))
4451}
4452
4453fn query_task_rejection_is_final(error: &Error) -> bool {
4454 matches!(
4455 error,
4456 Error::QueryFailed(failure)
4457 if QUERY_TASK_FINAL_REJECTION_REASONS.contains(&failure.reason.as_str())
4458 )
4459}
4460
4461fn activity_task_response<T>(
4462 response: Result<T>,
4463 operation: &str,
4464 task_id: &str,
4465 activity_attempt_id: &str,
4466) -> Result<T> {
4467 match response {
4468 Err(Error::Http { status, body }) => {
4469 let body = serde_json::from_str(&body).unwrap_or_else(|_| json!({"message": body}));
4470 Err(Error::ActivityTaskRejected(ActivityTaskRejection {
4471 operation: operation.to_string(),
4472 status: status.as_u16(),
4473 reason: body
4474 .get("reason")
4475 .and_then(Value::as_str)
4476 .unwrap_or("activity_task_rejected")
4477 .to_string(),
4478 task_id: body
4479 .get("task_id")
4480 .and_then(Value::as_str)
4481 .unwrap_or(task_id)
4482 .to_string(),
4483 activity_attempt_id: body
4484 .get("activity_attempt_id")
4485 .and_then(Value::as_str)
4486 .unwrap_or(activity_attempt_id)
4487 .to_string(),
4488 cancel_requested: body
4489 .get("cancel_requested")
4490 .and_then(Value::as_bool)
4491 .unwrap_or(false),
4492 can_continue: body.get("can_continue").and_then(Value::as_bool),
4493 run_closed_reason: body
4494 .get("run_closed_reason")
4495 .and_then(Value::as_str)
4496 .map(str::to_string),
4497 body,
4498 }))
4499 }
4500 response => response,
4501 }
4502}
4503
4504fn activity_task_rejection_is_final(error: &Error) -> bool {
4505 matches!(
4506 error,
4507 Error::ActivityTaskRejected(rejection)
4508 if matches!(
4509 rejection.reason.as_str(),
4510 "run_cancelled"
4511 | "run_terminated"
4512 | "attempt_closed"
4513 | "stale_attempt"
4514 | "activity_cancelled"
4515 | "task_cancelled"
4516 | "run_closed"
4517 | "activity_not_running"
4518 | "attempt_not_found"
4519 )
4520 )
4521}
4522
4523fn workflow_task_completion_is_terminal_timeout(
4524 error: &Error,
4525 task_id: &str,
4526 workflow_task_attempt: u64,
4527 run_id: Option<&str>,
4528) -> bool {
4529 let Error::Http { status, body } = error else {
4530 return false;
4531 };
4532 if *status != reqwest::StatusCode::CONFLICT {
4533 return false;
4534 }
4535
4536 let Some(run_id) = run_id else {
4537 return false;
4538 };
4539 let Ok(body) = serde_json::from_str::<Value>(body) else {
4540 return false;
4541 };
4542
4543 body.get("recorded").and_then(Value::as_bool) == Some(false)
4544 && body.get("reason").and_then(Value::as_str) == Some("run_timed_out")
4545 && body.get("run_status").and_then(Value::as_str) == Some("failed")
4546 && body.get("run_id").and_then(Value::as_str) == Some(run_id)
4547 && body.get("task_id").and_then(Value::as_str) == Some(task_id)
4548 && body.get("workflow_task_attempt").and_then(Value::as_u64) == Some(workflow_task_attempt)
4549}
4550
4551fn protocol_failure(status: reqwest::StatusCode, raw_body: &str) -> Option<ProtocolFailure> {
4552 let body: Value = serde_json::from_str(raw_body).ok()?;
4553 let reason = body.get("reason")?.as_str()?;
4554 if !matches!(
4555 reason,
4556 "missing_protocol_version"
4557 | "unsupported_protocol_version"
4558 | "missing_control_plane_version"
4559 | "unsupported_control_plane_version"
4560 ) {
4561 return None;
4562 }
4563
4564 Some(ProtocolFailure {
4565 status: status.as_u16(),
4566 reason: reason.to_string(),
4567 message: body
4568 .get("message")
4569 .or_else(|| body.get("error"))
4570 .and_then(Value::as_str)
4571 .unwrap_or("protocol version rejected")
4572 .to_string(),
4573 supported_version: body
4574 .get("supported_version")
4575 .and_then(Value::as_str)
4576 .map(str::to_string),
4577 requested_version: body
4578 .get("requested_version")
4579 .and_then(Value::as_str)
4580 .map(str::to_string),
4581 body,
4582 })
4583}
4584
4585fn long_poll_timeout_seconds(timeout: Duration) -> u64 {
4586 timeout
4587 .as_secs()
4588 .saturating_add(u64::from(timeout.subsec_nanos() > 0))
4589 .min(MAX_LONG_POLL_TIMEOUT_SECONDS)
4590}
4591
4592fn worker_operation_is_retryable(error: &Error) -> bool {
4593 if worker_poll_capacity_retry_after(error).is_some()
4594 || worker_storage_admission_body(error).is_some()
4595 || worker_backend_unavailable_body(error).is_some()
4596 || worker_operation_is_explicitly_non_retryable(error)
4597 {
4598 return false;
4599 }
4600
4601 match error {
4602 Error::Transport(error) => {
4603 error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()
4604 }
4605 Error::Http { status, .. } => {
4606 matches!(
4607 *status,
4608 reqwest::StatusCode::REQUEST_TIMEOUT | reqwest::StatusCode::TOO_MANY_REQUESTS
4609 ) || status.is_server_error()
4610 }
4611 _ => false,
4612 }
4613}
4614
4615fn worker_storage_admission_body(error: &Error) -> Option<Value> {
4616 let body: Value = match error {
4617 Error::Http { body, .. } => serde_json::from_str(body).ok()?,
4618 Error::ActivityTaskRejected(rejection) => rejection.body.clone(),
4619 _ => return None,
4620 };
4621 matches!(
4622 body.get("reason").and_then(Value::as_str),
4623 Some("storage_pressure" | "storage_admission_unavailable")
4624 )
4625 .then_some(body)
4626}
4627
4628fn worker_storage_admission_retry_after(
4629 error: &Error,
4630 poll_request_id: Option<&str>,
4631) -> Option<Duration> {
4632 let Error::Http { status, .. } = error else {
4633 return None;
4634 };
4635 let body = worker_storage_admission_body(error)?;
4636 let delay = body.get("retry_after_seconds")?.as_u64()?;
4637 if *status != reqwest::StatusCode::SERVICE_UNAVAILABLE
4638 || delay == 0
4639 || body.get("retryable") != Some(&Value::Bool(true))
4640 || !matches!(body.get("storage_state")?.as_str()?, "draining" | "fenced")
4641 || (body["reason"] == "storage_admission_unavailable" && body["storage_state"] != "fenced")
4642 || body
4643 .get("request_admitted")
4644 .is_some_and(|admitted| admitted != &Value::Bool(false))
4645 {
4646 return None;
4647 }
4648 match poll_request_id {
4649 Some(id) => {
4650 if id.is_empty()
4651 || body.get("task") != Some(&Value::Null)
4652 || body.get("poll_request_id").and_then(Value::as_str) != Some(id)
4653 || body.get("poll_status") != body.get("reason")
4654 || body.get("retry_same_poll_request_id") != Some(&Value::Bool(true))
4655 || body.get("claim_admitted") != Some(&Value::Bool(false))
4656 {
4657 return None;
4658 }
4659 }
4660 None if body.get("request_admitted") != Some(&Value::Bool(false)) => return None,
4661 None => {}
4662 }
4663 Some(Duration::from_secs(delay))
4664}
4665
4666fn worker_backend_unavailable_retry_after(
4667 error: &Error,
4668 path: &str,
4669 poll_request_id: Option<&str>,
4670) -> Option<Duration> {
4671 let operation = match path {
4672 "/worker/register" => "register_worker",
4673 "/worker/heartbeat" => "heartbeat_worker",
4674 "/worker/workflow-tasks/poll" => "poll_workflow_task",
4675 "/worker/activity-tasks/poll" => "poll_activity_task",
4676 "/worker/query-tasks/poll" => "poll_query_task",
4677 "/worker/update-validation-tasks/poll" => "poll_update_validation_task",
4678 _ => return None,
4679 };
4680 let body = worker_backend_unavailable_body(error)?;
4681 let delay = body.get("retry_after_seconds")?.as_u64()?;
4682 if delay == 0
4683 || body.get("operation")?.as_str()? != operation
4684 || body.get("outcome")?.as_str()? != "unknown"
4685 || body.get("retryable") != Some(&Value::Bool(true))
4686 || body.get("worker_id")?.as_str()?.is_empty()
4687 {
4688 return None;
4689 }
4690 if let Some(id) = poll_request_id {
4691 if id.is_empty()
4692 || body.get("task") != Some(&Value::Null)
4693 || body.get("poll_status")?.as_str()? != "backend_unavailable"
4694 || body.get("poll_request_id")?.as_str()? != id
4695 || body.get("retry_same_poll_request_id") != Some(&Value::Bool(true))
4696 {
4697 return None;
4698 }
4699 } else if path.ends_with("/poll") {
4700 return None;
4701 }
4702 Some(Duration::from_secs(delay))
4703}
4704
4705fn worker_backend_unavailable_body(error: &Error) -> Option<Value> {
4706 let Error::Http { status, body } = error else {
4707 return None;
4708 };
4709 if *status != reqwest::StatusCode::SERVICE_UNAVAILABLE {
4710 return None;
4711 }
4712 let body: Value = serde_json::from_str(body).ok()?;
4713 (body.get("reason")?.as_str()? == "backend_unavailable").then_some(body)
4714}
4715
4716fn worker_operation_is_explicitly_non_retryable(error: &Error) -> bool {
4717 let Error::Http { body, .. } = error else {
4718 return false;
4719 };
4720
4721 serde_json::from_str::<Value>(body)
4722 .ok()
4723 .and_then(|body| body.get("retryable").and_then(Value::as_bool))
4724 == Some(false)
4725}
4726
4727fn worker_poll_capacity_retry_after(error: &Error) -> Option<Duration> {
4728 let Error::Http { status, body } = error else {
4729 return None;
4730 };
4731 if *status != reqwest::StatusCode::TOO_MANY_REQUESTS {
4732 return None;
4733 }
4734
4735 let body = serde_json::from_str::<Value>(body).ok()?;
4736 let capacity_exhausted = body.get("poll_status").and_then(Value::as_str)
4737 == Some("long_poll_capacity_exhausted")
4738 || body.get("reason").and_then(Value::as_str) == Some("long_poll_capacity_exhausted");
4739 if !capacity_exhausted || body.get("retryable").and_then(Value::as_bool) != Some(true) {
4740 return None;
4741 }
4742
4743 Some(Duration::from_secs(
4744 body.get("retry_after_seconds")
4745 .and_then(Value::as_u64)
4746 .unwrap_or_default(),
4747 ))
4748}
4749
4750fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration {
4751 let exponent = retry.saturating_sub(1).min(31) as u32;
4752 policy
4753 .initial_backoff
4754 .saturating_mul(1_u32 << exponent)
4755 .min(policy.max_backoff)
4756}
4757
4758#[derive(Debug)]
4759pub struct ClientBuilder {
4760 base_url: String,
4761 token: Option<String>,
4762 control_token: Option<String>,
4763 worker_token: Option<String>,
4764 namespace: String,
4765 timeout: Duration,
4766 max_external_payload_bytes: usize,
4767}
4768
4769impl ClientBuilder {
4770 pub fn token(mut self, token: Option<String>) -> Self {
4771 self.token = token;
4772 self
4773 }
4774
4775 pub fn control_token(mut self, token: Option<String>) -> Self {
4776 self.control_token = token;
4777 self
4778 }
4779
4780 pub fn worker_token(mut self, token: Option<String>) -> Self {
4781 self.worker_token = token;
4782 self
4783 }
4784
4785 pub fn namespace(mut self, namespace: impl Into<String>) -> Self {
4786 self.namespace = namespace.into();
4787 self
4788 }
4789
4790 pub fn timeout(mut self, timeout: Duration) -> Self {
4791 self.timeout = timeout;
4792 self
4793 }
4794
4795 pub fn max_external_payload_bytes(mut self, bytes: usize) -> Self {
4798 self.max_external_payload_bytes = bytes;
4799 self
4800 }
4801
4802 pub fn build(self) -> Result<Client> {
4803 let base_url = self.base_url.trim_end_matches('/').to_string();
4804 let has_sdk_api_suffix = reqwest::Url::parse(&base_url)
4805 .map(|url| url.path().trim_end_matches('/').ends_with("/api"))
4806 .unwrap_or_else(|_| base_url.ends_with("/api"));
4807
4808 if has_sdk_api_suffix {
4809 return Err(Error::InvalidBaseUrl);
4810 }
4811
4812 Ok(Client {
4813 http: reqwest::Client::builder()
4814 .timeout(self.timeout)
4815 .redirect(reqwest::redirect::Policy::none())
4816 .build()?,
4817 base_url,
4818 token: self.token,
4819 control_token: self.control_token,
4820 worker_token: self.worker_token,
4821 namespace: self.namespace,
4822 max_external_payload_bytes: self.max_external_payload_bytes,
4823 worker_storage_admission: None,
4824 runtime_upload_policy: Arc::new(Mutex::new([None, None])),
4825 })
4826 }
4827}
4828
4829#[derive(Clone, Debug)]
4830pub struct WorkflowHandle {
4831 client: Client,
4832 pub workflow_id: String,
4833 pub run_id: Option<String>,
4834 pub workflow_type: String,
4835}
4836
4837impl WorkflowHandle {
4838 pub async fn describe(&self) -> Result<WorkflowDescription> {
4840 self.client.describe_workflow(&self.workflow_id).await
4841 }
4842
4843 pub async fn describe_selected_run(&self) -> Result<WorkflowDescription> {
4845 let run_id = self.run_id.as_deref().ok_or_else(|| {
4846 Error::Codec("run_id is required for selected-run description".to_string())
4847 })?;
4848 self.client
4849 .describe_workflow_run(&self.workflow_id, run_id)
4850 .await
4851 }
4852
4853 pub async fn signal<T: Serialize>(&self, signal_name: &str, input: T) -> Result<Value> {
4854 self.client
4855 .signal_workflow(&self.workflow_id, signal_name, input)
4856 .await
4857 }
4858
4859 pub async fn append_message<T: Serialize>(
4860 &self,
4861 stream_name: &str,
4862 message_id: &str,
4863 input: T,
4864 ) -> Result<Value> {
4865 self.client
4866 .append_message_stream(&self.workflow_id, stream_name, message_id, input)
4867 .await
4868 }
4869
4870 pub async fn signal_selected_run<T: Serialize>(
4872 &self,
4873 signal_name: &str,
4874 input: T,
4875 ) -> Result<Value> {
4876 let run_id = self.run_id.as_deref().ok_or_else(|| {
4877 Error::Codec("run_id is required for selected-run signaling".to_string())
4878 })?;
4879 self.client
4880 .signal_workflow_run(&self.workflow_id, run_id, signal_name, input)
4881 .await
4882 }
4883
4884 pub async fn cancel(&self, options: WorkflowCommandOptions) -> Result<WorkflowCommandResult> {
4886 self.client
4887 .cancel_workflow(&self.workflow_id, options)
4888 .await
4889 }
4890
4891 pub async fn cancel_selected_run(
4893 &self,
4894 options: WorkflowCommandOptions,
4895 ) -> Result<WorkflowCommandResult> {
4896 let run_id = self.run_id.as_deref().ok_or_else(|| {
4897 Error::Codec("run_id is required for selected-run cancellation".to_string())
4898 })?;
4899 self.client
4900 .cancel_workflow_run(&self.workflow_id, run_id, options)
4901 .await
4902 }
4903
4904 pub async fn terminate(
4906 &self,
4907 options: WorkflowCommandOptions,
4908 ) -> Result<WorkflowCommandResult> {
4909 self.client
4910 .terminate_workflow(&self.workflow_id, options)
4911 .await
4912 }
4913
4914 pub async fn terminate_selected_run(
4916 &self,
4917 options: WorkflowCommandOptions,
4918 ) -> Result<WorkflowCommandResult> {
4919 let run_id = self.run_id.as_deref().ok_or_else(|| {
4920 Error::Codec("run_id is required for selected-run termination".to_string())
4921 })?;
4922 self.client
4923 .terminate_workflow_run(&self.workflow_id, run_id, options)
4924 .await
4925 }
4926
4927 pub async fn redrive(&self, request_id: Option<&str>) -> Result<WorkflowRedriveResult> {
4929 let run_id = self
4930 .run_id
4931 .as_deref()
4932 .ok_or_else(|| Error::Codec("run_id is required for redrive".to_string()))?;
4933 self.client
4934 .redrive_workflow_run(&self.workflow_id, run_id, request_id)
4935 .await
4936 }
4937
4938 pub async fn query<T: Serialize>(&self, query_name: &str, input: T) -> Result<Value> {
4940 self.client
4941 .query_workflow(&self.workflow_id, query_name, input)
4942 .await
4943 }
4944
4945 pub async fn query_avro_value<T: Serialize>(
4946 &self,
4947 query_name: &str,
4948 input: T,
4949 ) -> Result<AvroValue> {
4950 self.client
4951 .query_workflow_avro_value(&self.workflow_id, query_name, input)
4952 .await
4953 }
4954
4955 pub async fn update<T: Serialize>(
4956 &self,
4957 update_name: &str,
4958 input: T,
4959 request_id: Option<&str>,
4960 ) -> Result<Value> {
4961 self.client
4962 .update_workflow(&self.workflow_id, update_name, input, request_id)
4963 .await
4964 }
4965
4966 pub async fn update_avro_value<T: Serialize>(
4967 &self,
4968 update_name: &str,
4969 input: T,
4970 request_id: Option<&str>,
4971 ) -> Result<AvroValue> {
4972 self.client
4973 .update_workflow_avro_value(&self.workflow_id, update_name, input, request_id)
4974 .await
4975 }
4976
4977 pub async fn query_selected_run<T: Serialize>(
4979 &self,
4980 query_name: &str,
4981 input: T,
4982 ) -> Result<Value> {
4983 let run_id = self
4984 .run_id
4985 .as_deref()
4986 .ok_or_else(|| Error::Codec("run_id is required for selected-run query".to_string()))?;
4987 self.client
4988 .query_workflow_run(&self.workflow_id, run_id, query_name, input)
4989 .await
4990 }
4991
4992 pub async fn result(&self, options: WorkflowResultOptions) -> Result<Value> {
4994 self.result_target(options, None).await
4995 }
4996
4997 pub async fn result_avro_value(&self, options: WorkflowResultOptions) -> Result<AvroValue> {
4999 self.result_avro_value_target(options, None).await
5000 }
5001
5002 pub async fn result_typed<T: DeserializeOwned>(
5004 &self,
5005 options: WorkflowResultOptions,
5006 ) -> Result<T> {
5007 let result = self.result_avro_value(options).await?;
5008 decode_handler_result(result, HandlerKind::Workflow, &self.workflow_type)
5009 }
5010
5011 pub async fn result_selected_run(&self, options: WorkflowResultOptions) -> Result<Value> {
5013 let run_id = self.run_id.as_deref().ok_or_else(|| {
5014 Error::Codec("run_id is required for selected-run result".to_string())
5015 })?;
5016 self.result_target(options, Some(run_id)).await
5017 }
5018
5019 pub async fn result_selected_run_avro_value(
5021 &self,
5022 options: WorkflowResultOptions,
5023 ) -> Result<AvroValue> {
5024 let run_id = self.run_id.as_deref().ok_or_else(|| {
5025 Error::Codec("run_id is required for selected-run result".to_string())
5026 })?;
5027 self.result_avro_value_target(options, Some(run_id)).await
5028 }
5029
5030 pub async fn result_selected_run_typed<T: DeserializeOwned>(
5032 &self,
5033 options: WorkflowResultOptions,
5034 ) -> Result<T> {
5035 let result = self.result_selected_run_avro_value(options).await?;
5036 decode_handler_result(result, HandlerKind::Workflow, &self.workflow_type)
5037 }
5038
5039 async fn result_avro_value_target(
5040 &self,
5041 options: WorkflowResultOptions,
5042 selected_run_id: Option<&str>,
5043 ) -> Result<AvroValue> {
5044 let started = Instant::now();
5045
5046 loop {
5047 let description = match selected_run_id {
5048 Some(run_id) => {
5049 self.client
5050 .describe_workflow_run(&self.workflow_id, run_id)
5051 .await?
5052 }
5053 None => self.describe().await?,
5054 };
5055 if description.is_completed() {
5056 return description.output_avro_value.ok_or_else(|| {
5057 Error::Codec(
5058 "missing_payload_envelope: typed workflow result requires output_envelope"
5059 .to_string(),
5060 )
5061 });
5062 }
5063 if description.is_terminal() {
5064 let outcome =
5065 workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
5066 return Err(match outcome.kind {
5067 WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
5068 WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
5069 WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
5070 WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
5071 });
5072 }
5073 if started.elapsed() >= options.timeout {
5074 return Err(Error::Timeout);
5075 }
5076 tokio::time::sleep(options.poll_interval).await;
5077 }
5078 }
5079
5080 async fn result_target(
5081 &self,
5082 options: WorkflowResultOptions,
5083 selected_run_id: Option<&str>,
5084 ) -> Result<Value> {
5085 let started = Instant::now();
5086
5087 loop {
5088 let description = match selected_run_id {
5089 Some(run_id) => {
5090 self.client
5091 .describe_workflow_run(&self.workflow_id, run_id)
5092 .await?
5093 }
5094 None => self.describe().await?,
5095 };
5096 if description.is_completed() {
5097 return Ok(description.output.unwrap_or(Value::Null));
5098 }
5099
5100 if description.is_terminal() {
5101 let outcome =
5102 workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
5103 return Err(match outcome.kind {
5104 WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
5105 WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
5106 WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
5107 WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
5108 });
5109 }
5110
5111 if started.elapsed() >= options.timeout {
5112 return Err(Error::WorkflowTimedOut(WorkflowTerminalOutcome {
5113 kind: WorkflowTerminalKind::TimedOut,
5114 workflow_id: description
5115 .workflow_id
5116 .clone()
5117 .unwrap_or_else(|| self.workflow_id.clone()),
5118 run_id: description
5119 .run_id
5120 .clone()
5121 .or_else(|| selected_run_id.map(str::to_string)),
5122 reason: "result_wait_timeout".to_string(),
5123 failure_category: Some("client_timeout".to_string()),
5124 failure_id: None,
5125 exception_type: None,
5126 exception_class: None,
5127 non_retryable: None,
5128 message: Some(format!(
5129 "workflow result was not terminal within {:?}",
5130 options.timeout
5131 )),
5132 exception: None,
5133 raw: description.raw_value(),
5134 }));
5135 }
5136
5137 tokio::time::sleep(options.poll_interval).await;
5138 }
5139 }
5140}
5141
5142#[derive(Clone, Copy, Debug)]
5143pub struct WorkflowResultOptions {
5144 pub poll_interval: Duration,
5145 pub timeout: Duration,
5146}
5147
5148impl Default for WorkflowResultOptions {
5149 fn default() -> Self {
5150 Self {
5151 poll_interval: Duration::from_millis(500),
5152 timeout: Duration::from_secs(30),
5153 }
5154 }
5155}
5156
5157#[derive(Clone, Debug, Deserialize)]
5158pub struct WorkflowDescription {
5159 pub workflow_id: Option<String>,
5160 pub run_id: Option<String>,
5161 pub workflow_type: Option<String>,
5162 pub status: Option<String>,
5163 #[serde(default)]
5164 pub closed_reason: Option<String>,
5165 #[serde(default)]
5166 pub error: Option<String>,
5167 #[serde(default)]
5168 pub failure: Option<Value>,
5169 #[serde(default)]
5170 pub exception: Option<Value>,
5171 #[serde(default)]
5172 pub failures: Vec<Value>,
5173 #[serde(default)]
5174 pub output: Option<Value>,
5175 #[serde(default)]
5176 pub output_envelope: Option<Value>,
5177 #[serde(skip)]
5178 pub output_avro_value: Option<AvroValue>,
5179 #[serde(flatten)]
5180 pub raw: HashMap<String, Value>,
5181}
5182
5183#[derive(Clone, Debug, Deserialize)]
5185pub struct WorkflowStreamDescription {
5186 pub stream_name: String,
5187 pub status: String,
5188 pub last_offset: i64,
5189 pub total_items: u64,
5190 pub pending_items: u64,
5191 #[serde(default)]
5192 pub opened_at: Option<String>,
5193 #[serde(default)]
5194 pub last_appended_at: Option<String>,
5195 #[serde(default)]
5196 pub closed_at: Option<String>,
5197 #[serde(default)]
5198 pub error_reason: Option<String>,
5199 #[serde(default)]
5200 pub retention_seconds: Option<u64>,
5201 #[serde(flatten)]
5202 pub raw: HashMap<String, Value>,
5203}
5204
5205impl WorkflowStreamDescription {
5206 pub fn is_terminal(&self) -> bool {
5207 matches!(self.status.as_str(), "closed" | "errored")
5208 }
5209}
5210
5211#[derive(Clone, Debug, Default)]
5213pub struct WorkflowStreamAppendItem {
5214 pub payload_envelope: Option<Value>,
5215 pub payload_reference: Option<String>,
5216 pub item_type: Option<String>,
5217 pub content_type: Option<String>,
5218 pub idempotency_key: Option<String>,
5219}
5220
5221impl WorkflowStreamAppendItem {
5222 pub fn new<T: Serialize>(payload: T) -> Result<Self> {
5224 let value = AvroValue::from_serialize(&payload)?;
5225 Ok(Self {
5226 payload_envelope: Some(encode_typed_envelope(&value, DEFAULT_CODEC)?),
5227 ..Self::default()
5228 })
5229 }
5230
5231 pub fn from_reference(reference: impl Into<String>) -> Self {
5233 Self {
5234 payload_reference: Some(reference.into()),
5235 ..Self::default()
5236 }
5237 }
5238
5239 pub fn item_type(mut self, item_type: impl Into<String>) -> Self {
5240 self.item_type = Some(item_type.into());
5241 self
5242 }
5243
5244 pub fn content_type(mut self, content_type: impl Into<String>) -> Self {
5245 self.content_type = Some(content_type.into());
5246 self
5247 }
5248
5249 pub fn idempotency_key(mut self, idempotency_key: impl Into<String>) -> Self {
5250 self.idempotency_key = Some(idempotency_key.into());
5251 self
5252 }
5253
5254 fn wire_value(&self, derived_idempotency_key: Option<String>) -> Value {
5255 let mut item = serde_json::Map::new();
5256 if let Some(payload) = &self.payload_envelope {
5257 item.insert("payload".to_string(), payload.clone());
5258 item.insert("payload_codec".to_string(), json!(DEFAULT_CODEC));
5259 }
5260 if let Some(reference) = &self.payload_reference {
5261 item.insert("payload_reference".to_string(), json!(reference));
5262 }
5263 if let Some(item_type) = &self.item_type {
5264 item.insert("item_type".to_string(), json!(item_type));
5265 }
5266 if let Some(content_type) = &self.content_type {
5267 item.insert("content_type".to_string(), json!(content_type));
5268 }
5269 if let Some(key) = derived_idempotency_key
5270 .as_ref()
5271 .or(self.idempotency_key.as_ref())
5272 {
5273 item.insert("idempotency_key".to_string(), json!(key));
5274 }
5275 Value::Object(item)
5276 }
5277}
5278
5279#[derive(Clone, Debug)]
5281pub struct WorkflowStreamItem {
5282 pub offset: u64,
5283 pub payload: Option<Value>,
5284 pub payload_envelope: Option<Value>,
5285 pub payload_reference: Option<String>,
5286 pub payload_codec: Option<String>,
5287 pub idempotency_key: Option<String>,
5288 pub item_type: Option<String>,
5289 pub content_type: Option<String>,
5290 pub origin: Option<String>,
5291 pub origin_reference: Option<String>,
5292 pub emitted_at: Option<String>,
5293 pub raw: Value,
5294}
5295
5296#[derive(Clone, Debug)]
5298pub struct WorkflowStreamPage {
5299 pub stream: WorkflowStreamDescription,
5300 pub items: Vec<WorkflowStreamItem>,
5301 pub next_offset: u64,
5302 pub terminal: bool,
5303}
5304
5305#[derive(Clone, Debug)]
5307pub struct WorkflowStreamAppendResult {
5308 pub stream: WorkflowStreamDescription,
5309 pub accepted_offsets: Vec<u64>,
5310 pub accepted: u64,
5311 pub deduped: u64,
5312}
5313
5314#[derive(Deserialize)]
5315struct WorkflowStreamListResponse {
5316 #[serde(default)]
5317 streams: Vec<WorkflowStreamDescription>,
5318}
5319
5320#[derive(Deserialize)]
5321struct WorkflowStreamDescriptionResponse {
5322 stream: WorkflowStreamDescription,
5323}
5324
5325#[derive(Deserialize)]
5326struct WorkflowStreamPageResponse {
5327 stream: WorkflowStreamDescription,
5328 #[serde(default)]
5329 items: Vec<Value>,
5330 next_offset: u64,
5331 terminal: bool,
5332}
5333
5334#[derive(Deserialize)]
5335struct WorkflowStreamAppendResponse {
5336 stream: WorkflowStreamDescription,
5337 #[serde(default)]
5338 accepted_offsets: Vec<u64>,
5339 accepted: u64,
5340 deduped: u64,
5341}
5342
5343impl WorkflowDescription {
5344 pub fn is_completed(&self) -> bool {
5345 matches!(self.status.as_deref(), Some("completed" | "Completed"))
5346 }
5347
5348 pub fn is_terminal(&self) -> bool {
5349 matches!(
5350 self.status.as_deref(),
5351 Some(
5352 "completed"
5353 | "Completed"
5354 | "failed"
5355 | "Failed"
5356 | "cancelled"
5357 | "Cancelled"
5358 | "terminated"
5359 | "Terminated"
5360 | "timed_out"
5361 | "TimedOut",
5362 )
5363 )
5364 }
5365
5366 fn decode_payloads(&mut self) -> Result<()> {
5367 if let Some(envelope) = &self.output_envelope {
5368 let value = decode_wire_avro_value(envelope, DEFAULT_CODEC)?;
5369 self.output = Some(value.clone().into_json()?);
5370 self.output_avro_value = Some(value);
5371 }
5372
5373 Ok(())
5374 }
5375
5376 fn raw_value(&self) -> Value {
5377 let mut data = self.raw.clone();
5378 data.insert(
5379 "workflow_id".to_string(),
5380 self.workflow_id
5381 .clone()
5382 .map(Value::String)
5383 .unwrap_or(Value::Null),
5384 );
5385 data.insert(
5386 "run_id".to_string(),
5387 self.run_id
5388 .clone()
5389 .map(Value::String)
5390 .unwrap_or(Value::Null),
5391 );
5392 data.insert(
5393 "workflow_type".to_string(),
5394 self.workflow_type
5395 .clone()
5396 .map(Value::String)
5397 .unwrap_or(Value::Null),
5398 );
5399 data.insert(
5400 "status".to_string(),
5401 self.status
5402 .clone()
5403 .map(Value::String)
5404 .unwrap_or(Value::Null),
5405 );
5406 data.insert(
5407 "closed_reason".to_string(),
5408 self.closed_reason
5409 .clone()
5410 .map(Value::String)
5411 .unwrap_or(Value::Null),
5412 );
5413 if let Some(failure) = &self.failure {
5414 data.insert("failure".to_string(), failure.clone());
5415 }
5416 if let Some(exception) = &self.exception {
5417 data.insert("exception".to_string(), exception.clone());
5418 }
5419 Value::Object(data.into_iter().collect())
5420 }
5421}
5422
5423fn workflow_terminal_outcome(
5424 description: &WorkflowDescription,
5425 workflow_id: &str,
5426 run_id: Option<&str>,
5427) -> WorkflowTerminalOutcome {
5428 let terminal_kind = description
5429 .closed_reason
5430 .as_deref()
5431 .or(description.status.as_deref())
5432 .unwrap_or("failed")
5433 .to_ascii_lowercase();
5434 let kind = match terminal_kind.as_str() {
5435 "cancelled" | "canceled" => WorkflowTerminalKind::Cancelled,
5436 "terminated" => WorkflowTerminalKind::Terminated,
5437 "timed_out" | "timedout" => WorkflowTerminalKind::TimedOut,
5438 _ => WorkflowTerminalKind::Failed,
5439 };
5440 let default_reason = match kind {
5441 WorkflowTerminalKind::Failed => "workflow_failed",
5442 WorkflowTerminalKind::Cancelled => "cancelled",
5443 WorkflowTerminalKind::Terminated => "terminated",
5444 WorkflowTerminalKind::TimedOut => "timed_out",
5445 };
5446 let failure = description
5447 .failure
5448 .as_ref()
5449 .filter(|value| value.is_object());
5450 let nested_failure = failure
5451 .and_then(|value| value.get("failures"))
5452 .and_then(Value::as_array)
5453 .and_then(|failures| failures.last())
5454 .or_else(|| description.failures.last());
5455 let exception = description
5456 .exception
5457 .clone()
5458 .or_else(|| failure.and_then(|value| value.get("exception")).cloned())
5459 .or_else(|| {
5460 nested_failure
5461 .and_then(|value| value.get("exception_payload"))
5462 .cloned()
5463 });
5464 let string_field = |name: &str| {
5465 failure
5466 .and_then(|value| value.get(name))
5467 .and_then(Value::as_str)
5468 .or_else(|| {
5469 nested_failure
5470 .and_then(|value| value.get(name))
5471 .and_then(Value::as_str)
5472 })
5473 .map(str::to_string)
5474 };
5475 let exception_field = |name: &str| {
5476 exception
5477 .as_ref()
5478 .and_then(|value| value.get(name))
5479 .and_then(Value::as_str)
5480 .map(str::to_string)
5481 };
5482 let message = description
5483 .error
5484 .clone()
5485 .or_else(|| string_field("message"))
5486 .or_else(|| exception_field("message"));
5487 let reason = description
5488 .raw
5489 .get("reason")
5490 .and_then(Value::as_str)
5491 .map(str::to_string)
5492 .or_else(|| {
5493 failure
5494 .and_then(|value| value.get("reason"))
5495 .and_then(Value::as_str)
5496 .map(str::to_string)
5497 })
5498 .or_else(|| description.closed_reason.clone())
5499 .unwrap_or_else(|| default_reason.to_string());
5500 let failure_id = string_field("failure_id").or_else(|| {
5501 nested_failure
5502 .and_then(|value| value.get("id"))
5503 .and_then(Value::as_str)
5504 .map(str::to_string)
5505 });
5506
5507 WorkflowTerminalOutcome {
5508 kind,
5509 workflow_id: description
5510 .workflow_id
5511 .clone()
5512 .unwrap_or_else(|| workflow_id.to_string()),
5513 run_id: description
5514 .run_id
5515 .clone()
5516 .or_else(|| run_id.map(str::to_string)),
5517 reason,
5518 failure_category: string_field("failure_category")
5519 .or_else(|| Some(default_reason.to_string())),
5520 failure_id,
5521 exception_type: string_field("exception_type").or_else(|| exception_field("type")),
5522 exception_class: string_field("exception_class").or_else(|| exception_field("class")),
5523 non_retryable: failure
5524 .and_then(|value| value.get("non_retryable"))
5525 .and_then(Value::as_bool)
5526 .or_else(|| {
5527 nested_failure
5528 .and_then(|value| value.get("non_retryable"))
5529 .and_then(Value::as_bool)
5530 }),
5531 message,
5532 exception,
5533 raw: description.raw_value(),
5534 }
5535}
5536
5537#[derive(Clone, Debug, Deserialize)]
5538pub struct RegisterWorkerResponse {
5539 pub worker_id: String,
5540 pub registered: bool,
5541 #[serde(default)]
5542 pub heartbeat_interval_seconds: Option<u64>,
5543 #[serde(default)]
5544 pub protocol_version: Option<String>,
5545 #[serde(default)]
5546 pub server_capabilities: Option<Value>,
5547}
5548
5549#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
5551pub struct WorkerDeregistrationEnvelope {
5552 pub worker_id: String,
5553 pub outcome: String,
5554 pub recovered_workflow_task_count: u64,
5555}
5556
5557#[derive(Clone, Debug, Deserialize)]
5558pub struct PollWorkflowTaskResponse {
5559 #[serde(default)]
5560 pub task: Option<WorkflowTask>,
5561 #[serde(default)]
5562 pub poll_status: Option<String>,
5563 #[serde(default)]
5564 pub reason: Option<String>,
5565 #[serde(default)]
5566 pub protocol_version: Option<String>,
5567 #[serde(default)]
5568 pub server_capabilities: Option<Value>,
5569}
5570
5571impl PollWorkflowTaskResponse {
5572 pub fn outcome(&self) -> WorkerPollOutcome {
5574 worker_poll_outcome(
5575 self.task.is_some(),
5576 self.poll_status.as_deref(),
5577 self.reason.as_deref(),
5578 )
5579 }
5580}
5581
5582fn runtime_supports_workflow_memo_updates(capabilities: Option<&Value>) -> bool {
5583 let Some(capabilities) = capabilities.and_then(Value::as_object) else {
5584 return false;
5585 };
5586 let supported = capabilities
5587 .get("workflow_memo_updates")
5588 .and_then(Value::as_object)
5589 .and_then(|memo| memo.get("supported"))
5590 .and_then(Value::as_bool)
5591 == Some(true);
5592 let command_advertised = capabilities
5593 .get("supported_workflow_task_commands")
5594 .and_then(Value::as_array)
5595 .is_some_and(|commands| {
5596 commands
5597 .iter()
5598 .any(|command| command.as_str() == Some("upsert_memo"))
5599 });
5600 supported && command_advertised
5601}
5602
5603fn commands_use_workflow_memo_updates(commands: &[Value]) -> bool {
5604 commands
5605 .iter()
5606 .any(|command| command.get("type").and_then(Value::as_str) == Some("upsert_memo"))
5607}
5608
5609#[derive(Clone, Debug, Deserialize)]
5610pub struct PollActivityTaskResponse {
5611 #[serde(default)]
5612 pub task: Option<ActivityTask>,
5613 #[serde(default)]
5614 pub poll_status: Option<String>,
5615 #[serde(default)]
5616 pub reason: Option<String>,
5617}
5618
5619impl PollActivityTaskResponse {
5620 pub fn outcome(&self) -> WorkerPollOutcome {
5622 worker_poll_outcome(
5623 self.task.is_some(),
5624 self.poll_status.as_deref(),
5625 self.reason.as_deref(),
5626 )
5627 }
5628}
5629
5630#[derive(Clone, Debug, Deserialize)]
5631pub struct PollQueryTaskResponse {
5632 #[serde(default)]
5633 pub task: Option<QueryTask>,
5634 #[serde(default)]
5635 pub poll_status: Option<String>,
5636 #[serde(default)]
5637 pub reason: Option<String>,
5638}
5639
5640impl PollQueryTaskResponse {
5641 pub fn outcome(&self) -> WorkerPollOutcome {
5643 worker_poll_outcome(
5644 self.task.is_some(),
5645 self.poll_status.as_deref(),
5646 self.reason.as_deref(),
5647 )
5648 }
5649}
5650
5651#[derive(Clone, Debug, PartialEq, Eq)]
5653pub enum WorkerPollOutcome {
5654 Task,
5656 Idle {
5658 poll_status: Option<String>,
5659 reason: Option<String>,
5660 },
5661 Stop {
5663 poll_status: Option<String>,
5664 reason: Option<String>,
5665 },
5666}
5667
5668impl WorkerPollOutcome {
5669 pub fn should_stop(&self) -> bool {
5670 matches!(self, Self::Stop { .. })
5671 }
5672}
5673
5674fn worker_poll_outcome(
5675 has_task: bool,
5676 poll_status: Option<&str>,
5677 reason: Option<&str>,
5678) -> WorkerPollOutcome {
5679 if worker_poll_is_stop(poll_status, reason) {
5680 return WorkerPollOutcome::Stop {
5681 poll_status: poll_status.map(str::to_string),
5682 reason: reason.map(str::to_string),
5683 };
5684 }
5685
5686 if has_task {
5687 WorkerPollOutcome::Task
5688 } else {
5689 WorkerPollOutcome::Idle {
5690 poll_status: poll_status.map(str::to_string),
5691 reason: reason.map(str::to_string),
5692 }
5693 }
5694}
5695
5696#[derive(Clone, Debug, Deserialize)]
5698pub struct QueryTask {
5699 pub query_task_id: String,
5700 #[serde(default = "default_workflow_task_attempt")]
5701 pub query_task_attempt: u64,
5702 #[serde(default)]
5703 pub lease_owner: Option<String>,
5704 #[serde(default)]
5705 pub workflow_id: Option<String>,
5706 #[serde(default)]
5707 pub run_id: Option<String>,
5708 pub workflow_type: String,
5709 pub query_name: String,
5710 #[serde(
5711 default = "missing_task_payload_codec",
5712 deserialize_with = "deserialize_task_payload_codec"
5713 )]
5714 pub payload_codec: String,
5715 #[serde(default)]
5716 pub workflow_arguments: Option<Value>,
5717 #[serde(default)]
5718 pub query_arguments: Option<Value>,
5719 #[serde(default)]
5720 pub history_events: Vec<HistoryEvent>,
5721 #[serde(default)]
5722 pub history_export: Option<Value>,
5723 #[serde(default)]
5724 pub run_status: Option<String>,
5725}
5726
5727#[derive(Clone, Debug, Deserialize)]
5728pub struct WorkflowTask {
5729 pub task_id: String,
5730 #[serde(default)]
5731 pub workflow_command_id: Option<String>,
5732 #[serde(default)]
5733 pub workflow_id: Option<String>,
5734 #[serde(default)]
5735 pub run_id: Option<String>,
5736 pub workflow_type: String,
5737 #[serde(default)]
5738 pub cancel_requested: bool,
5739 #[serde(
5740 default = "missing_task_payload_codec",
5741 deserialize_with = "deserialize_task_payload_codec"
5742 )]
5743 pub payload_codec: String,
5744 #[serde(default)]
5745 pub arguments: Option<Value>,
5746 #[serde(default)]
5747 pub history_events: Vec<HistoryEvent>,
5748 #[serde(default)]
5749 pub total_history_events: Option<u64>,
5750 #[serde(default)]
5751 pub history_size_bytes: Option<u64>,
5752 #[serde(default)]
5753 pub continue_as_new_recommended: Option<bool>,
5754 #[serde(default)]
5755 pub history_budget_pressure: Option<String>,
5756 #[serde(default)]
5757 pub next_history_page_token: Option<String>,
5758 #[serde(default = "default_workflow_task_attempt")]
5759 pub workflow_task_attempt: u64,
5760 #[serde(default)]
5761 pub workflow_signal_id: Option<String>,
5762 #[serde(default)]
5763 pub signal_name: Option<String>,
5764 #[serde(default)]
5765 pub signal_arguments: Option<Value>,
5766 #[serde(default)]
5767 pub workflow_update_id: Option<String>,
5768 #[serde(default)]
5769 pub update_name: Option<String>,
5770 #[serde(default)]
5771 pub lease_owner: Option<String>,
5772}
5773
5774impl WorkflowTask {
5775 fn append_history_page(&mut self, page: WorkflowTaskHistoryPage) {
5776 self.history_events.extend(page.history_events);
5777
5778 if page.total_history_events.is_some() {
5779 self.total_history_events = page.total_history_events;
5780 }
5781
5782 self.next_history_page_token = page
5783 .next_history_page_token
5784 .filter(|token| !token.is_empty());
5785 }
5786}
5787
5788#[derive(Clone, Debug, Deserialize)]
5789struct WorkflowTaskHistoryPage {
5790 #[serde(default)]
5791 history_events: Vec<HistoryEvent>,
5792 #[serde(default)]
5793 total_history_events: Option<u64>,
5794 #[serde(default)]
5795 next_history_page_token: Option<String>,
5796}
5797
5798#[derive(Clone, Debug, Deserialize)]
5799pub struct ActivityTask {
5800 pub task_id: String,
5801 #[serde(default)]
5802 pub activity_attempt_id: Option<String>,
5803 #[serde(default)]
5804 pub attempt_id: Option<String>,
5805 pub activity_type: String,
5806 #[serde(
5807 default = "missing_task_payload_codec",
5808 deserialize_with = "deserialize_task_payload_codec"
5809 )]
5810 pub payload_codec: String,
5811 #[serde(default)]
5812 pub arguments: Option<Value>,
5813 #[serde(default = "default_attempt_number")]
5814 pub attempt_number: u64,
5815 #[serde(default)]
5816 pub lease_owner: Option<String>,
5817}
5818
5819#[derive(Clone, Debug, Deserialize)]
5820pub struct HistoryEvent {
5821 #[serde(alias = "type")]
5822 pub event_type: String,
5823 #[serde(default)]
5824 pub payload: Value,
5825 #[serde(flatten)]
5826 pub raw: HashMap<String, Value>,
5827}
5828
5829#[derive(Clone, Debug, PartialEq)]
5831pub struct QuerySignal {
5832 pub id: Option<String>,
5833 pub name: String,
5834 pub arguments: Vec<Value>,
5835 avro_arguments: Vec<AvroValue>,
5836 pub workflow_sequence: Option<u64>,
5837}
5838
5839impl QuerySignal {
5840 pub fn arguments_avro_value(&self) -> &[AvroValue] {
5842 &self.avro_arguments
5843 }
5844}
5845
5846#[derive(Clone, Debug)]
5852pub struct QueryContext {
5853 pub workflow_id: Option<String>,
5854 pub run_id: Option<String>,
5855 pub workflow_type: String,
5856 pub run_status: Option<String>,
5857 workflow_input: Value,
5858 workflow_input_avro_value: AvroValue,
5859 history_events: Arc<Vec<HistoryEvent>>,
5860 signal_events: Arc<Vec<QuerySignal>>,
5861}
5862
5863impl QueryContext {
5864 pub fn workflow_input(&self) -> &Value {
5866 &self.workflow_input
5867 }
5868
5869 pub fn workflow_input_avro_value(&self) -> &AvroValue {
5871 &self.workflow_input_avro_value
5872 }
5873
5874 pub fn history_events(&self) -> &[HistoryEvent] {
5876 self.history_events.as_slice()
5877 }
5878
5879 pub fn signal_events(&self) -> &[QuerySignal] {
5881 self.signal_events.as_slice()
5882 }
5883
5884 pub fn signals(&self, signal_name: &str) -> Vec<Vec<Value>> {
5886 self.signal_events
5887 .iter()
5888 .filter(|signal| signal.name == signal_name)
5889 .map(|signal| signal.arguments.clone())
5890 .collect()
5891 }
5892
5893 pub fn signals_avro_value(&self, signal_name: &str) -> Vec<Vec<AvroValue>> {
5895 self.signal_events
5896 .iter()
5897 .filter(|signal| signal.name == signal_name)
5898 .map(|signal| signal.avro_arguments.clone())
5899 .collect()
5900 }
5901}
5902
5903#[derive(Clone, Debug, Deserialize)]
5904pub struct ActivityHeartbeatResponse {
5905 #[serde(default)]
5906 pub cancel_requested: bool,
5907 #[serde(default)]
5908 pub heartbeat_recorded: bool,
5909 #[serde(default)]
5910 pub can_continue: Option<bool>,
5911 #[serde(default)]
5912 pub reason: Option<String>,
5913 #[serde(default)]
5914 pub run_closed_reason: Option<String>,
5915 #[serde(default)]
5916 pub run_closed_at: Option<String>,
5917 #[serde(default)]
5918 pub lease_expires_at: Option<String>,
5919 #[serde(default)]
5920 pub last_heartbeat_at: Option<String>,
5921}
5922
5923impl ActivityHeartbeatResponse {
5924 pub fn should_stop(&self) -> bool {
5926 self.cancel_requested || self.can_continue == Some(false)
5927 }
5928}
5929
5930fn missing_task_payload_codec() -> String {
5931 MISSING_TASK_PAYLOAD_CODEC.to_string()
5932}
5933
5934fn deserialize_task_payload_codec<'de, D>(deserializer: D) -> std::result::Result<String, D::Error>
5935where
5936 D: Deserializer<'de>,
5937{
5938 Ok(match Value::deserialize(deserializer)? {
5939 Value::String(codec) => codec,
5940 Value::Null => NULL_TASK_PAYLOAD_CODEC.to_string(),
5941 _ => NON_STRING_TASK_PAYLOAD_CODEC.to_string(),
5942 })
5943}
5944
5945fn default_workflow_task_attempt() -> u64 {
5946 1
5947}
5948
5949fn default_attempt_number() -> u64 {
5950 1
5951}
5952
5953type WorkflowFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5954type WorkflowHandler = Arc<dyn Fn(WorkflowContext, AvroValue) -> WorkflowFuture + Send + Sync>;
5955type ErasedWorkflowState = Arc<dyn Any + Send + Sync>;
5956type WorkflowStateSnapshot = Arc<dyn Fn() -> Result<ErasedWorkflowState> + Send + Sync>;
5957type ReplayedWorkflowHandler =
5958 Arc<dyn Fn(WorkflowContext, AvroValue) -> ReplayedWorkflowInvocation + Send + Sync>;
5959type ActivityFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5960type ActivityHandler = Arc<dyn Fn(ActivityContext, AvroValue) -> ActivityFuture + Send + Sync>;
5961type QueryFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5962type QueryHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
5963type UpdateHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
5964type ReplayedQueryHandler = Arc<
5965 dyn Fn(QueryContext, ErasedWorkflowState, AvroValue) -> std::result::Result<QueryFuture, String>
5966 + Send
5967 + Sync,
5968>;
5969type WorkerHeartbeatObserver = Arc<dyn Fn(&WorkerHeartbeatObservation) + Send + Sync>;
5970
5971struct ReplayedWorkflowInvocation {
5972 future: WorkflowFuture,
5973 snapshot: WorkflowStateSnapshot,
5974}
5975
5976#[derive(Clone)]
5977struct RegisteredWorkflow {
5978 execute: WorkflowHandler,
5979 replay: Option<ReplayedWorkflowHandler>,
5980 state_type: Option<TypeId>,
5981 definition_fingerprint: Option<String>,
5982}
5983
5984#[derive(Debug)]
5985struct WorkflowTaskDecision {
5986 commands: Vec<Value>,
5987 message_stream_cursors: Vec<Value>,
5988 message_stream_waits: Vec<Value>,
5989}
5990
5991impl WorkflowTaskDecision {
5992 fn without_message_streams(commands: Vec<Value>) -> Self {
5993 Self {
5994 commands,
5995 message_stream_cursors: Vec::new(),
5996 message_stream_waits: Vec::new(),
5997 }
5998 }
5999}
6000
6001#[derive(Clone)]
6002enum RegisteredQuery {
6003 Snapshot(QueryHandler),
6004 Replayed {
6005 state_type: TypeId,
6006 handler: ReplayedQueryHandler,
6007 },
6008}
6009
6010#[derive(Clone, Debug)]
6011pub struct WorkerHeartbeatObservation {
6012 pub worker_id: String,
6013 pub task_queue: String,
6014 pub acknowledged_at_unix_millis: u64,
6015 pub acknowledgement: Value,
6016}
6017
6018#[derive(Clone, Copy, Debug)]
6030pub struct WorkerRetryPolicy {
6031 pub max_retries: usize,
6033 pub initial_backoff: Duration,
6035 pub max_backoff: Duration,
6037}
6038
6039impl Default for WorkerRetryPolicy {
6040 fn default() -> Self {
6041 Self {
6042 max_retries: 5,
6043 initial_backoff: Duration::from_millis(100),
6044 max_backoff: Duration::from_secs(5),
6045 }
6046 }
6047}
6048
6049#[derive(Clone, Debug)]
6050struct WorkerStorageAdmission {
6051 policy: WorkerRetryPolicy,
6052 stop: Arc<AtomicBool>,
6053}
6054
6055struct StopWorkerOnDrop(Arc<AtomicBool>);
6056
6057impl Drop for StopWorkerOnDrop {
6058 fn drop(&mut self) {
6059 self.0.store(true, Ordering::SeqCst);
6060 }
6061}
6062
6063async fn wait_for_worker_stop(stop: &AtomicBool) {
6064 while !stop.load(Ordering::SeqCst) {
6065 tokio::time::sleep(Duration::from_millis(100)).await;
6066 }
6067}
6068
6069#[derive(Clone, Copy, Debug, PartialEq, Eq)]
6070enum ManagedPollOutcome {
6071 Idle,
6072 Handled,
6073 Stop,
6074}
6075
6076#[derive(Clone)]
6077pub struct Worker {
6078 client: Client,
6079 worker_id: String,
6080 task_queue: String,
6081 workflows: HashMap<String, RegisteredWorkflow>,
6082 activities: HashMap<String, ActivityHandler>,
6083 queries: HashMap<String, HashMap<String, RegisteredQuery>>,
6084 updates: HashMap<String, HashMap<String, UpdateHandler>>,
6085 max_concurrent_workflow_tasks: usize,
6086 max_concurrent_activity_tasks: usize,
6087 poll_timeout: Duration,
6088 heartbeat_interval: Duration,
6089 retry_policy: WorkerRetryPolicy,
6090 heartbeat_observer: Option<WorkerHeartbeatObserver>,
6091}
6092
6093impl Worker {
6094 pub fn new(client: Client, task_queue: impl Into<String>) -> Self {
6095 Self {
6096 client,
6097 worker_id: default_worker_id(),
6098 task_queue: task_queue.into(),
6099 workflows: HashMap::new(),
6100 activities: HashMap::new(),
6101 queries: HashMap::new(),
6102 updates: HashMap::new(),
6103 max_concurrent_workflow_tasks: 10,
6104 max_concurrent_activity_tasks: 10,
6105 poll_timeout: Duration::from_secs(30),
6106 heartbeat_interval: Duration::from_secs(60),
6107 retry_policy: WorkerRetryPolicy::default(),
6108 heartbeat_observer: None,
6109 }
6110 }
6111
6112 pub fn worker_id(mut self, worker_id: impl Into<String>) -> Self {
6113 self.worker_id = worker_id.into();
6114 self
6115 }
6116
6117 pub fn poll_timeout(mut self, timeout: Duration) -> Self {
6118 self.poll_timeout = timeout;
6119 self
6120 }
6121
6122 pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
6123 self.heartbeat_interval = interval;
6124 self
6125 }
6126
6127 pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self {
6129 self.retry_policy = policy;
6130 self
6131 }
6132
6133 pub fn on_worker_heartbeat<F>(mut self, observer: F) -> Self
6134 where
6135 F: Fn(&WorkerHeartbeatObservation) + Send + Sync + 'static,
6136 {
6137 self.heartbeat_observer = Some(Arc::new(observer));
6138 self
6139 }
6140
6141 pub fn max_concurrent_workflow_tasks(mut self, count: usize) -> Self {
6142 self.max_concurrent_workflow_tasks = count.max(1);
6143 self
6144 }
6145
6146 pub fn max_concurrent_activity_tasks(mut self, count: usize) -> Self {
6147 self.max_concurrent_activity_tasks = count.max(1);
6148 self
6149 }
6150
6151 pub fn register_workflow<F, Fut>(&mut self, workflow_type: impl Into<String>, handler: F)
6158 where
6159 F: Fn(WorkflowContext, Value) -> Fut + Send + Sync + 'static,
6160 Fut: Future<Output = Result<Value>> + Send + 'static,
6161 {
6162 let handler = Arc::new(handler);
6163 self.workflows.insert(
6164 workflow_type.into(),
6165 RegisteredWorkflow {
6166 execute: Arc::new(move |ctx, input| {
6167 let handler = Arc::clone(&handler);
6168 Box::pin(async move {
6169 let result = handler(ctx, input.into_json()?).await?;
6170 AvroValue::from_serialize(&result)
6171 })
6172 }),
6173 replay: None,
6174 state_type: None,
6175 definition_fingerprint: None,
6176 },
6177 );
6178 }
6179
6180 pub fn register_typed_workflow<I, O, F, Fut>(
6191 &mut self,
6192 workflow_type: impl Into<String>,
6193 handler: F,
6194 ) where
6195 I: DeserializeOwned + Send + 'static,
6196 O: Serialize + Send + 'static,
6197 F: Fn(WorkflowContext, I) -> Fut + Send + Sync + 'static,
6198 Fut: Future<Output = Result<O>> + Send + 'static,
6199 {
6200 let workflow_type = workflow_type.into();
6201 let handler_name = workflow_type.clone();
6202 let handler = Arc::new(handler);
6203 self.workflows.insert(
6204 workflow_type,
6205 RegisteredWorkflow {
6206 execute: Arc::new(move |ctx, input| {
6207 let handler = Arc::clone(&handler);
6208 let handler_name = handler_name.clone();
6209 Box::pin(async move {
6210 let input =
6211 decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
6212 let result = handler(ctx, input).await?;
6213 encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
6214 })
6215 }),
6216 replay: None,
6217 state_type: None,
6218 definition_fingerprint: None,
6219 },
6220 );
6221 }
6222
6223 pub fn register_workflow_avro_value<F, Fut>(
6225 &mut self,
6226 workflow_type: impl Into<String>,
6227 handler: F,
6228 ) where
6229 F: Fn(WorkflowContext, AvroValue) -> Fut + Send + Sync + 'static,
6230 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6231 {
6232 self.workflows.insert(
6233 workflow_type.into(),
6234 RegisteredWorkflow {
6235 execute: Arc::new(move |ctx, input| Box::pin(handler(ctx, input))),
6236 replay: None,
6237 state_type: None,
6238 definition_fingerprint: None,
6239 },
6240 );
6241 }
6242
6243 pub fn register_replayed_workflow<S, Factory, F, Fut>(
6251 &mut self,
6252 workflow_type: impl Into<String>,
6253 state_factory: Factory,
6254 handler: F,
6255 ) where
6256 S: Clone + Send + Sync + 'static,
6257 Factory: Fn() -> S + Send + Sync + 'static,
6258 F: Fn(WorkflowContext, Value, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
6259 Fut: Future<Output = Result<Value>> + Send + 'static,
6260 {
6261 let state_factory = Arc::new(state_factory);
6262 let handler = Arc::new(handler);
6263
6264 let execute_factory = Arc::clone(&state_factory);
6265 let execute_handler = Arc::clone(&handler);
6266 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6267 let state = WorkflowInstance::new(execute_factory());
6268 let handler = Arc::clone(&execute_handler);
6269 Box::pin(async move {
6270 let result = handler(ctx, input.into_json()?, state).await?;
6271 AvroValue::from_serialize(&result)
6272 }) as WorkflowFuture
6273 });
6274
6275 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6276 let state = WorkflowInstance::new(state_factory());
6277 let snapshot_state = state.clone();
6278 let snapshot: WorkflowStateSnapshot =
6279 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6280 let replay_handler = Arc::clone(&handler);
6281 let future = async move {
6282 let result = replay_handler(ctx, input.into_json()?, state).await?;
6283 AvroValue::from_serialize(&result)
6284 };
6285 ReplayedWorkflowInvocation {
6286 future: Box::pin(future),
6287 snapshot,
6288 }
6289 });
6290
6291 self.workflows.insert(
6292 workflow_type.into(),
6293 RegisteredWorkflow {
6294 execute,
6295 replay: Some(replay),
6296 state_type: Some(TypeId::of::<S>()),
6297 definition_fingerprint: None,
6298 },
6299 );
6300 }
6301
6302 pub fn register_typed_replayed_workflow<I, O, S, Factory, F, Fut>(
6308 &mut self,
6309 workflow_type: impl Into<String>,
6310 state_factory: Factory,
6311 handler: F,
6312 ) where
6313 I: DeserializeOwned + Send + 'static,
6314 O: Serialize + Send + 'static,
6315 S: Clone + Send + Sync + 'static,
6316 Factory: Fn() -> S + Send + Sync + 'static,
6317 F: Fn(WorkflowContext, I, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
6318 Fut: Future<Output = Result<O>> + Send + 'static,
6319 {
6320 let workflow_type = workflow_type.into();
6321 let state_factory = Arc::new(state_factory);
6322 let handler = Arc::new(handler);
6323
6324 let execute_name = workflow_type.clone();
6325 let execute_factory = Arc::clone(&state_factory);
6326 let execute_handler = Arc::clone(&handler);
6327 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6328 let state = WorkflowInstance::new(execute_factory());
6329 let handler = Arc::clone(&execute_handler);
6330 let handler_name = execute_name.clone();
6331 Box::pin(async move {
6332 let input = decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
6333 let result = handler(ctx, input, state).await?;
6334 encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
6335 }) as WorkflowFuture
6336 });
6337
6338 let replay_name = workflow_type.clone();
6339 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6340 let state = WorkflowInstance::new(state_factory());
6341 let snapshot_state = state.clone();
6342 let snapshot: WorkflowStateSnapshot =
6343 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6344 let handler = Arc::clone(&handler);
6345 let handler_name = replay_name.clone();
6346 let future = async move {
6347 let input = decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
6348 let result = handler(ctx, input, state).await?;
6349 encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
6350 };
6351 ReplayedWorkflowInvocation {
6352 future: Box::pin(future),
6353 snapshot,
6354 }
6355 });
6356
6357 self.workflows.insert(
6358 workflow_type,
6359 RegisteredWorkflow {
6360 execute,
6361 replay: Some(replay),
6362 state_type: Some(TypeId::of::<S>()),
6363 definition_fingerprint: None,
6364 },
6365 );
6366 }
6367
6368 pub fn set_workflow_definition_sources(
6378 &mut self,
6379 workflow_type: &str,
6380 sources: &[&str],
6381 ) -> Result<()> {
6382 if sources.is_empty() || sources.iter().any(|source| source.is_empty()) {
6383 return Err(Error::Codec(
6384 "workflow definition sources must be non-empty".to_string(),
6385 ));
6386 }
6387 let workflow = self.workflows.get_mut(workflow_type).ok_or_else(|| {
6388 Error::Codec(format!("workflow type {workflow_type:?} is not registered"))
6389 })?;
6390 let mut hasher = Sha256::new();
6391 hasher.update(b"durable-workflow-rust.workflow-definition.v1\0");
6392 hasher.update((workflow_type.len() as u64).to_be_bytes());
6393 hasher.update(workflow_type.as_bytes());
6394 for source in sources {
6395 hasher.update((source.len() as u64).to_be_bytes());
6396 hasher.update(source.as_bytes());
6397 }
6398 workflow.definition_fingerprint = Some(format!("sha256:{:x}", hasher.finalize()));
6399 Ok(())
6400 }
6401
6402 pub fn register_replayed_workflow_avro_value<S, Factory, F, Fut>(
6404 &mut self,
6405 workflow_type: impl Into<String>,
6406 state_factory: Factory,
6407 handler: F,
6408 ) where
6409 S: Clone + Send + Sync + 'static,
6410 Factory: Fn() -> S + Send + Sync + 'static,
6411 F: Fn(WorkflowContext, AvroValue, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
6412 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6413 {
6414 let state_factory = Arc::new(state_factory);
6415 let handler = Arc::new(handler);
6416
6417 let execute_factory = Arc::clone(&state_factory);
6418 let execute_handler = Arc::clone(&handler);
6419 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6420 let state = WorkflowInstance::new(execute_factory());
6421 Box::pin(execute_handler(ctx, input, state)) as WorkflowFuture
6422 });
6423
6424 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6425 let state = WorkflowInstance::new(state_factory());
6426 let snapshot_state = state.clone();
6427 let snapshot: WorkflowStateSnapshot =
6428 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6429 ReplayedWorkflowInvocation {
6430 future: Box::pin(handler(ctx, input, state)),
6431 snapshot,
6432 }
6433 });
6434
6435 self.workflows.insert(
6436 workflow_type.into(),
6437 RegisteredWorkflow {
6438 execute,
6439 replay: Some(replay),
6440 state_type: Some(TypeId::of::<S>()),
6441 definition_fingerprint: None,
6442 },
6443 );
6444 }
6445
6446 pub fn register_activity<F, Fut>(&mut self, activity_type: impl Into<String>, handler: F)
6447 where
6448 F: Fn(ActivityContext, Value) -> Fut + Send + Sync + 'static,
6449 Fut: Future<Output = Result<Value>> + Send + 'static,
6450 {
6451 let handler = Arc::new(handler);
6452 self.activities.insert(
6453 activity_type.into(),
6454 Arc::new(move |ctx, args| {
6455 let handler = Arc::clone(&handler);
6456 Box::pin(async move {
6457 let result = handler(ctx, args.into_json()?).await?;
6458 AvroValue::from_serialize(&result)
6459 })
6460 }),
6461 );
6462 }
6463
6464 pub fn register_typed_activity<I, O, F, Fut>(
6470 &mut self,
6471 activity_type: impl Into<String>,
6472 handler: F,
6473 ) where
6474 I: DeserializeOwned + Send + 'static,
6475 O: Serialize + Send + 'static,
6476 F: Fn(ActivityContext, I) -> Fut + Send + Sync + 'static,
6477 Fut: Future<Output = Result<O>> + Send + 'static,
6478 {
6479 let activity_type = activity_type.into();
6480 let handler_name = activity_type.clone();
6481 let handler = Arc::new(handler);
6482 self.activities.insert(
6483 activity_type,
6484 Arc::new(move |ctx, input| {
6485 let handler = Arc::clone(&handler);
6486 let handler_name = handler_name.clone();
6487 Box::pin(async move {
6488 let input =
6489 decode_handler_input::<I>(input, HandlerKind::Activity, &handler_name)?;
6490 let result = handler(ctx, input).await?;
6491 encode_handler_result(&result, HandlerKind::Activity, &handler_name)
6492 })
6493 }),
6494 );
6495 }
6496
6497 pub fn register_activity_avro_value<F, Fut>(
6499 &mut self,
6500 activity_type: impl Into<String>,
6501 handler: F,
6502 ) where
6503 F: Fn(ActivityContext, AvroValue) -> Fut + Send + Sync + 'static,
6504 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6505 {
6506 self.activities.insert(
6507 activity_type.into(),
6508 Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
6509 );
6510 }
6511
6512 pub fn register_query<F, Fut>(
6518 &mut self,
6519 workflow_type: impl Into<String>,
6520 query_name: impl Into<String>,
6521 handler: F,
6522 ) where
6523 F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
6524 Fut: Future<Output = Result<Value>> + Send + 'static,
6525 {
6526 let handler = Arc::new(handler);
6527 self.queries
6528 .entry(workflow_type.into())
6529 .or_default()
6530 .insert(
6531 query_name.into(),
6532 RegisteredQuery::Snapshot(Arc::new(move |ctx, args| {
6533 let handler = Arc::clone(&handler);
6534 Box::pin(async move {
6535 let result = handler(ctx, args.into_json()?).await?;
6536 AvroValue::from_serialize(&result)
6537 })
6538 })),
6539 );
6540 }
6541
6542 pub fn register_query_avro_value<F, Fut>(
6544 &mut self,
6545 workflow_type: impl Into<String>,
6546 query_name: impl Into<String>,
6547 handler: F,
6548 ) where
6549 F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
6550 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6551 {
6552 self.queries
6553 .entry(workflow_type.into())
6554 .or_default()
6555 .insert(
6556 query_name.into(),
6557 RegisteredQuery::Snapshot(Arc::new(move |ctx, args| Box::pin(handler(ctx, args)))),
6558 );
6559 }
6560
6561 pub fn register_replayed_query<S, F, Fut>(
6568 &mut self,
6569 workflow_type: impl Into<String>,
6570 query_name: impl Into<String>,
6571 handler: F,
6572 ) where
6573 S: Clone + Send + Sync + 'static,
6574 F: Fn(QueryContext, Arc<S>, Value) -> Fut + Send + Sync + 'static,
6575 Fut: Future<Output = Result<Value>> + Send + 'static,
6576 {
6577 let handler = Arc::new(handler);
6578 let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
6579 let state = state.downcast::<S>().map_err(|_| {
6580 "registered query state type does not match the replayed workflow state".to_string()
6581 })?;
6582 let handler = Arc::clone(&handler);
6583 Ok(Box::pin(async move {
6584 let result = handler(ctx, state, args.into_json()?).await?;
6585 AvroValue::from_serialize(&result)
6586 }))
6587 });
6588
6589 self.queries
6590 .entry(workflow_type.into())
6591 .or_default()
6592 .insert(
6593 query_name.into(),
6594 RegisteredQuery::Replayed {
6595 state_type: TypeId::of::<S>(),
6596 handler: erased_handler,
6597 },
6598 );
6599 }
6600
6601 pub fn register_replayed_query_avro_value<S, F, Fut>(
6603 &mut self,
6604 workflow_type: impl Into<String>,
6605 query_name: impl Into<String>,
6606 handler: F,
6607 ) where
6608 S: Clone + Send + Sync + 'static,
6609 F: Fn(QueryContext, Arc<S>, AvroValue) -> Fut + Send + Sync + 'static,
6610 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6611 {
6612 let handler = Arc::new(handler);
6613 let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
6614 let state = state.downcast::<S>().map_err(|_| {
6615 "registered query state type does not match the replayed workflow state".to_string()
6616 })?;
6617 Ok(Box::pin(handler(ctx, state, args)))
6618 });
6619
6620 self.queries
6621 .entry(workflow_type.into())
6622 .or_default()
6623 .insert(
6624 query_name.into(),
6625 RegisteredQuery::Replayed {
6626 state_type: TypeId::of::<S>(),
6627 handler: erased_handler,
6628 },
6629 );
6630 }
6631
6632 pub fn register_update<F, Fut>(
6634 &mut self,
6635 workflow_type: impl Into<String>,
6636 update_name: impl Into<String>,
6637 handler: F,
6638 ) where
6639 F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
6640 Fut: Future<Output = Result<Value>> + Send + 'static,
6641 {
6642 let handler = Arc::new(handler);
6643 self.updates
6644 .entry(workflow_type.into())
6645 .or_default()
6646 .insert(
6647 update_name.into(),
6648 Arc::new(move |ctx, args| {
6649 let handler = Arc::clone(&handler);
6650 Box::pin(async move {
6651 let result = handler(ctx, args.into_json()?).await?;
6652 AvroValue::from_serialize(&result)
6653 })
6654 }),
6655 );
6656 }
6657
6658 pub fn register_update_avro_value<F, Fut>(
6660 &mut self,
6661 workflow_type: impl Into<String>,
6662 update_name: impl Into<String>,
6663 handler: F,
6664 ) where
6665 F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
6666 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6667 {
6668 self.updates
6669 .entry(workflow_type.into())
6670 .or_default()
6671 .insert(
6672 update_name.into(),
6673 Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
6674 );
6675 }
6676
6677 pub async fn register(&self) -> Result<RegisterWorkerResponse> {
6678 let mut command_contracts = serde_json::Map::new();
6679 for workflow_type in self.workflows.keys() {
6680 let mut queries = self
6681 .queries
6682 .get(workflow_type)
6683 .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
6684 .unwrap_or_default();
6685 queries.sort();
6686 let mut updates = self
6687 .updates
6688 .get(workflow_type)
6689 .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
6690 .unwrap_or_default();
6691 updates.sort();
6692 command_contracts.insert(
6693 workflow_type.clone(),
6694 json!({
6695 "queries": queries,
6696 "query_contracts": [],
6697 "signals": [],
6698 "signal_contracts": [],
6699 "updates": updates,
6700 "update_contracts": [],
6701 "update_validators": [],
6702 }),
6703 );
6704 }
6705
6706 self.client
6707 .register_worker_with_definition_fingerprints(
6708 &self.worker_id,
6709 &self.task_queue,
6710 self.workflows.keys().cloned().collect(),
6711 self.activities.keys().cloned().collect(),
6712 self.max_concurrent_workflow_tasks,
6713 self.max_concurrent_activity_tasks,
6714 [
6715 Some(CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY.to_string()),
6716 Some(DURABLE_SELECTION_CAPABILITY.to_string()),
6717 Some(MEMO_UPSERTS_CAPABILITY.to_string()),
6718 Some(TYPED_SEARCH_ATTRIBUTES_CAPABILITY.to_string()),
6719 (!self.queries.is_empty()).then(|| QUERY_TASKS_CAPABILITY.to_string()),
6720 (!self.updates.is_empty()).then(|| WORKFLOW_UPDATES_CAPABILITY.to_string()),
6721 worker_protocol_supports_message_streams(WORKER_PROTOCOL_VERSION)
6722 .then(|| MESSAGE_STREAMS_CAPABILITY.to_string()),
6723 ]
6724 .into_iter()
6725 .flatten()
6726 .collect(),
6727 Value::Object(command_contracts),
6728 Some(
6729 self.workflows
6730 .iter()
6731 .filter_map(|(workflow_type, workflow)| {
6732 workflow
6733 .definition_fingerprint
6734 .as_ref()
6735 .map(|fingerprint| (workflow_type.clone(), fingerprint.clone()))
6736 })
6737 .collect(),
6738 ),
6739 )
6740 .await
6741 }
6742
6743 pub async fn run(&self) -> Result<()> {
6749 self.run_until(std::future::pending::<()>()).await
6750 }
6751
6752 pub async fn run_until<F>(&self, shutdown: F) -> Result<()>
6756 where
6757 F: Future<Output = ()>,
6758 {
6759 let stop = Arc::new(AtomicBool::new(false));
6760 let _stop_on_drop = StopWorkerOnDrop(Arc::clone(&stop));
6761 let worker = self.with_storage_admission(Arc::clone(&stop));
6762 let run = worker.run_with_storage_admission(Arc::clone(&stop));
6763 tokio::pin!(run);
6764 tokio::pin!(shutdown);
6765 tokio::select! {
6766 result = &mut run => result,
6767 _ = &mut shutdown => {
6768 stop.store(true, Ordering::SeqCst);
6769 run.await
6770 }
6771 }
6772 }
6773
6774 fn with_storage_admission(&self, stop: Arc<AtomicBool>) -> Self {
6775 let mut worker = self.clone();
6776 worker.client.worker_storage_admission = Some(WorkerStorageAdmission {
6777 policy: self.retry_policy,
6778 stop,
6779 });
6780 worker
6781 }
6782
6783 async fn run_with_storage_admission(&self, stop: Arc<AtomicBool>) -> Result<()> {
6784 let registration = self.register().await?;
6785 if !registration.registered {
6786 return Err(Error::WorkerLoop(format!(
6787 "worker registration for {:?} was not accepted",
6788 self.worker_id
6789 )));
6790 }
6791 let registered_worker_id = registration.worker_id.clone();
6792 let primary = self.run_registered_until(stop, registration).await;
6793 let deregistration = self
6794 .client
6795 .deregister_worker_registration(®istered_worker_id)
6796 .await;
6797
6798 match (primary, deregistration) {
6799 (Ok(()), Ok(_)) => Ok(()),
6800 (Ok(()), Err(deregistration)) => Err(deregistration),
6801 (Err(primary), Ok(_)) => Err(primary),
6802 (Err(primary), Err(deregistration)) => Err(Error::WorkerShutdown {
6803 primary: Box::new(primary),
6804 deregistration: Box::new(deregistration),
6805 }),
6806 }
6807 }
6808
6809 async fn run_registered_until(
6810 &self,
6811 stop: Arc<AtomicBool>,
6812 registration: RegisterWorkerResponse,
6813 ) -> Result<()> {
6814 let heartbeat_interval = Duration::from_secs(
6815 registration
6816 .heartbeat_interval_seconds
6817 .unwrap_or(self.heartbeat_interval.as_secs().max(1)),
6818 );
6819 let heartbeat = tokio::time::sleep(Duration::ZERO);
6825 tokio::pin!(heartbeat);
6826 let mut workflow_poller = (!self.workflows.is_empty()).then(|| {
6830 let worker = self.clone();
6831 let stop = Arc::clone(&stop);
6832 tokio::spawn(async move { worker.poll_workflows_until_stopped(stop).await })
6833 });
6834 let mut activity_poller = (!self.activities.is_empty()).then(|| {
6835 let worker = self.clone();
6836 let stop = Arc::clone(&stop);
6837 tokio::spawn(async move { worker.poll_activities_until_stopped(stop).await })
6838 });
6839 let mut query_poller = (!self.queries.is_empty()).then(|| {
6840 let worker = self.clone();
6841 let stop = Arc::clone(&stop);
6842 tokio::spawn(async move { worker.poll_queries_until_stopped(stop).await })
6843 });
6844
6845 loop {
6846 tokio::select! {
6847 _ = wait_for_worker_stop(&stop) => {
6848 stop.store(true, Ordering::SeqCst);
6849 break;
6850 }
6851 _ = &mut heartbeat => {
6852 let result = self.retry_worker_operation(|| {
6853 self.client.heartbeat_worker(
6854 &self.worker_id,
6855 self.max_concurrent_workflow_tasks,
6856 self.max_concurrent_activity_tasks,
6857 )
6858 }).await;
6859 heartbeat
6860 .as_mut()
6861 .reset(tokio::time::Instant::now() + heartbeat_interval);
6862 match result {
6863 Ok(acknowledgement) => {
6864 if let Some(observer) = &self.heartbeat_observer {
6865 observer(&WorkerHeartbeatObservation {
6866 worker_id: self.worker_id.clone(),
6867 task_queue: self.task_queue.clone(),
6868 acknowledged_at_unix_millis: SystemTime::now()
6869 .duration_since(UNIX_EPOCH)
6870 .unwrap_or_default()
6871 .as_millis()
6872 .min(u64::MAX as u128)
6873 as u64,
6874 acknowledgement,
6875 });
6876 }
6877 }
6878 Err(error) => {
6879 stop.store(true, Ordering::SeqCst);
6880 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await?;
6881 return Err(error);
6882 }
6883 }
6884 }
6885 result = OptionFuture::from(workflow_poller.as_mut()), if workflow_poller.is_some() => {
6886 workflow_poller = None;
6887 let stopped_by_server = stop.load(Ordering::SeqCst);
6888 stop.store(true, Ordering::SeqCst);
6889 let poller_result = optional_poller_result("workflow", result);
6890 let join_result =
6891 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6892 poller_result?;
6893 join_result?;
6894 if stopped_by_server {
6895 return Ok(());
6896 }
6897 return Err(Error::WorkerLoop(
6898 "workflow poller stopped unexpectedly".to_string(),
6899 ));
6900 }
6901 result = OptionFuture::from(activity_poller.as_mut()), if activity_poller.is_some() => {
6902 activity_poller = None;
6903 let stopped_by_server = stop.load(Ordering::SeqCst);
6904 stop.store(true, Ordering::SeqCst);
6905 let poller_result = optional_poller_result("activity", result);
6906 let join_result =
6907 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6908 poller_result?;
6909 join_result?;
6910 if stopped_by_server {
6911 return Ok(());
6912 }
6913 return Err(Error::WorkerLoop(
6914 "activity poller stopped unexpectedly".to_string(),
6915 ));
6916 }
6917 result = OptionFuture::from(query_poller.as_mut()), if query_poller.is_some() => {
6918 query_poller = None;
6919 let stopped_by_server = stop.load(Ordering::SeqCst);
6920 stop.store(true, Ordering::SeqCst);
6921 let poller_result = optional_poller_result("query", result);
6922 let join_result =
6923 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6924 poller_result?;
6925 join_result?;
6926 if stopped_by_server {
6927 return Ok(());
6928 }
6929 return Err(Error::WorkerLoop(
6930 "query poller stopped unexpectedly".to_string(),
6931 ));
6932 }
6933 }
6934 }
6935
6936 join_pollers(
6937 workflow_poller.take(),
6938 activity_poller.take(),
6939 query_poller.take(),
6940 )
6941 .await
6942 }
6943
6944 pub async fn run_once(&self) -> Result<usize> {
6958 let worker = self.with_storage_admission(Arc::new(AtomicBool::new(false)));
6959 let mut handled = 0;
6960 match worker.poll_workflow_once().await? {
6961 ManagedPollOutcome::Handled => handled += 1,
6962 ManagedPollOutcome::Stop => return Ok(handled),
6963 ManagedPollOutcome::Idle => {}
6964 }
6965 match worker.poll_activity_once().await? {
6966 ManagedPollOutcome::Handled => handled += 1,
6967 ManagedPollOutcome::Stop => return Ok(handled),
6968 ManagedPollOutcome::Idle => {}
6969 }
6970 if !self.queries.is_empty() {
6971 match worker.poll_query_once().await? {
6972 ManagedPollOutcome::Handled => handled += 1,
6973 ManagedPollOutcome::Stop => return Ok(handled),
6974 ManagedPollOutcome::Idle => {}
6975 }
6976 }
6977 Ok(handled)
6978 }
6979
6980 async fn poll_workflow_once(&self) -> Result<ManagedPollOutcome> {
6981 let poll_request_id = unique_request_id("rust-workflow-poll");
6982 let response = self
6983 .retry_worker_operation(|| {
6984 self.client.poll_workflow_task_response_with_request_id(
6985 &self.worker_id,
6986 &self.task_queue,
6987 self.poll_timeout,
6988 &poll_request_id,
6989 0,
6990 )
6991 })
6992 .await;
6993 let Some(response) = self.settle_worker_poll_response(response).await? else {
6994 return Ok(ManagedPollOutcome::Idle);
6995 };
6996 if response.outcome().should_stop() {
6997 return Ok(ManagedPollOutcome::Stop);
6998 }
6999 let memo_updates_supported =
7000 runtime_supports_workflow_memo_updates(response.server_capabilities.as_ref());
7001 let Some(task) = response.task else {
7002 return Ok(ManagedPollOutcome::Idle);
7003 };
7004
7005 let task_id = task.task_id.clone();
7006 let attempt = task.workflow_task_attempt;
7007 let run_id = task.run_id.clone();
7008 let lease_owner = task
7009 .lease_owner
7010 .clone()
7011 .unwrap_or_else(|| self.worker_id.clone());
7012
7013 match self.execute_workflow_task_decision(task) {
7014 Ok(decision)
7015 if commands_use_workflow_memo_updates(&decision.commands)
7016 && !memo_updates_supported =>
7017 {
7018 self.client
7019 .fail_workflow_task(
7020 &task_id,
7021 &lease_owner,
7022 attempt,
7023 Error::WorkflowMemoUpdatesUnavailable.to_string(),
7024 )
7025 .await?;
7026 }
7027 Ok(decision) if decision.commands.is_empty() => {
7028 self.client
7034 .fail_workflow_task_with_type(
7035 &task_id,
7036 &lease_owner,
7037 attempt,
7038 WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE,
7039 WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE,
7040 )
7041 .await?;
7042 }
7043 Ok(decision) => {
7044 let completion = self
7045 .client
7046 .complete_workflow_task_with_message_streams(
7047 &task_id,
7048 &lease_owner,
7049 attempt,
7050 decision.commands,
7051 decision.message_stream_cursors,
7052 decision.message_stream_waits,
7053 )
7054 .await;
7055 if let Err(error) = completion {
7056 if !workflow_task_completion_is_terminal_timeout(
7057 &error,
7058 &task_id,
7059 attempt,
7060 run_id.as_deref(),
7061 ) {
7062 return Err(error);
7063 }
7064 }
7065 }
7066 Err(error) => {
7067 self.client
7068 .fail_workflow_task(&task_id, &lease_owner, attempt, error.to_string())
7069 .await?;
7070 }
7071 }
7072
7073 Ok(ManagedPollOutcome::Handled)
7074 }
7075
7076 async fn poll_workflows_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
7077 while !stop.load(Ordering::SeqCst) {
7078 if self.poll_workflow_once().await? == ManagedPollOutcome::Stop {
7079 stop.store(true, Ordering::SeqCst);
7080 break;
7081 }
7082 }
7083
7084 Ok(())
7085 }
7086
7087 async fn poll_activity_once(&self) -> Result<ManagedPollOutcome> {
7088 let poll_request_id = unique_request_id("rust-activity-poll");
7089 let response = self
7090 .retry_worker_operation(|| {
7091 self.client.poll_activity_task_response_with_request_id(
7092 &self.worker_id,
7093 &self.task_queue,
7094 self.poll_timeout,
7095 &poll_request_id,
7096 0,
7097 )
7098 })
7099 .await;
7100 let Some(response) = self.settle_worker_poll_response(response).await? else {
7101 return Ok(ManagedPollOutcome::Idle);
7102 };
7103 if response.outcome().should_stop() {
7104 return Ok(ManagedPollOutcome::Stop);
7105 }
7106 let Some(task) = response.task else {
7107 return Ok(ManagedPollOutcome::Idle);
7108 };
7109
7110 let task_id = task.task_id.clone();
7111 let attempt_id = task
7112 .activity_attempt_id
7113 .clone()
7114 .or(task.attempt_id.clone())
7115 .unwrap_or_default();
7116 let lease_owner = task
7117 .lease_owner
7118 .clone()
7119 .unwrap_or_else(|| self.worker_id.clone());
7120 let codec = task.payload_codec.clone();
7121 let result = self.execute_activity_task(task).await;
7122 match result {
7123 Err(error) if worker_storage_admission_body(&error).is_some() => return Err(error),
7124 Ok(value) => {
7125 let completion = self
7126 .client
7127 .complete_activity_task(&task_id, &attempt_id, &lease_owner, value, &codec)
7128 .await;
7129 if let Err(error) = completion {
7130 if !activity_task_rejection_is_final(&error) {
7131 return Err(error);
7132 }
7133 }
7134 }
7135 Err(error) => {
7136 let failure = self
7137 .client
7138 .fail_activity_task(
7139 &task_id,
7140 &attempt_id,
7141 &lease_owner,
7142 error.to_string(),
7143 false,
7144 )
7145 .await;
7146 if let Err(error) = failure {
7147 if !activity_task_rejection_is_final(&error) {
7148 return Err(error);
7149 }
7150 }
7151 }
7152 }
7153
7154 Ok(ManagedPollOutcome::Handled)
7155 }
7156
7157 async fn poll_activities_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
7158 while !stop.load(Ordering::SeqCst) {
7159 if self.poll_activity_once().await? == ManagedPollOutcome::Stop {
7160 stop.store(true, Ordering::SeqCst);
7161 break;
7162 }
7163 }
7164
7165 Ok(())
7166 }
7167
7168 async fn poll_query_once(&self) -> Result<ManagedPollOutcome> {
7169 let poll_request_id = unique_request_id("rust-query-poll");
7170 let response = self
7171 .retry_worker_operation(|| {
7172 self.client.poll_query_task_response_with_request_id(
7173 &self.worker_id,
7174 &self.task_queue,
7175 self.poll_timeout,
7176 &poll_request_id,
7177 0,
7178 )
7179 })
7180 .await;
7181 let Some(response) = self.settle_worker_poll_response(response).await? else {
7182 return Ok(ManagedPollOutcome::Idle);
7183 };
7184 if response.outcome().should_stop() {
7185 return Ok(ManagedPollOutcome::Stop);
7186 }
7187 let Some(task) = response.task else {
7188 return Ok(ManagedPollOutcome::Idle);
7189 };
7190
7191 let query_task_id = task.query_task_id.clone();
7192 let attempt = task.query_task_attempt;
7193 let lease_owner = task
7194 .lease_owner
7195 .clone()
7196 .unwrap_or_else(|| self.worker_id.clone());
7197 let codec = task.payload_codec.clone();
7198
7199 match self.execute_query_task(task).await {
7200 Ok(value) => {
7201 let result_envelope = match encode_typed_envelope(&value, &codec) {
7202 Ok(result_envelope) => result_envelope,
7203 Err(error) => {
7204 let failure = self
7205 .client
7206 .fail_query_task(
7207 &query_task_id,
7208 &lease_owner,
7209 attempt,
7210 error.to_string(),
7211 "query_result_encode_failed",
7212 "QueryResultEncodeFailed",
7213 )
7214 .await;
7215 if let Err(error) = failure {
7216 if !query_task_rejection_is_final(&error) {
7217 return Err(error);
7218 }
7219 }
7220 return Ok(ManagedPollOutcome::Handled);
7221 }
7222 };
7223
7224 if let Err(error) = self
7225 .client
7226 .complete_query_task_with_envelope(
7227 &query_task_id,
7228 &lease_owner,
7229 attempt,
7230 value.clone().into_json()?,
7231 result_envelope,
7232 )
7233 .await
7234 {
7235 if !query_task_rejection_is_final(&error) {
7236 return Err(error);
7237 }
7238 }
7239 }
7240 Err(failure) => {
7241 let result = self
7242 .client
7243 .fail_query_task(
7244 &query_task_id,
7245 &lease_owner,
7246 attempt,
7247 failure.message,
7248 failure.reason,
7249 failure.failure_type,
7250 )
7251 .await;
7252 if let Err(error) = result {
7253 if !query_task_rejection_is_final(&error) {
7254 return Err(error);
7255 }
7256 }
7257 }
7258 }
7259
7260 Ok(ManagedPollOutcome::Handled)
7261 }
7262
7263 async fn poll_queries_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
7264 while !stop.load(Ordering::SeqCst) {
7265 if self.poll_query_once().await? == ManagedPollOutcome::Stop {
7266 stop.store(true, Ordering::SeqCst);
7267 break;
7268 }
7269 }
7270
7271 Ok(())
7272 }
7273
7274 async fn retry_worker_operation<T, F, Fut>(&self, mut operation: F) -> Result<T>
7275 where
7276 F: FnMut() -> Fut,
7277 Fut: Future<Output = Result<T>>,
7278 {
7279 let mut retries = 0;
7280
7281 loop {
7282 match operation().await {
7283 Err(error)
7284 if worker_operation_is_retryable(&error)
7285 && retries < self.retry_policy.max_retries =>
7286 {
7287 retries += 1;
7288 tokio::time::sleep(worker_retry_delay(self.retry_policy, retries)).await;
7289 }
7290 result => return result,
7291 }
7292 }
7293 }
7294
7295 async fn settle_worker_poll_response<T>(&self, response: Result<T>) -> Result<Option<T>> {
7296 match response {
7297 Ok(response) => Ok(Some(response)),
7298 Err(error) => {
7299 let Some(advertised_delay) = worker_poll_capacity_retry_after(&error) else {
7300 return Err(error);
7301 };
7302 let minimum_delay = self
7303 .retry_policy
7304 .initial_backoff
7305 .max(Duration::from_millis(1));
7306 let maximum_delay = self.retry_policy.max_backoff.max(minimum_delay);
7307 tokio::time::sleep(advertised_delay.max(minimum_delay).min(maximum_delay)).await;
7308 Ok(None)
7309 }
7310 }
7311 }
7312
7313 async fn execute_query_task(
7314 &self,
7315 mut task: QueryTask,
7316 ) -> std::result::Result<AvroValue, QueryTaskExecutionFailure> {
7317 validate_query_task_payloads(&task).map_err(|error| {
7318 QueryTaskExecutionFailure::new(
7319 "query_payload_decode_failed",
7320 error.to_string(),
7321 "QueryPayloadDecodeFailed",
7322 )
7323 })?;
7324
7325 if !self.workflows.contains_key(&task.workflow_type) {
7326 return Err(QueryTaskExecutionFailure::new(
7327 "query_workflow_type_not_registered",
7328 format!("no workflow registered for type {:?}", task.workflow_type),
7329 "WorkflowTypeNotRegistered",
7330 ));
7331 }
7332
7333 let Some(handlers) = self.queries.get(&task.workflow_type) else {
7334 return Err(QueryTaskExecutionFailure::new(
7335 "query_handler_unavailable",
7336 format!(
7337 "query handlers are unavailable for workflow type {:?}",
7338 task.workflow_type
7339 ),
7340 "QueryHandlerUnavailable",
7341 ));
7342 };
7343 let Some(query) = handlers.get(&task.query_name) else {
7344 return Err(QueryTaskExecutionFailure::new(
7345 "rejected_unknown_query",
7346 format!("unknown query {:?}", task.query_name),
7347 "QueryFailed",
7348 ));
7349 };
7350
7351 let args = decode_task_avro_arguments(task.query_arguments.as_ref(), &task.payload_codec)
7352 .map_err(|error| {
7353 QueryTaskExecutionFailure::new(
7354 "query_payload_decode_failed",
7355 format!("cannot decode query arguments: {error}"),
7356 "QueryPayloadDecodeFailed",
7357 )
7358 })?;
7359 let workflow_input_typed =
7360 decode_task_avro_arguments(task.workflow_arguments.as_ref(), &task.payload_codec)
7361 .map_err(|error| {
7362 QueryTaskExecutionFailure::new(
7363 "query_workflow_state_unavailable",
7364 format!("cannot decode workflow start input: {error}"),
7365 "QueryWorkflowStateUnavailable",
7366 )
7367 })?;
7368 let workflow_input = workflow_input_typed.clone().into_json().map_err(|error| {
7369 QueryTaskExecutionFailure::new(
7370 "query_workflow_state_unavailable",
7371 format!("cannot project workflow start input: {error}"),
7372 "QueryWorkflowStateUnavailable",
7373 )
7374 })?;
7375 hydrate_query_history_from_export(&mut task).map_err(|error| {
7376 QueryTaskExecutionFailure::new(
7377 "query_workflow_state_unavailable",
7378 format!("cannot restore query history snapshot: {error}"),
7379 "QueryWorkflowStateUnavailable",
7380 )
7381 })?;
7382 enrich_query_history_from_export(&mut task).map_err(|error| {
7383 QueryTaskExecutionFailure::new(
7384 "query_workflow_state_unavailable",
7385 format!("cannot restore compact query history payloads: {error}"),
7386 "QueryWorkflowStateUnavailable",
7387 )
7388 })?;
7389 let signal_events = query_signal_events(&task).map_err(|error| {
7390 QueryTaskExecutionFailure::new(
7391 "query_workflow_state_unavailable",
7392 format!("cannot decode committed workflow signals: {error}"),
7393 "QueryWorkflowStateUnavailable",
7394 )
7395 })?;
7396 let history_events = Arc::new(std::mem::take(&mut task.history_events));
7397 let context = QueryContext {
7398 workflow_id: task.workflow_id,
7399 run_id: task.run_id,
7400 workflow_type: task.workflow_type.clone(),
7401 run_status: task.run_status,
7402 workflow_input,
7403 workflow_input_avro_value: workflow_input_typed.clone(),
7404 history_events: Arc::clone(&history_events),
7405 signal_events: Arc::new(signal_events),
7406 };
7407
7408 let future = match query {
7409 RegisteredQuery::Snapshot(handler) => handler(context, args),
7410 RegisteredQuery::Replayed {
7411 state_type,
7412 handler,
7413 } => {
7414 let workflow = self
7415 .workflows
7416 .get(&task.workflow_type)
7417 .expect("workflow registration was checked above");
7418 if workflow.state_type != Some(*state_type) {
7419 return Err(QueryTaskExecutionFailure::new(
7420 "query_workflow_state_unavailable",
7421 "replayed query state type does not match its workflow registration",
7422 "QueryWorkflowStateUnavailable",
7423 ));
7424 }
7425 let replay = workflow.replay.as_ref().ok_or_else(|| {
7426 QueryTaskExecutionFailure::new(
7427 "query_workflow_state_unavailable",
7428 format!(
7429 "workflow type {:?} is not registered for instance-state replay",
7430 task.workflow_type
7431 ),
7432 "QueryWorkflowStateUnavailable",
7433 )
7434 })?;
7435 let workflow_state = Arc::new(Mutex::new(
7436 WorkflowState::new_with_identity(
7437 history_events.as_ref().clone(),
7438 context.workflow_id.clone(),
7439 context.run_id.clone(),
7440 self.task_queue.clone(),
7441 task.payload_codec,
7442 None,
7443 )
7444 .map_err(|error| {
7445 QueryTaskExecutionFailure::new(
7446 "query_workflow_state_unavailable",
7447 format!("workflow replay failed before query: {error}"),
7448 "QueryWorkflowStateUnavailable",
7449 )
7450 })?,
7451 ));
7452 let workflow_context = WorkflowContext {
7453 state: workflow_state,
7454 };
7455 let mut invocation = replay(workflow_context.clone(), workflow_input_typed.clone());
7456 let mut cx = TaskContext::from_waker(noop_waker_ref());
7457 match invocation.future.as_mut().poll(&mut cx) {
7458 Poll::Ready(Ok(_)) => {
7459 workflow_context
7460 .ensure_history_consumed()
7461 .map_err(|error| {
7462 QueryTaskExecutionFailure::new(
7463 "query_workflow_state_unavailable",
7464 format!("workflow replay failed before query: {error}"),
7465 "QueryWorkflowStateUnavailable",
7466 )
7467 })?;
7468 }
7469 Poll::Ready(Err(error)) => {
7470 return Err(QueryTaskExecutionFailure::new(
7471 "query_workflow_state_unavailable",
7472 format!("workflow replay failed before query: {error}"),
7473 "QueryWorkflowStateUnavailable",
7474 ));
7475 }
7476 Poll::Pending => {
7477 let commands = workflow_context.take_commands().map_err(|error| {
7478 QueryTaskExecutionFailure::new(
7479 "query_workflow_state_unavailable",
7480 format!("workflow replay failed before query: {error}"),
7481 "QueryWorkflowStateUnavailable",
7482 )
7483 })?;
7484 if commands.is_empty()
7485 && !workflow_context
7486 .matched_recorded_pending()
7487 .map_err(|error| {
7488 QueryTaskExecutionFailure::new(
7489 "query_workflow_state_unavailable",
7490 format!("workflow replay failed before query: {error}"),
7491 "QueryWorkflowStateUnavailable",
7492 )
7493 })?
7494 {
7495 return Err(QueryTaskExecutionFailure::new(
7496 "query_workflow_state_unavailable",
7497 "workflow replay yielded without a durable command",
7498 "QueryWorkflowStateUnavailable",
7499 ));
7500 }
7501 }
7502 }
7503 let state = (invocation.snapshot)().map_err(|error| {
7504 QueryTaskExecutionFailure::new(
7505 "query_workflow_state_unavailable",
7506 format!("cannot snapshot replayed workflow state: {error}"),
7507 "QueryWorkflowStateUnavailable",
7508 )
7509 })?;
7510 handler(context, state, args).map_err(|message| {
7511 QueryTaskExecutionFailure::new(
7512 "query_workflow_state_unavailable",
7513 message,
7514 "QueryWorkflowStateUnavailable",
7515 )
7516 })?
7517 }
7518 };
7519
7520 future.await.map_err(|error| {
7521 QueryTaskExecutionFailure::new("query_rejected", error.to_string(), "QueryFailed")
7522 })
7523 }
7524
7525 #[cfg(test)]
7526 fn execute_workflow_task(&self, task: WorkflowTask) -> Result<Vec<Value>> {
7527 Ok(self.execute_workflow_task_decision(task)?.commands)
7528 }
7529
7530 fn execute_workflow_task_decision(&self, task: WorkflowTask) -> Result<WorkflowTaskDecision> {
7531 validate_workflow_task_payloads(&task)?;
7532
7533 if let Some(update_id) = task
7534 .workflow_update_id
7535 .as_deref()
7536 .filter(|update_id| !update_id.is_empty())
7537 {
7538 return self
7539 .execute_update_task(&task, update_id)
7540 .map(WorkflowTaskDecision::without_message_streams);
7541 }
7542
7543 let workflow = self
7544 .workflows
7545 .get(&task.workflow_type)
7546 .ok_or_else(|| Error::WorkflowNotRegistered(task.workflow_type.clone()))?;
7547 let input = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
7548 let resume_signal = decode_resume_signal(&task)?;
7549 let history_budget = WorkflowHistoryBudget {
7550 event_count: task
7551 .total_history_events
7552 .unwrap_or_else(|| u64::try_from(task.history_events.len()).unwrap_or(u64::MAX)),
7553 size_bytes: task.history_size_bytes,
7554 continue_as_new_recommended: task.continue_as_new_recommended.unwrap_or(false),
7555 pressure: task.history_budget_pressure.clone(),
7556 };
7557 let workflow_command_identity = task
7558 .workflow_command_id
7559 .clone()
7560 .filter(|identity| !identity.is_empty())
7561 .unwrap_or_default();
7562 let mut workflow_state = WorkflowState::new_with_identity(
7563 task.history_events,
7564 task.workflow_id,
7565 task.run_id,
7566 self.task_queue.clone(),
7567 task.payload_codec.clone(),
7568 resume_signal,
7569 )?;
7570 workflow_state.history_budget = history_budget;
7571 workflow_state.workflow_command_identity = workflow_command_identity;
7572 workflow_state.cancel_requested = task.cancel_requested;
7573 let state = Arc::new(Mutex::new(workflow_state));
7574 let ctx = WorkflowContext { state };
7575 let mut future = (workflow.execute)(ctx.clone(), input);
7576 let mut cx = TaskContext::from_waker(noop_waker_ref());
7577
7578 match future.as_mut().poll(&mut cx) {
7579 Poll::Ready(Ok(result)) => {
7580 ctx.ensure_history_consumed()?;
7581 let result = encode_typed_envelope(&result, &task.payload_codec)?;
7582 let mut commands = ctx.take_commands()?;
7583 commands.push(json!({
7584 "type": "complete_workflow",
7585 "result": result
7586 }));
7587 self.message_stream_decision(&ctx, commands)
7588 }
7589 Poll::Ready(Err(error)) => {
7590 if let Error::ContinueAsNew(request) = error {
7591 let mut commands = ctx.take_commands()?;
7592 if let Some(command) = ctx.continue_as_new_command(request)? {
7593 commands.push(command);
7594 }
7595 ctx.ensure_history_consumed()?;
7596 return self.message_stream_decision(&ctx, commands);
7597 }
7598 if workflow_task_integrity_error(&error) {
7599 return Err(error);
7604 }
7605 ctx.ensure_history_consumed()?;
7608 let mut commands = ctx.take_commands()?;
7609 commands.push(workflow_failure_command(
7610 &error,
7611 recorded_activity_failure_boundary(&ctx, &error),
7612 ));
7613 self.message_stream_decision(&ctx, commands)
7614 }
7615 Poll::Pending => {
7616 let commands = ctx.take_commands()?;
7617 if commands.is_empty() && !ctx.matched_recorded_pending()? {
7618 Err(Error::WorkflowYieldedWithoutCommand)
7619 } else {
7620 self.message_stream_decision(&ctx, commands)
7621 }
7622 }
7623 }
7624 }
7625
7626 fn message_stream_decision(
7627 &self,
7628 ctx: &WorkflowContext,
7629 commands: Vec<Value>,
7630 ) -> Result<WorkflowTaskDecision> {
7631 let (message_stream_cursors, message_stream_waits) = ctx.message_stream_metadata()?;
7632 Ok(WorkflowTaskDecision {
7633 commands,
7634 message_stream_cursors,
7635 message_stream_waits,
7636 })
7637 }
7638
7639 fn execute_update_task(&self, task: &WorkflowTask, update_id: &str) -> Result<Vec<Value>> {
7640 if !self.workflows.contains_key(&task.workflow_type) {
7641 return Err(Error::WorkflowNotRegistered(task.workflow_type.clone()));
7642 }
7643
7644 let accepted = task.history_events.iter().rev().find_map(|event| {
7645 (event.event_type == "UpdateAccepted"
7646 && event.payload.get("update_id").and_then(Value::as_str) == Some(update_id))
7647 .then_some(&event.payload)
7648 });
7649 let update_name = accepted
7650 .and_then(|payload| payload.get("update_name"))
7651 .and_then(Value::as_str)
7652 .or(task.update_name.as_deref())
7653 .unwrap_or_default();
7654 let Some(handler) = self
7655 .updates
7656 .get(&task.workflow_type)
7657 .and_then(|handlers| handlers.get(update_name))
7658 else {
7659 return Ok(vec![json!({
7660 "type": "fail_update",
7661 "update_id": update_id,
7662 "message": format!(
7663 "no update handler is registered for {}.{update_name}",
7664 task.workflow_type
7665 ),
7666 "exception_type": "UnknownUpdate",
7667 "non_retryable": true,
7668 })]);
7669 };
7670 let arguments = accepted
7671 .and_then(|payload| payload.get("arguments"))
7672 .or(task.arguments.as_ref());
7673 let arguments = decode_task_avro_arguments(arguments, &task.payload_codec)?;
7674 let context = QueryContext {
7675 workflow_id: task.workflow_id.clone(),
7676 run_id: task.run_id.clone(),
7677 workflow_type: task.workflow_type.clone(),
7678 run_status: Some("running".to_string()),
7679 workflow_input: Value::Null,
7680 workflow_input_avro_value: AvroValue::Null,
7681 history_events: Arc::new(task.history_events.clone()),
7682 signal_events: Arc::new(Vec::new()),
7683 };
7684 let mut future = handler(context, arguments);
7685 let mut cx = TaskContext::from_waker(noop_waker_ref());
7686
7687 match future.as_mut().poll(&mut cx) {
7688 Poll::Ready(Ok(result)) => Ok(vec![json!({
7689 "type": "complete_update",
7690 "update_id": update_id,
7691 "result": encode_typed_envelope(&result, &task.payload_codec)?,
7692 })]),
7693 Poll::Ready(Err(error)) => Ok(vec![json!({
7694 "type": "fail_update",
7695 "update_id": update_id,
7696 "message": error.to_string(),
7697 "exception_type": "UpdateFailed",
7698 "non_retryable": true,
7699 })]),
7700 Poll::Pending => Err(Error::WorkflowYieldedWithoutCommand),
7701 }
7702 }
7703
7704 async fn execute_activity_task(&self, task: ActivityTask) -> Result<AvroValue> {
7705 validate_activity_task_payloads(&task)?;
7706
7707 let handler = self
7708 .activities
7709 .get(&task.activity_type)
7710 .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?;
7711 let args = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
7712 let attempt_id = task
7713 .activity_attempt_id
7714 .clone()
7715 .or(task.attempt_id.clone())
7716 .unwrap_or_default();
7717 let lease_owner = task
7718 .lease_owner
7719 .clone()
7720 .unwrap_or_else(|| self.worker_id.clone());
7721 let ctx = ActivityContext {
7722 client: self.client.clone(),
7723 task_id: task.task_id,
7724 activity_attempt_id: attempt_id,
7725 lease_owner,
7726 activity_type: task.activity_type,
7727 attempt_number: task.attempt_number,
7728 task_queue: self.task_queue.clone(),
7729 worker_id: self.worker_id.clone(),
7730 };
7731
7732 handler(ctx, args).await
7733 }
7734}
7735
7736fn poller_result(
7737 kind: &str,
7738 result: std::result::Result<Result<()>, tokio::task::JoinError>,
7739) -> Result<()> {
7740 match result {
7741 Ok(result) => result,
7742 Err(error) => Err(Error::WorkerLoop(format!(
7743 "{kind} poller join error: {error}"
7744 ))),
7745 }
7746}
7747
7748fn optional_poller_result(
7749 kind: &str,
7750 result: Option<std::result::Result<Result<()>, tokio::task::JoinError>>,
7751) -> Result<()> {
7752 match result {
7753 Some(result) => poller_result(kind, result),
7754 None => Ok(()),
7755 }
7756}
7757
7758async fn join_pollers(
7759 workflow_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7760 activity_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7761 query_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7762) -> Result<()> {
7763 let mut first_error = None;
7764
7765 if let Some(handle) = workflow_poller {
7766 if let Err(error) = poller_result("workflow", handle.await) {
7767 first_error.get_or_insert(error);
7768 }
7769 }
7770
7771 if let Some(handle) = activity_poller {
7772 if let Err(error) = poller_result("activity", handle.await) {
7773 first_error.get_or_insert(error);
7774 }
7775 }
7776
7777 if let Some(handle) = query_poller {
7778 if let Err(error) = poller_result("query", handle.await) {
7779 first_error.get_or_insert(error);
7780 }
7781 }
7782
7783 if let Some(error) = first_error {
7784 Err(error)
7785 } else {
7786 Ok(())
7787 }
7788}
7789
7790fn default_worker_id() -> String {
7791 let millis = SystemTime::now()
7792 .duration_since(UNIX_EPOCH)
7793 .unwrap_or_default()
7794 .as_millis();
7795 format!("rust-worker-{}-{millis}", std::process::id())
7796}
7797
7798fn percent_encode_path_segment(segment: &str) -> String {
7799 const HEX: &[u8; 16] = b"0123456789ABCDEF";
7800 let mut encoded = String::with_capacity(segment.len());
7801
7802 for byte in segment.bytes() {
7803 if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') {
7804 encoded.push(char::from(byte));
7805 } else {
7806 encoded.push('%');
7807 encoded.push(char::from(HEX[(byte >> 4) as usize]));
7808 encoded.push(char::from(HEX[(byte & 0x0f) as usize]));
7809 }
7810 }
7811
7812 encoded
7813}
7814
7815fn unique_request_id(prefix: &str) -> String {
7816 let nanos = SystemTime::now()
7817 .duration_since(UNIX_EPOCH)
7818 .unwrap_or_default()
7819 .as_nanos();
7820 format!("{prefix}-{}-{nanos}", std::process::id())
7821}
7822
7823#[derive(Debug)]
7824struct QueryTaskExecutionFailure {
7825 reason: String,
7826 message: String,
7827 failure_type: String,
7828}
7829
7830impl QueryTaskExecutionFailure {
7831 fn new(
7832 reason: impl Into<String>,
7833 message: impl Into<String>,
7834 failure_type: impl Into<String>,
7835 ) -> Self {
7836 Self {
7837 reason: reason.into(),
7838 message: message.into(),
7839 failure_type: failure_type.into(),
7840 }
7841 }
7842}
7843
7844#[derive(Clone, Debug)]
7850pub struct WorkflowInstance<S> {
7851 state: Arc<Mutex<S>>,
7852}
7853
7854impl<S> WorkflowInstance<S> {
7855 fn new(state: S) -> Self {
7856 Self {
7857 state: Arc::new(Mutex::new(state)),
7858 }
7859 }
7860
7861 pub fn read<R>(&self, reader: impl FnOnce(&S) -> R) -> Result<R> {
7863 let state = self
7864 .state
7865 .lock()
7866 .map_err(|_| Error::WorkflowStatePoisoned)?;
7867 Ok(reader(&state))
7868 }
7869
7870 pub fn update<R>(&self, transition: impl FnOnce(&mut S) -> R) -> Result<R> {
7872 let mut state = self
7873 .state
7874 .lock()
7875 .map_err(|_| Error::WorkflowStatePoisoned)?;
7876 Ok(transition(&mut state))
7877 }
7878}
7879
7880impl<S: Clone> WorkflowInstance<S> {
7881 fn snapshot(&self) -> Result<S> {
7882 self.read(Clone::clone)
7883 }
7884}
7885
7886#[derive(Clone, Debug, PartialEq)]
7887pub struct MessageStreamMessage {
7888 pub stream_name: String,
7889 pub message_id: String,
7890 pub position: u64,
7891 pub arguments: Vec<AvroValue>,
7892}
7893
7894#[derive(Clone, Debug)]
7895pub struct MessageStream {
7896 ctx: WorkflowContext,
7897 name: String,
7898}
7899
7900impl MessageStream {
7901 pub async fn receive(&self, max_items: usize) -> Result<Vec<MessageStreamMessage>> {
7903 if !(1..=MESSAGE_STREAM_MAX_BATCH).contains(&max_items) {
7904 return Err(Error::Codec(format!(
7905 "message stream max_items must be between 1 and {MESSAGE_STREAM_MAX_BATCH}"
7906 )));
7907 }
7908 loop {
7909 if let Some(batch) = self.ctx.take_message_stream_batch(&self.name, max_items)? {
7910 return Ok(batch);
7911 }
7912
7913 self.ctx.record_message_stream_wait(&self.name)?;
7914 let replay_wait_sequence = self.ctx.next_message_stream_wait_sequence()?;
7915 let arguments = self.ctx.wait_runtime_signal(MESSAGE_STREAM_SIGNAL).await?;
7916 self.ctx.buffer_message_stream_delivery(arguments)?;
7917 if let Some(sequence) = replay_wait_sequence {
7918 self.ctx.buffer_message_stream_history_for_wait(sequence)?;
7919 }
7920 }
7921 }
7922
7923 pub async fn receive_one(&self) -> Result<MessageStreamMessage> {
7924 self.receive(1)
7925 .await?
7926 .into_iter()
7927 .next()
7928 .ok_or_else(|| Error::Codec("message stream resumed without a message".to_string()))
7929 }
7930}
7931
7932#[derive(Clone, Debug)]
7933pub struct WorkflowContext {
7934 state: Arc<Mutex<WorkflowState>>,
7935}
7936
7937fn valid_memo_key(key: &str) -> bool {
7938 let numeric_candidate = key.strip_prefix('-').unwrap_or(key);
7939
7940 !key.is_empty()
7941 && key.len() <= 64
7942 && (numeric_candidate.is_empty()
7943 || !numeric_candidate.bytes().all(|byte| byte.is_ascii_digit()))
7944 && key
7945 .bytes()
7946 .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b':' | b'-'))
7947}
7948
7949fn avro_encoded_size(value: &AvroValue) -> Result<usize> {
7950 BASE64
7951 .decode(encode_avro_value(value)?.blob)
7952 .map(|bytes| bytes.len())
7953 .map_err(|error| Error::Codec(format!("memo Avro encoding was not strict base64: {error}")))
7954}
7955
7956fn canonical_memo_entries(value: AvroValue, require_entries: bool) -> Result<AvroValue> {
7957 let AvroValue::Map(entries) = value else {
7958 return Err(Error::InvalidMemoUpdate(
7959 "entries must serialize to an Avro string-keyed map".to_string(),
7960 ));
7961 };
7962 if require_entries && entries.is_empty() {
7963 return Err(Error::InvalidMemoUpdate(
7964 "at least one entry is required".to_string(),
7965 ));
7966 }
7967 if entries.len() > MAX_MEMO_ENTRIES {
7968 return Err(Error::InvalidMemoUpdate(format!(
7969 "at most {MAX_MEMO_ENTRIES} entries are allowed"
7970 )));
7971 }
7972
7973 for (key, value) in &entries {
7974 if !valid_memo_key(&key) {
7975 return Err(Error::InvalidMemoUpdate(
7976 "keys must match ^(?!-?[0-9]+$)[A-Za-z0-9_.:-]{1,64}$".to_string(),
7977 ));
7978 }
7979 if avro_encoded_size(value)? > MAX_MEMO_VALUE_SIZE_BYTES {
7980 return Err(Error::InvalidMemoUpdate(format!(
7981 "value {key:?} exceeds the {MAX_MEMO_VALUE_SIZE_BYTES}-byte limit"
7982 )));
7983 }
7984 }
7985
7986 let value = AvroValue::Map(entries);
7987 if avro_encoded_size(&value)? > MAX_MEMO_TOTAL_SIZE_BYTES {
7988 return Err(Error::InvalidMemoUpdate(format!(
7989 "update exceeds the {MAX_MEMO_TOTAL_SIZE_BYTES}-byte total limit"
7990 )));
7991 }
7992 Ok(value)
7993}
7994
7995fn decode_memo_history_map(envelope: &Value, require_entries: bool) -> Result<AvroValue> {
7996 let object = envelope.as_object().ok_or_else(|| {
7997 Error::InvalidMemoUpdate(
7998 "history field must use the public {codec, blob} payload envelope".to_string(),
7999 )
8000 })?;
8001 if object.len() != 2 || !object.contains_key("codec") || !object.contains_key("blob") {
8002 return Err(Error::InvalidMemoUpdate(
8003 "history field must use exactly the public {codec, blob} payload envelope".to_string(),
8004 ));
8005 }
8006
8007 canonical_memo_entries(
8008 decode_wire_avro_value(envelope, DEFAULT_CODEC)?,
8009 require_entries,
8010 )
8011}
8012
8013impl WorkflowContext {
8014 pub fn message_stream(&self, name: impl Into<String>) -> Result<MessageStream> {
8015 let name = name.into();
8016 if name.is_empty()
8017 || name.len() > 128
8018 || !name.bytes().all(|byte| {
8019 byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b':' | b'-')
8020 })
8021 {
8022 return Err(Error::Codec(
8023 "message stream names must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens"
8024 .to_string(),
8025 ));
8026 }
8027 Ok(MessageStream {
8028 ctx: self.clone(),
8029 name,
8030 })
8031 }
8032
8033 fn record_message_stream_wait(&self, name: &str) -> Result<()> {
8034 let mut state = self
8035 .state
8036 .lock()
8037 .map_err(|_| Error::WorkflowStatePoisoned)?;
8038 let position = state.message_stream_cursors.get(name).copied().unwrap_or(0);
8039 state
8040 .message_stream_waits
8041 .insert(name.to_string(), position);
8042 Ok(())
8043 }
8044
8045 fn buffer_message_stream(&self, message: MessageStreamMessage) -> Result<()> {
8046 let mut state = self
8047 .state
8048 .lock()
8049 .map_err(|_| Error::WorkflowStatePoisoned)?;
8050 let cursor = state
8051 .message_stream_cursors
8052 .get(&message.stream_name)
8053 .copied()
8054 .unwrap_or(0);
8055 if message.position <= cursor {
8056 return Ok(());
8057 }
8058 let pending = state
8059 .message_stream_messages
8060 .entry(message.stream_name.clone())
8061 .or_default();
8062 if pending.iter().any(|candidate| {
8063 candidate.position == message.position || candidate.message_id == message.message_id
8064 }) {
8065 return Ok(());
8066 }
8067 pending.push(message);
8068 pending.sort_by_key(|candidate| candidate.position);
8069 Ok(())
8070 }
8071
8072 fn buffer_message_stream_delivery(&self, arguments: Vec<Value>) -> Result<Option<String>> {
8073 if let Some(delivery) = decode_message_stream_delivery(arguments)? {
8074 match delivery {
8075 MessageStreamDelivery::Message(message) => {
8076 let stream_name = message.stream_name.clone();
8077 self.buffer_message_stream(message)?;
8078 return Ok(Some(stream_name));
8079 }
8080 MessageStreamDelivery::Cursor {
8081 stream_name,
8082 through_position,
8083 } => self.apply_message_stream_cursor(&stream_name, through_position)?,
8084 }
8085 }
8086 Ok(None)
8087 }
8088
8089 fn next_message_stream_wait_sequence(&self) -> Result<Option<u64>> {
8090 let state = self
8091 .state
8092 .lock()
8093 .map_err(|_| Error::WorkflowStatePoisoned)?;
8094 Ok(match state.recorded_commands.get(state.command_cursor) {
8095 Some(RecordedCommand::SignalWait {
8096 sequence,
8097 signal_name,
8098 ..
8099 }) if signal_name == MESSAGE_STREAM_SIGNAL => Some(*sequence),
8100 _ => None,
8101 })
8102 }
8103
8104 fn buffer_message_stream_history_for_wait(&self, wait_sequence: u64) -> Result<()> {
8105 let (history, payload_codec) = {
8106 let state = self
8107 .state
8108 .lock()
8109 .map_err(|_| Error::WorkflowStatePoisoned)?;
8110 (
8111 Arc::clone(&state.history_events),
8112 state.payload_codec.clone(),
8113 )
8114 };
8115
8116 let Some(opened_index) = history.iter().position(|event| {
8117 event.event_type == "SignalWaitOpened"
8118 && durable_event_sequence(event) == Some(wait_sequence)
8119 && event.payload.get("signal_name").and_then(Value::as_str)
8120 == Some(MESSAGE_STREAM_SIGNAL)
8121 }) else {
8122 return Ok(());
8123 };
8124 let boundary_index = history
8125 .iter()
8126 .enumerate()
8127 .skip(opened_index + 1)
8128 .find_map(|(index, event)| {
8129 (durable_event_sequence(event).is_some_and(|sequence| sequence > wait_sequence)
8130 && is_authored_command_open_event(event))
8131 .then_some(index)
8132 })
8133 .unwrap_or(history.len());
8134
8135 for event in history[opened_index + 1..boundary_index]
8136 .iter()
8137 .filter(|event| {
8138 event.event_type == "SignalReceived"
8139 && event.payload.get("signal_name").and_then(Value::as_str)
8140 == Some(MESSAGE_STREAM_SIGNAL)
8141 })
8142 {
8143 let arguments = decode_signal_event_arguments(event, &payload_codec)?
8144 .into_iter()
8145 .map(AvroValue::into_json)
8146 .collect::<Result<Vec<_>>>()?;
8147 self.buffer_message_stream_delivery(arguments)?;
8148 }
8149 Ok(())
8150 }
8151
8152 fn apply_message_stream_cursor(&self, name: &str, through_position: u64) -> Result<()> {
8153 let mut state = self
8154 .state
8155 .lock()
8156 .map_err(|_| Error::WorkflowStatePoisoned)?;
8157 let cursor = state
8158 .message_stream_cursors
8159 .entry(name.to_string())
8160 .or_default();
8161 *cursor = (*cursor).max(through_position);
8162 if let Some(pending) = state.message_stream_messages.get_mut(name) {
8163 pending.retain(|message| message.position > through_position);
8164 }
8165 Ok(())
8166 }
8167
8168 fn take_message_stream_batch(
8169 &self,
8170 name: &str,
8171 max_items: usize,
8172 ) -> Result<Option<Vec<MessageStreamMessage>>> {
8173 let mut state = self
8174 .state
8175 .lock()
8176 .map_err(|_| Error::WorkflowStatePoisoned)?;
8177 let cursor = state.message_stream_cursors.get(name).copied().unwrap_or(0);
8178 let pending = state
8179 .message_stream_messages
8180 .entry(name.to_string())
8181 .or_default();
8182 let count = contiguous_message_stream_count(pending, cursor, max_items);
8183 if count == 0 {
8184 return Ok(None);
8185 }
8186 let batch = pending.drain(..count).collect::<Vec<_>>();
8187 let position = batch.last().map(|message| message.position).unwrap_or(0);
8188 state
8189 .message_stream_cursors
8190 .insert(name.to_string(), position);
8191 state.message_stream_waits.remove(name);
8192 Ok(Some(batch))
8193 }
8194
8195 fn message_stream_metadata(&self) -> Result<(Vec<Value>, Vec<Value>)> {
8196 let state = self
8197 .state
8198 .lock()
8199 .map_err(|_| Error::WorkflowStatePoisoned)?;
8200 let mut cursors = state.message_stream_cursors.iter().collect::<Vec<_>>();
8201 cursors.sort_by_key(|(name, _)| *name);
8202 let mut waits = state.message_stream_waits.iter().collect::<Vec<_>>();
8203 waits.sort_by_key(|(name, _)| *name);
8204 Ok((
8205 cursors
8206 .into_iter()
8207 .map(|(name, position)| json!({"stream_name": name, "through_position": position}))
8208 .collect(),
8209 waits
8210 .into_iter()
8211 .map(|(name, position)| json!({"stream_name": name, "after_position": position}))
8212 .collect(),
8213 ))
8214 }
8215 pub fn workflow_identity(&self) -> Result<WorkflowIdentity> {
8217 let state = self
8218 .state
8219 .lock()
8220 .map_err(|_| Error::WorkflowStatePoisoned)?;
8221 Ok(WorkflowIdentity {
8222 workflow_id: state.workflow_id.clone(),
8223 run_id: state.run_id.clone(),
8224 })
8225 }
8226
8227 pub fn history_budget(&self) -> Result<WorkflowHistoryBudget> {
8229 let state = self
8230 .state
8231 .lock()
8232 .map_err(|_| Error::WorkflowStatePoisoned)?;
8233 Ok(state.history_budget.clone())
8234 }
8235
8236 pub fn continue_as_new<T: Serialize>(&self, args: T) -> Result<Value> {
8242 self.continue_as_new_with_options(ContinueAsNewOptions::new(), args)
8243 }
8244
8245 pub fn continue_as_new_with_options<T: Serialize>(
8247 &self,
8248 options: ContinueAsNewOptions,
8249 args: T,
8250 ) -> Result<Value> {
8251 options.validate()?;
8252 Err(Error::ContinueAsNew(ContinueAsNewRequest {
8253 arguments: normalize_avro_arguments(AvroValue::from_serialize(&args)?),
8254 options,
8255 }))
8256 }
8257
8258 pub fn activity<T: Serialize>(
8259 &self,
8260 activity_type: impl Into<String>,
8261 args: T,
8262 ) -> ActivityCall {
8263 self.activity_with_options(activity_type, ActivityOptions::new(), args)
8264 }
8265
8266 pub fn activity_on_queue<T, Q>(
8267 &self,
8268 activity_type: impl Into<String>,
8269 task_queue: Option<Q>,
8270 args: T,
8271 ) -> ActivityCall
8272 where
8273 T: Serialize,
8274 Q: Into<String>,
8275 {
8276 let mut options = ActivityOptions::new();
8277 options.task_queue = task_queue.map(Into::into);
8278 self.activity_with_options(activity_type, options, args)
8279 }
8280
8281 pub fn activity_with_options<T: Serialize>(
8319 &self,
8320 activity_type: impl Into<String>,
8321 options: ActivityOptions,
8322 args: T,
8323 ) -> ActivityCall {
8324 ActivityCall {
8325 ctx: self.clone(),
8326 activity_type: activity_type.into(),
8327 options,
8328 args: Some(AvroValue::from_serialize(&args)),
8329 scheduled: false,
8330 parallel_group_path: Vec::new(),
8331 }
8332 }
8333
8334 pub async fn activity_avro_value<T: Serialize>(
8335 &self,
8336 activity_type: impl Into<String>,
8337 args: T,
8338 ) -> Result<AvroValue> {
8339 let mut call = self.activity(activity_type, args);
8340 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8341 }
8342
8343 pub async fn activity_avro_value_with_options<T: Serialize>(
8344 &self,
8345 activity_type: impl Into<String>,
8346 options: ActivityOptions,
8347 args: T,
8348 ) -> Result<AvroValue> {
8349 let mut call = self.activity_with_options(activity_type, options, args);
8350 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8351 }
8352
8353 pub async fn activity_typed<I, O>(&self, activity_type: impl Into<String>, args: I) -> Result<O>
8355 where
8356 I: Serialize,
8357 O: DeserializeOwned,
8358 {
8359 self.activity_typed_with_options(activity_type, ActivityOptions::new(), args)
8360 .await
8361 }
8362
8363 pub async fn activity_typed_with_options<I, O>(
8369 &self,
8370 activity_type: impl Into<String>,
8371 options: ActivityOptions,
8372 args: I,
8373 ) -> Result<O>
8374 where
8375 I: Serialize,
8376 O: DeserializeOwned,
8377 {
8378 let activity_type = activity_type.into();
8379 let encoded = AvroValue::from_serialize(&args).map_err(|error| {
8380 handler_type_error::<I>(
8381 HandlerKind::Activity,
8382 &activity_type,
8383 HandlerValueKind::Input,
8384 error.to_string(),
8385 )
8386 });
8387 let mut call = ActivityCall {
8388 ctx: self.clone(),
8389 activity_type: activity_type.clone(),
8390 options,
8391 args: Some(encoded),
8392 scheduled: false,
8393 parallel_group_path: Vec::new(),
8394 };
8395 let result = std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await?;
8396 decode_handler_result(result, HandlerKind::Activity, &activity_type)
8397 }
8398
8399 pub fn parallel(&self, operations: Vec<ParallelOperation>) -> ParallelCall {
8406 ParallelCall::new(self.clone(), operations)
8407 }
8408
8409 pub fn join(&self, operations: Vec<ParallelOperation>) -> ParallelCall {
8411 self.parallel(operations)
8412 }
8413
8414 pub async fn parallel_avro_value(
8416 &self,
8417 operations: Vec<ParallelOperation>,
8418 ) -> Result<Vec<ParallelAvroResult>> {
8419 let mut call = self.parallel(operations);
8420 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8421 }
8422
8423 pub fn select(&self, operations: Vec<ParallelOperation>) -> SelectCall {
8426 let operations = operations
8427 .into_iter()
8428 .enumerate()
8429 .map(|(index, operation)| (SelectionKey::Index(index), operation))
8430 .collect();
8431 SelectCall::new(self.clone(), operations)
8432 }
8433
8434 pub fn select_keyed<K>(&self, operations: Vec<(K, ParallelOperation)>) -> SelectCall
8436 where
8437 K: Into<SelectionKey>,
8438 {
8439 SelectCall::new(
8440 self.clone(),
8441 operations
8442 .into_iter()
8443 .map(|(key, operation)| (key.into(), operation))
8444 .collect(),
8445 )
8446 }
8447
8448 pub fn saga(&self) -> Saga {
8450 Saga::new(self.clone())
8451 }
8452
8453 pub fn is_cancellation_requested(&self) -> Result<bool> {
8455 let state = self
8456 .state
8457 .lock()
8458 .map_err(|_| Error::WorkflowStatePoisoned)?;
8459 Ok(state.cancel_requested)
8460 }
8461
8462 pub fn throw_if_cancellation_requested(&self) -> Result<()> {
8467 if self.is_cancellation_requested()? {
8468 return Err(Error::WorkflowCancellationRequested(
8469 WorkflowCancellationRequested,
8470 ));
8471 }
8472 Ok(())
8473 }
8474
8475 pub fn wait_signal(&self, signal_name: impl Into<String>) -> SignalCall {
8476 SignalCall {
8477 ctx: self.clone(),
8478 signal_name: signal_name.into(),
8479 runtime_reserved_allowed: false,
8480 opened_wait: false,
8481 matched_pending: false,
8482 parallel_group_path: Vec::new(),
8483 }
8484 }
8485
8486 fn wait_runtime_signal(&self, signal_name: impl Into<String>) -> SignalCall {
8487 SignalCall {
8488 ctx: self.clone(),
8489 signal_name: signal_name.into(),
8490 runtime_reserved_allowed: true,
8491 opened_wait: false,
8492 matched_pending: false,
8493 parallel_group_path: Vec::new(),
8494 }
8495 }
8496
8497 pub async fn wait_signal_avro_value(
8498 &self,
8499 signal_name: impl Into<String>,
8500 ) -> Result<Vec<AvroValue>> {
8501 let mut call = self.wait_signal(signal_name);
8502 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8503 }
8504
8505 pub fn signals(&self, signal_name: &str) -> Result<Vec<Vec<Value>>> {
8511 self.signals_avro_value(signal_name)?
8512 .into_iter()
8513 .map(|arguments| {
8514 arguments
8515 .into_iter()
8516 .map(AvroValue::into_json)
8517 .collect::<Result<Vec<_>>>()
8518 })
8519 .collect()
8520 }
8521
8522 pub fn signals_avro_value(&self, signal_name: &str) -> Result<Vec<Vec<AvroValue>>> {
8524 let state = self
8525 .state
8526 .lock()
8527 .map_err(|_| Error::WorkflowStatePoisoned)?;
8528 state
8529 .history_events
8530 .iter()
8531 .filter(|event| {
8532 event.event_type == "SignalReceived"
8533 && event.payload.get("signal_name").and_then(Value::as_str) == Some(signal_name)
8534 })
8535 .map(|event| decode_signal_event_arguments(event, &state.payload_codec))
8536 .collect()
8537 }
8538
8539 pub fn updates(&self, update_name: &str) -> Result<Vec<Vec<Value>>> {
8545 self.updates_avro_value(update_name)?
8546 .into_iter()
8547 .map(|arguments| {
8548 arguments
8549 .into_iter()
8550 .map(AvroValue::into_json)
8551 .collect::<Result<Vec<_>>>()
8552 })
8553 .collect()
8554 }
8555
8556 pub fn updates_avro_value(&self, update_name: &str) -> Result<Vec<Vec<AvroValue>>> {
8558 let state = self
8559 .state
8560 .lock()
8561 .map_err(|_| Error::WorkflowStatePoisoned)?;
8562 let mut seen = Vec::new();
8563 let mut updates = Vec::new();
8564 for event in state.history_events.iter() {
8565 if !matches!(
8566 event.event_type.as_str(),
8567 "UpdateAccepted" | "UpdateApplied"
8568 ) || event.payload.get("update_name").and_then(Value::as_str) != Some(update_name)
8569 || event.payload.get("arguments").is_none()
8570 {
8571 continue;
8572 }
8573 if let Some(update_id) = event.payload.get("update_id").and_then(Value::as_str) {
8574 if seen.iter().any(|recorded| recorded == update_id) {
8575 continue;
8576 }
8577 seen.push(update_id.to_string());
8578 }
8579 updates.push(decode_update_event_arguments(event, &state.payload_codec)?);
8580 }
8581 Ok(updates)
8582 }
8583
8584 pub fn wait_condition<F>(
8592 &self,
8593 options: ConditionWaitOptions,
8594 predicate: F,
8595 ) -> ConditionWaitCall
8596 where
8597 F: Fn() -> Result<bool> + Send + 'static,
8598 {
8599 ConditionWaitCall {
8600 ctx: self.clone(),
8601 options,
8602 predicate: Box::new(predicate),
8603 occurrence_id: None,
8604 opened_wait: false,
8605 parallel_group_path: Vec::new(),
8606 }
8607 }
8608
8609 pub fn sleep(&self, duration: Duration) -> TimerCall {
8630 let delay_seconds = duration
8631 .as_secs()
8632 .checked_add(u64::from(duration.subsec_nanos() > 0));
8633 TimerCall {
8634 ctx: self.clone(),
8635 delay_seconds,
8636 scheduled: false,
8637 matched_pending: false,
8638 parallel_group_path: Vec::new(),
8639 }
8640 }
8641
8642 pub fn start_timer(&self, duration: Duration) -> TimerCall {
8644 self.sleep(duration)
8645 }
8646
8647 pub fn side_effect<T, F>(&self, callback: F) -> Result<T>
8654 where
8655 T: Serialize + DeserializeOwned,
8656 F: FnOnce() -> T,
8657 {
8658 {
8659 let mut state = self
8660 .state
8661 .lock()
8662 .map_err(|_| Error::WorkflowStatePoisoned)?;
8663 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8664 return match recorded {
8665 RecordedCommand::SideEffect { sequence, value } => {
8666 state.command_cursor += 1;
8667 value.deserialize().map_err(|error| {
8668 Error::NonDeterministicReplay(ReplayFailure::new(
8669 "side_effect_type_mismatch",
8670 Some(sequence),
8671 Some(std::any::type_name::<T>().to_string()),
8672 Some(error.to_string()),
8673 "recorded side-effect value is incompatible with the requested Rust type",
8674 ))
8675 })
8676 }
8677 other => Err(command_mismatch(&other, "side effect")),
8678 };
8679 }
8680 }
8681
8682 let value = callback();
8683 let avro_value = AvroValue::from_serialize(&value)?;
8684 let mut state = self
8685 .state
8686 .lock()
8687 .map_err(|_| Error::WorkflowStatePoisoned)?;
8688 let result = encode_typed_envelope(&avro_value, &state.payload_codec)?;
8689 state.commands.push(json!({
8690 "type": "record_side_effect",
8691 "result": result,
8692 }));
8693 Ok(value)
8694 }
8695
8696 pub fn side_effect_avro_value<F>(&self, callback: F) -> Result<AvroValue>
8698 where
8699 F: FnOnce() -> AvroValue,
8700 {
8701 {
8702 let mut state = self
8703 .state
8704 .lock()
8705 .map_err(|_| Error::WorkflowStatePoisoned)?;
8706 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8707 return match recorded {
8708 RecordedCommand::SideEffect { value, .. } => {
8709 state.command_cursor += 1;
8710 Ok(value)
8711 }
8712 other => Err(command_mismatch(&other, "side effect")),
8713 };
8714 }
8715 }
8716
8717 let value = callback();
8718 let mut state = self
8719 .state
8720 .lock()
8721 .map_err(|_| Error::WorkflowStatePoisoned)?;
8722 let result = encode_typed_envelope(&value, &state.payload_codec)?;
8723 state.commands.push(json!({
8724 "type": "record_side_effect",
8725 "result": result,
8726 }));
8727 Ok(value)
8728 }
8729
8730 pub fn append_workflow_stream(
8736 &self,
8737 stream_name: impl Into<String>,
8738 items: &[WorkflowStreamAppendItem],
8739 max_pending_items: Option<u64>,
8740 ) -> Result<()> {
8741 if items.is_empty() {
8742 return Err(Error::Codec(
8743 "workflow_stream_items_empty: append requires at least one item".to_string(),
8744 ));
8745 }
8746 if max_pending_items == Some(0) {
8747 return Err(Error::Codec(
8748 "workflow_stream_pending_limit_invalid: max_pending_items must be positive"
8749 .to_string(),
8750 ));
8751 }
8752 let stream_name = stream_name.into();
8753 if stream_name.is_empty() {
8754 return Err(Error::Codec(
8755 "workflow_stream_name_invalid: stream name must not be empty".to_string(),
8756 ));
8757 }
8758
8759 let mut state = self
8760 .state
8761 .lock()
8762 .map_err(|_| Error::WorkflowStatePoisoned)?;
8763 let command_ordinal = state.workflow_stream_command_counter;
8764 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8765 state.workflow_stream_command_counter += 1;
8766 return match recorded {
8767 RecordedCommand::SideEffect { .. } => {
8768 state.command_cursor += 1;
8769 Ok(())
8770 }
8771 other => Err(command_mismatch(&other, "workflow stream append")),
8772 };
8773 }
8774
8775 let identity = Self::workflow_stream_command_identity(&state)?.to_string();
8776 state.workflow_stream_command_counter += 1;
8777 let wire_items = items
8778 .iter()
8779 .enumerate()
8780 .map(|(item_index, item)| {
8781 item.wire_value(Some(format!(
8782 "dw-stream:{identity}:{command_ordinal}:{item_index}"
8783 )))
8784 })
8785 .collect::<Vec<_>>();
8786 let mut directive = json!({
8787 "operation": "append",
8788 "stream_name": stream_name,
8789 "command_identity": identity,
8790 "command_ordinal": command_ordinal,
8791 "items": wire_items,
8792 });
8793 if let Some(max_pending_items) = max_pending_items {
8794 directive["max_pending_items"] = json!(max_pending_items);
8795 }
8796 let result = encode_typed_envelope(&AvroValue::Null, &state.payload_codec)?;
8797 state.commands.push(json!({
8798 "type": "record_side_effect",
8799 "result": result,
8800 "workflow_stream": directive,
8801 }));
8802 Ok(())
8803 }
8804
8805 pub fn close_workflow_stream(
8807 &self,
8808 stream_name: impl Into<String>,
8809 retention_seconds: Option<u64>,
8810 ) -> Result<()> {
8811 self.finish_workflow_stream(stream_name.into(), None, retention_seconds)
8812 }
8813
8814 pub fn error_workflow_stream(
8816 &self,
8817 stream_name: impl Into<String>,
8818 error_reason: impl Into<String>,
8819 retention_seconds: Option<u64>,
8820 ) -> Result<()> {
8821 let error_reason = error_reason.into();
8822 if error_reason.is_empty() {
8823 return Err(Error::Codec(
8824 "workflow_stream_error_invalid: error reason must not be empty".to_string(),
8825 ));
8826 }
8827 self.finish_workflow_stream(stream_name.into(), Some(error_reason), retention_seconds)
8828 }
8829
8830 fn finish_workflow_stream(
8831 &self,
8832 stream_name: String,
8833 error_reason: Option<String>,
8834 retention_seconds: Option<u64>,
8835 ) -> Result<()> {
8836 if stream_name.is_empty() {
8837 return Err(Error::Codec(
8838 "workflow_stream_name_invalid: stream name must not be empty".to_string(),
8839 ));
8840 }
8841 if retention_seconds == Some(0) {
8842 return Err(Error::Codec(
8843 "workflow_stream_retention_invalid: retention_seconds must be positive".to_string(),
8844 ));
8845 }
8846 let mut state = self
8847 .state
8848 .lock()
8849 .map_err(|_| Error::WorkflowStatePoisoned)?;
8850 let command_ordinal = state.workflow_stream_command_counter;
8851 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8852 state.workflow_stream_command_counter += 1;
8853 return match recorded {
8854 RecordedCommand::SideEffect { .. } => {
8855 state.command_cursor += 1;
8856 Ok(())
8857 }
8858 other => Err(command_mismatch(&other, "workflow stream close")),
8859 };
8860 }
8861 let identity = Self::workflow_stream_command_identity(&state)?.to_string();
8862 state.workflow_stream_command_counter += 1;
8863 let mut directive = json!({
8864 "operation": if error_reason.is_some() { "error" } else { "close" },
8865 "stream_name": stream_name,
8866 "command_identity": identity,
8867 "command_ordinal": command_ordinal,
8868 });
8869 if let Some(error_reason) = error_reason {
8870 directive["error_reason"] = json!(error_reason);
8871 }
8872 if let Some(retention_seconds) = retention_seconds {
8873 directive["retention_seconds"] = json!(retention_seconds);
8874 }
8875 let result = encode_typed_envelope(&AvroValue::Null, &state.payload_codec)?;
8876 state.commands.push(json!({
8877 "type": "record_side_effect",
8878 "result": result,
8879 "workflow_stream": directive,
8880 }));
8881 Ok(())
8882 }
8883
8884 fn workflow_stream_command_identity(state: &WorkflowState) -> Result<&str> {
8885 let identity = state.workflow_command_identity.as_str();
8886 if identity.is_empty() {
8887 return Err(Error::MissingWorkflowCommandIdentity);
8888 }
8889 Ok(identity)
8890 }
8891
8892 pub fn upsert_search_attributes(&self, update: SearchAttributeUpdate) -> Result<()> {
8898 update.validate()?;
8899 let (attributes, attribute_types) = update.into_wire_parts();
8900 let mut state = self
8901 .state
8902 .lock()
8903 .map_err(|_| Error::WorkflowStatePoisoned)?;
8904
8905 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8906 return match recorded {
8907 RecordedCommand::SearchAttributes {
8908 sequence,
8909 attributes: recorded_attributes,
8910 attribute_types: recorded_attribute_types,
8911 } => {
8912 if recorded_attributes != attributes {
8913 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8914 "search_attribute_value_mismatch",
8915 Some(sequence),
8916 Some(recorded_attributes.to_string()),
8917 Some(attributes.to_string()),
8918 "search-attribute values differ from the recorded durable command",
8919 )));
8920 }
8921 if let RecordedSnapshotValue::Known(recorded_types) = recorded_attribute_types {
8922 if recorded_types != attribute_types {
8923 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8924 "search_attribute_type_mismatch",
8925 Some(sequence),
8926 Some(json!(recorded_types).to_string()),
8927 Some(json!(attribute_types).to_string()),
8928 "search-attribute declared types differ from the recorded durable command",
8929 )));
8930 }
8931 }
8932 state.command_cursor += 1;
8933 Ok(())
8934 }
8935 other => Err(command_mismatch(&other, "search-attribute update")),
8936 };
8937 }
8938
8939 let mut command = serde_json::Map::from_iter([
8940 ("type".to_string(), json!("upsert_search_attributes")),
8941 ("attributes".to_string(), attributes),
8942 ]);
8943 if !attribute_types.is_empty() {
8944 command.insert("attribute_types".to_string(), json!(attribute_types));
8945 }
8946 state.commands.push(Value::Object(command));
8947 Ok(())
8948 }
8949
8950 pub fn uuid_v4(&self) -> Result<Uuid> {
8952 self.side_effect(Uuid::new_v4)
8953 }
8954
8955 pub fn get_version(
8958 &self,
8959 change_id: impl Into<String>,
8960 min_supported: i32,
8961 max_supported: i32,
8962 ) -> Result<i32> {
8963 let change_id = change_id.into();
8964 if change_id.trim().is_empty() {
8965 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8966 "version_change_id_invalid",
8967 None,
8968 Some("non-empty change ID".to_string()),
8969 Some(change_id),
8970 "version markers require a stable non-empty change ID",
8971 )));
8972 }
8973 if min_supported > max_supported {
8974 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8975 "version_range_invalid",
8976 None,
8977 Some("min_supported <= max_supported".to_string()),
8978 Some(format!("{min_supported}..={max_supported}")),
8979 "version marker supported range is invalid",
8980 )));
8981 }
8982
8983 let mut state = self
8984 .state
8985 .lock()
8986 .map_err(|_| Error::WorkflowStatePoisoned)?;
8987 if let Some((version, sequence)) = state.version_markers.get(&change_id).copied() {
8988 ensure_version_supported(&change_id, version, min_supported, max_supported, sequence)?;
8989 return Ok(version);
8990 }
8991
8992 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8993 return match recorded {
8994 RecordedCommand::VersionMarker {
8995 sequence,
8996 change_id: recorded_change_id,
8997 version,
8998 ..
8999 } => {
9000 if recorded_change_id != change_id {
9001 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
9002 "version_change_id_mismatch",
9003 Some(sequence),
9004 Some(recorded_change_id),
9005 Some(change_id),
9006 "recorded version marker change ID differs from current workflow code",
9007 )));
9008 }
9009 ensure_version_supported(
9010 &change_id,
9011 version,
9012 min_supported,
9013 max_supported,
9014 sequence,
9015 )?;
9016 state.command_cursor += 1;
9017 state.version_markers.insert(change_id, (version, sequence));
9018 Ok(version)
9019 }
9020 other => Err(command_mismatch(
9021 &other,
9022 format!("version marker:{change_id}"),
9023 )),
9024 };
9025 }
9026
9027 let version = max_supported;
9028 state.commands.push(json!({
9029 "type": "record_version_marker",
9030 "change_id": change_id,
9031 "version": version,
9032 "min_supported": min_supported,
9033 "max_supported": max_supported,
9034 }));
9035 state.version_markers.insert(change_id, (version, 0));
9038 Ok(version)
9039 }
9040
9041 pub fn patched(&self, change_id: impl Into<String>) -> Result<bool> {
9043 Ok(self.get_version(change_id, -1, 1)? == 1)
9044 }
9045
9046 pub fn deprecate_patch(&self, change_id: impl Into<String>) -> Result<()> {
9048 self.get_version(change_id, -1, 1).map(|_| ())
9049 }
9050
9051 pub fn upsert_memo<T: Serialize>(&self, entries: T) -> Result<()> {
9056 let entries = canonical_memo_entries(AvroValue::from_serialize(&entries)?, true)?;
9057 let mut state = self
9058 .state
9059 .lock()
9060 .map_err(|_| Error::WorkflowStatePoisoned)?;
9061
9062 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
9063 return match recorded {
9064 RecordedCommand::Memo {
9065 sequence,
9066 entries: recorded_entries,
9067 } => {
9068 if recorded_entries != entries {
9069 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
9070 "memo_update_mismatch",
9071 Some(sequence),
9072 Some(format!("{recorded_entries:?}")),
9073 Some(format!("{entries:?}")),
9074 "recorded memo entries differ from the current workflow update",
9075 )));
9076 }
9077 state.command_cursor += 1;
9078 Ok(())
9079 }
9080 other => Err(command_mismatch(&other, "memo upsert")),
9081 };
9082 }
9083
9084 let entries_envelope = encode_typed_envelope(&entries, DEFAULT_CODEC)?;
9085 state.commands.push(json!({
9086 "type": "upsert_memo",
9087 "entries": entries_envelope,
9088 }));
9089 Ok(())
9090 }
9091
9092 pub fn start_child_workflow<T: Serialize>(
9119 &self,
9120 workflow_type: impl Into<String>,
9121 options: ChildWorkflowOptions,
9122 args: T,
9123 ) -> ChildWorkflowCall {
9124 ChildWorkflowCall {
9125 ctx: self.clone(),
9126 workflow_type: workflow_type.into(),
9127 options,
9128 args: Some(AvroValue::from_serialize(&args)),
9129 scheduled: false,
9130 matched_pending: false,
9131 parallel_group_path: Vec::new(),
9132 }
9133 }
9134
9135 pub async fn start_child_workflow_avro_value<T: Serialize>(
9136 &self,
9137 workflow_type: impl Into<String>,
9138 options: ChildWorkflowOptions,
9139 args: T,
9140 ) -> Result<ChildWorkflowAvroResult> {
9141 let mut call = self.start_child_workflow(workflow_type, options, args);
9142 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
9143 }
9144
9145 fn take_commands(&self) -> Result<Vec<Value>> {
9146 let mut state = self
9147 .state
9148 .lock()
9149 .map_err(|_| Error::WorkflowStatePoisoned)?;
9150 Ok(std::mem::take(&mut state.commands))
9151 }
9152
9153 fn continue_as_new_command(&self, request: ContinueAsNewRequest) -> Result<Option<Value>> {
9154 let mut state = self
9155 .state
9156 .lock()
9157 .map_err(|_| Error::WorkflowStatePoisoned)?;
9158
9159 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
9160 return Err(command_mismatch(&recorded, "continue as new"));
9161 }
9162 if state.recorded_continue_as_new_sequence.is_some() {
9163 state.continue_as_new_consumed = true;
9164 return Ok(None);
9165 }
9166
9167 let arguments = encode_typed_envelope(&request.arguments, &state.payload_codec)?;
9168 let mut command = serde_json::Map::from_iter([
9169 ("type".to_string(), json!("continue_as_new")),
9170 ("arguments".to_string(), arguments),
9171 ("queue".to_string(), json!(state.task_queue.clone())),
9172 ]);
9173 if let Some(workflow_type) = request.options.workflow_type {
9174 command.insert("workflow_type".to_string(), json!(workflow_type));
9175 }
9176 if let Some(task_queue) = request.options.task_queue {
9177 command.insert("queue".to_string(), json!(task_queue));
9178 }
9179 Ok(Some(Value::Object(command)))
9180 }
9181
9182 fn matched_recorded_pending(&self) -> Result<bool> {
9183 let state = self
9184 .state
9185 .lock()
9186 .map_err(|_| Error::WorkflowStatePoisoned)?;
9187 Ok(state.matched_recorded_pending)
9188 }
9189
9190 fn ensure_history_consumed(&self) -> Result<()> {
9191 let state = self
9192 .state
9193 .lock()
9194 .map_err(|_| Error::WorkflowStatePoisoned)?;
9195 if let Some(command) = state.recorded_commands.get(state.command_cursor) {
9196 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
9197 "recorded_commands_unconsumed",
9198 Some(command.sequence()),
9199 Some(command.shape().to_string()),
9200 Some("workflow completion".to_string()),
9201 "workflow completed before consuming all recorded durable commands",
9202 )));
9203 }
9204 if let Some(sequence) = state
9205 .recorded_continue_as_new_sequence
9206 .filter(|_| !state.continue_as_new_consumed)
9207 {
9208 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
9209 "recorded_continue_as_new_unconsumed",
9210 Some(sequence),
9211 Some("continue as new".to_string()),
9212 Some("workflow completion".to_string()),
9213 "workflow completed without consuming its recorded continue-as-new transition",
9214 )));
9215 }
9216 Ok(())
9217 }
9218}
9219
9220fn contiguous_message_stream_count(
9221 pending: &[MessageStreamMessage],
9222 cursor: u64,
9223 max_items: usize,
9224) -> usize {
9225 pending
9226 .iter()
9227 .take(max_items)
9228 .enumerate()
9229 .take_while(|(offset, message)| {
9230 u64::try_from(*offset)
9231 .ok()
9232 .and_then(|offset| cursor.checked_add(offset + 1))
9233 == Some(message.position)
9234 })
9235 .count()
9236}
9237
9238fn is_authored_command_open_event(event: &HistoryEvent) -> bool {
9239 matches!(
9240 event.event_type.as_str(),
9241 "ActivityScheduled"
9242 | "TimerScheduled"
9243 | "ChildWorkflowScheduled"
9244 | "SignalWaitOpened"
9245 | "ConditionWaitOpened"
9246 | "SearchAttributesUpserted"
9247 | "SideEffectRecorded"
9248 | "VersionMarkerRecorded"
9249 | "MemoUpserted"
9250 | "WorkflowContinuedAsNew"
9251 )
9252}
9253
9254#[derive(Debug)]
9255struct WorkflowState {
9256 workflow_id: Option<String>,
9257 run_id: Option<String>,
9258 task_queue: String,
9259 payload_codec: String,
9260 history_events: Arc<Vec<HistoryEvent>>,
9261 history_budget: WorkflowHistoryBudget,
9262 cancel_requested: bool,
9263 resume_signal: Option<ResumeSignal>,
9264 recorded_commands: Vec<RecordedCommand>,
9265 selection_markers: Vec<SelectionMarker>,
9266 selection_marker_cursor: usize,
9267 cancelled_selection_members: Vec<SelectionCancellation>,
9268 recorded_continue_as_new_sequence: Option<u64>,
9269 continue_as_new_consumed: bool,
9270 command_cursor: usize,
9271 condition_wait_occurrence_counter: u64,
9272 matched_recorded_pending: bool,
9273 version_markers: HashMap<String, (i32, u64)>,
9274 workflow_command_identity: String,
9275 workflow_stream_command_counter: u64,
9276 commands: Vec<Value>,
9277 message_stream_messages: HashMap<String, Vec<MessageStreamMessage>>,
9278 message_stream_cursors: HashMap<String, u64>,
9279 message_stream_waits: HashMap<String, u64>,
9280}
9281
9282impl WorkflowState {
9283 #[cfg(test)]
9284 fn new(
9285 history: Vec<HistoryEvent>,
9286 task_queue: String,
9287 payload_codec: String,
9288 resume_signal: Option<ResumeSignal>,
9289 ) -> Result<Self> {
9290 Self::new_with_identity(
9291 history,
9292 None,
9293 None,
9294 task_queue,
9295 payload_codec,
9296 resume_signal,
9297 )
9298 }
9299
9300 fn new_with_identity(
9301 history: Vec<HistoryEvent>,
9302 workflow_id: Option<String>,
9303 run_id: Option<String>,
9304 task_queue: String,
9305 payload_codec: String,
9306 resume_signal: Option<ResumeSignal>,
9307 ) -> Result<Self> {
9308 let recorded_commands = recorded_commands(
9309 &history,
9310 &payload_codec,
9311 WorkflowIdentity {
9312 workflow_id: workflow_id.clone(),
9313 run_id: run_id.clone(),
9314 },
9315 )?;
9316 let selection_markers = recorded_selection_markers(&history)?;
9317 let cancelled_selection_members = recorded_selection_cancellations(&history)?;
9318 let recorded_continue_as_new = history
9319 .iter()
9320 .filter(|event| event.event_type == "WorkflowContinuedAsNew")
9321 .collect::<Vec<_>>();
9322 if recorded_continue_as_new.len() > 1 {
9323 return Err(invalid_recorded_history(
9324 "duplicate_continue_as_new_transition",
9325 recorded_continue_as_new
9326 .last()
9327 .and_then(|event| durable_event_sequence(event))
9328 .unwrap_or(0),
9329 "one WorkflowContinuedAsNew event",
9330 &format!(
9331 "{} WorkflowContinuedAsNew events",
9332 recorded_continue_as_new.len()
9333 ),
9334 "workflow history records one continue-as-new transition more than once",
9335 ));
9336 }
9337 let recorded_continue_as_new_sequence = recorded_continue_as_new
9338 .first()
9339 .map(|event| {
9340 durable_event_sequence(event).ok_or_else(|| {
9341 Error::NonDeterministicReplay(ReplayFailure::new(
9342 "continue_as_new_sequence_missing",
9343 None,
9344 Some("recorded transition sequence".to_string()),
9345 Some("missing sequence".to_string()),
9346 "WorkflowContinuedAsNew history is missing its recorded sequence",
9347 ))
9348 })
9349 })
9350 .transpose()?;
9351 let mut message_stream_cursors = HashMap::new();
9352 for event in &history {
9353 if !matches!(
9354 event.event_type.as_str(),
9355 "SignalReceived" | "SignalApplied"
9356 ) || event.payload.get("signal_name").and_then(Value::as_str)
9357 != Some(MESSAGE_STREAM_SIGNAL)
9358 {
9359 continue;
9360 }
9361 let arguments = decode_signal_event_arguments(event, &payload_codec)?;
9362 if arguments.len() != 1 {
9363 continue;
9364 }
9365 let envelope = arguments[0].clone().into_json()?;
9366 let Some(envelope) = envelope.as_object() else {
9367 continue;
9368 };
9369 if envelope.get("schema").and_then(Value::as_str) != Some(MESSAGE_STREAM_CURSOR_SCHEMA)
9370 {
9371 continue;
9372 }
9373 let Some(stream_name) = envelope.get("stream_name").and_then(Value::as_str) else {
9374 continue;
9375 };
9376 let Some(through_position) = envelope.get("through_position").and_then(Value::as_u64)
9377 else {
9378 continue;
9379 };
9380 let cursor = message_stream_cursors
9381 .entry(stream_name.to_string())
9382 .or_insert(0);
9383 *cursor = (*cursor).max(through_position);
9384 }
9385 let event_count = u64::try_from(history.len()).unwrap_or(u64::MAX);
9386 let cancel_requested = history.iter().any(|event| {
9387 matches!(
9388 event.event_type.as_str(),
9389 "WorkflowCancellationRequested" | "WorkflowCancelRequested"
9390 )
9391 });
9392 Ok(Self {
9393 workflow_command_identity: String::new(),
9394 workflow_stream_command_counter: 0,
9395 workflow_id,
9396 run_id,
9397 task_queue,
9398 payload_codec,
9399 history_events: Arc::new(history),
9400 history_budget: WorkflowHistoryBudget {
9401 event_count,
9402 ..WorkflowHistoryBudget::default()
9403 },
9404 cancel_requested,
9405 resume_signal,
9406 recorded_commands,
9407 selection_markers,
9408 selection_marker_cursor: 0,
9409 cancelled_selection_members,
9410 recorded_continue_as_new_sequence,
9411 continue_as_new_consumed: false,
9412 command_cursor: 0,
9413 condition_wait_occurrence_counter: 0,
9414 matched_recorded_pending: false,
9415 version_markers: HashMap::new(),
9416 commands: Vec::new(),
9417 message_stream_messages: HashMap::new(),
9418 message_stream_cursors,
9419 message_stream_waits: HashMap::new(),
9420 })
9421 }
9422}
9423
9424enum MessageStreamDelivery {
9425 Message(MessageStreamMessage),
9426 Cursor {
9427 stream_name: String,
9428 through_position: u64,
9429 },
9430}
9431
9432fn decode_message_stream_delivery(arguments: Vec<Value>) -> Result<Option<MessageStreamDelivery>> {
9433 if arguments.len() != 1 {
9434 return Ok(None);
9435 }
9436 let envelope = arguments
9437 .into_iter()
9438 .next()
9439 .expect("one argument was checked");
9440 let Some(envelope) = envelope.as_object() else {
9441 return Ok(None);
9442 };
9443 let Some(stream_name) = envelope.get("stream_name").and_then(Value::as_str) else {
9444 return Ok(None);
9445 };
9446 if envelope.get("schema").and_then(Value::as_str) == Some(MESSAGE_STREAM_CURSOR_SCHEMA) {
9447 let Some(through_position) = envelope.get("through_position").and_then(Value::as_u64)
9448 else {
9449 return Ok(None);
9450 };
9451 return Ok(Some(MessageStreamDelivery::Cursor {
9452 stream_name: stream_name.to_string(),
9453 through_position,
9454 }));
9455 }
9456 if envelope.get("schema").and_then(Value::as_str) != Some(MESSAGE_STREAM_SCHEMA) {
9457 return Ok(None);
9458 }
9459 let Some(message_id) = envelope.get("message_id").and_then(Value::as_str) else {
9460 return Ok(None);
9461 };
9462 let Some(position) = envelope
9463 .get("position")
9464 .and_then(Value::as_u64)
9465 .filter(|value| *value > 0)
9466 else {
9467 return Ok(None);
9468 };
9469 let Some(payload_envelope) = envelope.get("payload_envelope") else {
9470 return Ok(None);
9471 };
9472 let Ok(payload_envelope) = serde_json::from_value::<PayloadEnvelope>(payload_envelope.clone())
9473 else {
9474 return Ok(None);
9475 };
9476 let decoded = decode_avro_value(&payload_envelope)?;
9477 let AvroValue::Array(values) = decoded else {
9478 return Ok(None);
9479 };
9480 Ok(Some(MessageStreamDelivery::Message(MessageStreamMessage {
9481 stream_name: stream_name.to_string(),
9482 message_id: message_id.to_string(),
9483 position,
9484 arguments: values,
9485 })))
9486}
9487
9488#[derive(Clone, Debug)]
9489enum RecordedCommand {
9490 Activity {
9491 sequence: u64,
9492 activity_type: Option<String>,
9493 options: Option<RecordedActivityOptions>,
9494 outcome: Option<ActivityOutcome>,
9495 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9496 },
9497 Timer {
9498 sequence: u64,
9499 delay_seconds: u64,
9500 fired: bool,
9501 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9502 },
9503 ChildWorkflow {
9504 sequence: u64,
9505 workflow_type: Option<String>,
9506 outcome: Option<ChildWorkflowOutcome>,
9507 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9508 },
9509 SignalWait {
9510 sequence: u64,
9511 signal_name: String,
9512 value: Option<Vec<AvroValue>>,
9513 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9514 },
9515 ConditionWait {
9516 sequence: u64,
9517 occurrence_id: String,
9518 condition_key: Option<String>,
9519 predicate_identity: String,
9520 timeout_seconds: Option<u64>,
9521 result: Option<ConditionWaitResult>,
9522 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9523 },
9524 SearchAttributes {
9525 sequence: u64,
9526 attributes: Value,
9527 attribute_types: RecordedSnapshotValue<BTreeMap<String, String>>,
9528 },
9529 SideEffect {
9530 sequence: u64,
9531 value: AvroValue,
9532 },
9533 VersionMarker {
9534 sequence: u64,
9535 change_id: String,
9536 version: i32,
9537 },
9538 Memo {
9539 sequence: u64,
9540 entries: AvroValue,
9541 },
9542}
9543
9544#[derive(Clone, Debug, PartialEq, Eq)]
9545struct SelectionMarker {
9546 selection_group_id: String,
9547 selection_group_base_sequence: u64,
9548 selection_group_size: usize,
9549 member_key: SelectionKey,
9550 member_index: usize,
9551 member_base_sequence: u64,
9552 member_size: usize,
9553 operation_kind: String,
9554 operation_identity: String,
9555 outcome: String,
9556 resolution_event_id: String,
9557 resolution_event_type: String,
9558}
9559
9560#[derive(Clone, Debug, PartialEq, Eq)]
9561struct SelectionCancellation {
9562 selection_group_id: String,
9563 member_key: SelectionKey,
9564 member_index: usize,
9565 member_base_sequence: u64,
9566 member_size: usize,
9567 operation_kind: String,
9568 operation_identity: String,
9569}
9570
9571fn recorded_selection_markers(events: &[HistoryEvent]) -> Result<Vec<SelectionMarker>> {
9572 let mut markers: Vec<SelectionMarker> = Vec::new();
9573 for event in events
9574 .iter()
9575 .filter(|event| event.event_type == "SelectionResolved")
9576 {
9577 let payload = &event.payload;
9578 let base_sequence = required_selection_u64(payload, "selection_group_base_sequence")?;
9579 let group_size = required_selection_usize(payload, "selection_group_size")?;
9580 let member_base_sequence = required_selection_u64(payload, "member_base_sequence")?;
9581 let member_size = required_selection_usize(payload, "member_size")?;
9582 let member_index = required_selection_usize_allow_zero(payload, "member_index")?;
9583 let group_id = payload_string(payload, "selection_group_id").ok_or_else(|| {
9584 invalid_recorded_history(
9585 "selection_marker_invalid",
9586 base_sequence,
9587 "non-empty selection_group_id",
9588 &payload.to_string(),
9589 "selection winner history is missing its durable group identity",
9590 )
9591 })?;
9592 let expected_group_id = format!("select-calls:{base_sequence}:{group_size}");
9593 if group_id != expected_group_id {
9594 return Err(invalid_recorded_history(
9595 "selection_marker_invalid",
9596 base_sequence,
9597 &expected_group_id,
9598 &group_id,
9599 "selection winner history contains an incompatible group identity",
9600 ));
9601 }
9602 let group_end = base_sequence
9603 .checked_add(u64::try_from(group_size).unwrap_or(u64::MAX))
9604 .unwrap_or(u64::MAX);
9605 let member_end = member_base_sequence
9606 .checked_add(u64::try_from(member_size).unwrap_or(u64::MAX))
9607 .unwrap_or(u64::MAX);
9608 if member_index >= group_size
9609 || member_base_sequence < base_sequence
9610 || member_end > group_end
9611 {
9612 return Err(invalid_recorded_history(
9613 "selection_marker_invalid",
9614 base_sequence,
9615 "winner member within selection group bounds",
9616 &payload.to_string(),
9617 "selection winner history contains an invalid member range",
9618 ));
9619 }
9620 let operation_kind = payload_string(payload, "operation_kind").ok_or_else(|| {
9621 invalid_recorded_history(
9622 "selection_marker_invalid",
9623 base_sequence,
9624 "selection operation kind",
9625 &payload.to_string(),
9626 "selection winner history is missing its operation kind",
9627 )
9628 })?;
9629 if !matches!(
9630 operation_kind.as_str(),
9631 "activity" | "child" | "timer" | "signal" | "condition" | "group"
9632 ) {
9633 return Err(invalid_recorded_history(
9634 "selection_marker_invalid",
9635 base_sequence,
9636 "activity, child, timer, signal, condition, or group",
9637 &operation_kind,
9638 "selection winner history contains an unsupported operation kind",
9639 ));
9640 }
9641 let operation_identity =
9642 payload_string(payload, "operation_identity").ok_or_else(|| {
9643 invalid_recorded_history(
9644 "selection_marker_invalid",
9645 base_sequence,
9646 "non-empty operation identity",
9647 &payload.to_string(),
9648 "selection winner history is missing its durable operation identity",
9649 )
9650 })?;
9651 let outcome = payload_string(payload, "outcome").ok_or_else(|| {
9652 invalid_recorded_history(
9653 "selection_marker_invalid",
9654 base_sequence,
9655 "completed or failed selection outcome",
9656 &payload.to_string(),
9657 "selection winner history is missing its outcome",
9658 )
9659 })?;
9660 if !matches!(outcome.as_str(), "completed" | "failed") {
9661 return Err(invalid_recorded_history(
9662 "selection_marker_invalid",
9663 base_sequence,
9664 "completed or failed selection outcome",
9665 &outcome,
9666 "selection winner history contains an unsupported outcome",
9667 ));
9668 }
9669 let marker = SelectionMarker {
9670 selection_group_id: group_id,
9671 selection_group_base_sequence: base_sequence,
9672 selection_group_size: group_size,
9673 member_key: selection_key_from_value(payload.get("member_key"), base_sequence)?,
9674 member_index,
9675 member_base_sequence,
9676 member_size,
9677 operation_kind,
9678 operation_identity,
9679 outcome,
9680 resolution_event_id: payload_string(payload, "resolution_event_id").ok_or_else(
9681 || {
9682 invalid_recorded_history(
9683 "selection_marker_invalid",
9684 base_sequence,
9685 "durable resolution_event_id",
9686 &payload.to_string(),
9687 "selection winner history is missing its terminal event identity",
9688 )
9689 },
9690 )?,
9691 resolution_event_type: payload_string(payload, "resolution_event_type").ok_or_else(
9692 || {
9693 invalid_recorded_history(
9694 "selection_marker_invalid",
9695 base_sequence,
9696 "durable resolution_event_type",
9697 &payload.to_string(),
9698 "selection winner history is missing its terminal event type",
9699 )
9700 },
9701 )?,
9702 };
9703 if let Some(existing) = markers
9704 .iter()
9705 .find(|existing| existing.selection_group_id == marker.selection_group_id)
9706 {
9707 if existing != &marker {
9708 return Err(invalid_recorded_history(
9709 "selection_marker_conflict",
9710 base_sequence,
9711 &format!("one winner for {}", marker.selection_group_id),
9712 &payload.to_string(),
9713 "selection history records conflicting winners for one durable group",
9714 ));
9715 }
9716 continue;
9717 }
9718 markers.push(marker);
9719 }
9720 Ok(markers)
9721}
9722
9723fn recorded_selection_cancellations(events: &[HistoryEvent]) -> Result<Vec<SelectionCancellation>> {
9724 let mut cancelled: Vec<SelectionCancellation> = Vec::new();
9725 for event in events
9726 .iter()
9727 .filter(|event| event.event_type == "SelectionOperationCancelled")
9728 {
9729 let group_id = payload_string(&event.payload, "selection_group_id").ok_or_else(|| {
9730 invalid_recorded_history(
9731 "selection_cancellation_invalid",
9732 0,
9733 "non-empty selection_group_id",
9734 &event.payload.to_string(),
9735 "selection cancellation history is missing its group identity",
9736 )
9737 })?;
9738 let member_base_sequence = required_selection_u64(&event.payload, "member_base_sequence")?;
9739 let marker = SelectionCancellation {
9740 selection_group_id: group_id,
9741 member_key: selection_key_from_value(
9742 event.payload.get("member_key"),
9743 member_base_sequence,
9744 )?,
9745 member_index: required_selection_usize_allow_zero(&event.payload, "member_index")?,
9746 member_base_sequence,
9747 member_size: required_selection_usize(&event.payload, "member_size")?,
9748 operation_kind: payload_string(&event.payload, "operation_kind").ok_or_else(|| {
9749 invalid_recorded_history(
9750 "selection_cancellation_invalid",
9751 member_base_sequence,
9752 "selection operation kind",
9753 &event.payload.to_string(),
9754 "selection cancellation is missing its operation kind",
9755 )
9756 })?,
9757 operation_identity: payload_string(&event.payload, "operation_identity").ok_or_else(
9758 || {
9759 invalid_recorded_history(
9760 "selection_cancellation_invalid",
9761 member_base_sequence,
9762 "selection operation identity",
9763 &event.payload.to_string(),
9764 "selection cancellation is missing its operation identity",
9765 )
9766 },
9767 )?,
9768 };
9769 if let Some(existing) = cancelled.iter().find(|recorded| {
9770 recorded.selection_group_id == marker.selection_group_id
9771 && recorded.member_base_sequence == marker.member_base_sequence
9772 }) {
9773 if existing != &marker {
9774 return Err(invalid_recorded_history(
9775 "selection_cancellation_conflict",
9776 member_base_sequence,
9777 "one stable SelectionOperationCancelled marker",
9778 &event.payload.to_string(),
9779 "selection cancellation history contains conflicting member metadata",
9780 ));
9781 }
9782 } else {
9783 cancelled.push(marker);
9784 }
9785 }
9786 Ok(cancelled)
9787}
9788
9789fn required_selection_u64(payload: &Value, field: &str) -> Result<u64> {
9790 payload
9791 .get(field)
9792 .and_then(value_as_u64)
9793 .filter(|value| *value > 0)
9794 .ok_or_else(|| {
9795 invalid_recorded_history(
9796 "selection_marker_invalid",
9797 0,
9798 &format!("positive integer {field}"),
9799 &payload.to_string(),
9800 "selection history contains invalid durable identity metadata",
9801 )
9802 })
9803}
9804
9805fn required_selection_usize(payload: &Value, field: &str) -> Result<usize> {
9806 required_selection_usize_allow_zero(payload, field).and_then(|value| {
9807 if value > 0 {
9808 Ok(value)
9809 } else {
9810 Err(invalid_recorded_history(
9811 "selection_marker_invalid",
9812 0,
9813 &format!("positive integer {field}"),
9814 &payload.to_string(),
9815 "selection history contains invalid durable identity metadata",
9816 ))
9817 }
9818 })
9819}
9820
9821fn required_selection_usize_allow_zero(payload: &Value, field: &str) -> Result<usize> {
9822 payload
9823 .get(field)
9824 .and_then(value_as_u64)
9825 .and_then(|value| usize::try_from(value).ok())
9826 .ok_or_else(|| {
9827 invalid_recorded_history(
9828 "selection_marker_invalid",
9829 0,
9830 &format!("non-negative integer {field}"),
9831 &payload.to_string(),
9832 "selection history contains invalid durable identity metadata",
9833 )
9834 })
9835}
9836
9837#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9838struct RecordedActivityOptions {
9839 task_queue: RecordedSnapshotValue<Option<String>>,
9840 execution_mode: RecordedSnapshotValue<Option<String>>,
9841 retry_policy: ActivityRetrySnapshot,
9842}
9843
9844#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9845enum RecordedSnapshotValue<T> {
9846 Unknown,
9848 Known(T),
9849}
9850
9851impl<T: PartialEq> RecordedSnapshotValue<T> {
9852 fn matches_current(&self, current: &Self) -> bool {
9853 match self {
9854 Self::Unknown => true,
9855 Self::Known(recorded) => matches!(current, Self::Known(value) if value == recorded),
9856 }
9857 }
9858}
9859
9860#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9861struct ActivityRetrySnapshot {
9862 snapshot_version: RecordedSnapshotValue<Option<u64>>,
9863 max_attempts: RecordedSnapshotValue<Option<u64>>,
9864 backoff_seconds: RecordedSnapshotValue<Vec<u64>>,
9865 start_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
9866 schedule_to_start_timeout: RecordedSnapshotValue<Option<u64>>,
9867 schedule_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
9868 heartbeat_timeout: RecordedSnapshotValue<Option<u64>>,
9869 non_retryable_error_types: RecordedSnapshotValue<Vec<String>>,
9870}
9871
9872impl ActivityRetrySnapshot {
9873 fn matches_current(&self, current: &Self) -> bool {
9874 self.snapshot_version
9875 .matches_current(¤t.snapshot_version)
9876 && self.max_attempts.matches_current(¤t.max_attempts)
9877 && self
9878 .backoff_seconds
9879 .matches_current(¤t.backoff_seconds)
9880 && self
9881 .start_to_close_timeout
9882 .matches_current(¤t.start_to_close_timeout)
9883 && self
9884 .schedule_to_start_timeout
9885 .matches_current(¤t.schedule_to_start_timeout)
9886 && self
9887 .schedule_to_close_timeout
9888 .matches_current(¤t.schedule_to_close_timeout)
9889 && self
9890 .heartbeat_timeout
9891 .matches_current(¤t.heartbeat_timeout)
9892 && self
9893 .non_retryable_error_types
9894 .matches_current(¤t.non_retryable_error_types)
9895 }
9896}
9897
9898fn recorded_optional_u64(
9899 object: Option<&serde_json::Map<String, Value>>,
9900 field: &str,
9901) -> RecordedSnapshotValue<Option<u64>> {
9902 match object.and_then(|object| object.get(field)) {
9903 None => RecordedSnapshotValue::Unknown,
9904 Some(Value::Null) => RecordedSnapshotValue::Known(None),
9905 Some(value) => RecordedSnapshotValue::Known(value_as_u64(value)),
9906 }
9907}
9908
9909fn recorded_optional_string(
9910 object: &serde_json::Map<String, Value>,
9911 field: &str,
9912) -> RecordedSnapshotValue<Option<String>> {
9913 match object.get(field) {
9914 None => RecordedSnapshotValue::Unknown,
9915 Some(Value::Null) => RecordedSnapshotValue::Known(None),
9916 Some(value) => RecordedSnapshotValue::Known(value.as_str().map(str::to_string)),
9917 }
9918}
9919
9920fn recorded_activity_retry_snapshot(policy: Option<&Value>) -> ActivityRetrySnapshot {
9921 let policy = policy.and_then(Value::as_object);
9922 let backoff_seconds = policy
9923 .and_then(|policy| policy.get("backoff_seconds"))
9924 .and_then(Value::as_array)
9925 .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
9926 .map_or(RecordedSnapshotValue::Unknown, RecordedSnapshotValue::Known);
9927 let mut non_retryable_error_types = Vec::new();
9928 for error_type in policy
9929 .and_then(|policy| policy.get("non_retryable_error_types"))
9930 .and_then(Value::as_array)
9931 .into_iter()
9932 .flatten()
9933 .filter_map(Value::as_str)
9934 .map(str::trim)
9935 .filter(|error_type| !error_type.is_empty())
9936 {
9937 if !non_retryable_error_types
9938 .iter()
9939 .any(|recorded| recorded == error_type)
9940 {
9941 non_retryable_error_types.push(error_type.to_string());
9942 }
9943 }
9944
9945 ActivityRetrySnapshot {
9946 snapshot_version: recorded_optional_u64(policy, "snapshot_version"),
9947 max_attempts: recorded_optional_u64(policy, "max_attempts"),
9948 backoff_seconds,
9949 start_to_close_timeout: recorded_optional_u64(policy, "start_to_close_timeout"),
9950 schedule_to_start_timeout: recorded_optional_u64(policy, "schedule_to_start_timeout"),
9951 schedule_to_close_timeout: recorded_optional_u64(policy, "schedule_to_close_timeout"),
9952 heartbeat_timeout: recorded_optional_u64(policy, "heartbeat_timeout"),
9953 non_retryable_error_types: if policy
9954 .is_some_and(|policy| policy.contains_key("non_retryable_error_types"))
9955 {
9956 RecordedSnapshotValue::Known(non_retryable_error_types)
9957 } else {
9958 RecordedSnapshotValue::Unknown
9959 },
9960 }
9961}
9962
9963fn current_activity_retry_snapshot(options: &ValidatedActivityOptions) -> ActivityRetrySnapshot {
9964 let policy = options.retry_policy.as_ref();
9965 let max_attempts = match policy.and_then(|policy| policy.get("max_attempts")) {
9966 Some(Value::Null) => None,
9967 Some(value) => value_as_u64(value),
9968 None => Some(1),
9969 };
9970 let backoff_seconds = policy
9971 .and_then(|policy| policy.get("backoff_seconds"))
9972 .and_then(Value::as_array)
9973 .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
9974 .unwrap_or_default();
9975 let non_retryable_error_types = policy
9976 .and_then(|policy| policy.get("non_retryable_error_types"))
9977 .and_then(Value::as_array)
9978 .into_iter()
9979 .flatten()
9980 .filter_map(Value::as_str)
9981 .map(str::to_string)
9982 .collect();
9983
9984 ActivityRetrySnapshot {
9985 snapshot_version: RecordedSnapshotValue::Known(Some(1)),
9986 max_attempts: RecordedSnapshotValue::Known(max_attempts),
9987 backoff_seconds: RecordedSnapshotValue::Known(backoff_seconds),
9988 start_to_close_timeout: RecordedSnapshotValue::Known(options.start_to_close_timeout),
9989 schedule_to_start_timeout: RecordedSnapshotValue::Known(options.schedule_to_start_timeout),
9990 schedule_to_close_timeout: RecordedSnapshotValue::Known(options.schedule_to_close_timeout),
9991 heartbeat_timeout: RecordedSnapshotValue::Known(options.heartbeat_timeout),
9992 non_retryable_error_types: RecordedSnapshotValue::Known(non_retryable_error_types),
9993 }
9994}
9995
9996fn activity_options_description(options: &RecordedActivityOptions) -> String {
9997 serde_json::to_string(options).unwrap_or_else(|_| format!("{options:?}"))
9998}
9999
10000impl RecordedCommand {
10001 fn sequence(&self) -> u64 {
10002 match self {
10003 Self::Activity { sequence, .. }
10004 | Self::Timer { sequence, .. }
10005 | Self::ChildWorkflow { sequence, .. }
10006 | Self::SignalWait { sequence, .. }
10007 | Self::ConditionWait { sequence, .. }
10008 | Self::SearchAttributes { sequence, .. }
10009 | Self::SideEffect { sequence, .. }
10010 | Self::VersionMarker { sequence, .. }
10011 | Self::Memo { sequence, .. } => *sequence,
10012 }
10013 }
10014
10015 fn shape(&self) -> &'static str {
10016 match self {
10017 Self::Activity { .. } => "activity",
10018 Self::Timer { .. } => "timer",
10019 Self::ChildWorkflow { .. } => "child workflow",
10020 Self::SignalWait { .. } => "signal wait",
10021 Self::ConditionWait { .. } => "condition wait",
10022 Self::SearchAttributes { .. } => "search-attribute update",
10023 Self::SideEffect { .. } => "side effect",
10024 Self::VersionMarker { .. } => "version marker",
10025 Self::Memo { .. } => "memo upsert",
10026 }
10027 }
10028}
10029
10030fn ensure_version_supported(
10031 change_id: &str,
10032 version: i32,
10033 min_supported: i32,
10034 max_supported: i32,
10035 sequence: u64,
10036) -> Result<()> {
10037 if (min_supported..=max_supported).contains(&version) {
10038 return Ok(());
10039 }
10040 Err(Error::NonDeterministicReplay(ReplayFailure::new(
10041 "version_marker_incompatible_range",
10042 (sequence != 0).then_some(sequence),
10043 Some(format!("{min_supported}..={max_supported}")),
10044 Some(format!("{change_id}:{version}")),
10045 "recorded workflow version is outside the range supported by current code",
10046 )))
10047}
10048
10049#[derive(Clone, Debug)]
10050struct ResumeSignal {
10051 signal_name: String,
10052 arguments: Vec<AvroValue>,
10053}
10054
10055const MAX_PARALLEL_OPERATIONS: usize = 1000;
10056
10057fn parallel_group_prefix(kind: &str) -> &'static str {
10058 match kind {
10059 "activity" => "parallel-activities",
10060 "child" => "parallel-children",
10061 "timer" => "parallel-timers",
10062 _ => "parallel-calls",
10063 }
10064}
10065
10066fn parallel_group_entry(
10067 base_sequence: u64,
10068 size: usize,
10069 index: usize,
10070 kind: &str,
10071) -> ParallelGroupMetadata {
10072 ParallelGroupMetadata {
10073 parallel_group_id: format!("{}:{base_sequence}:{size}", parallel_group_prefix(kind)),
10074 parallel_group_kind: kind.to_string(),
10075 parallel_group_base_sequence: base_sequence,
10076 parallel_group_size: size,
10077 parallel_group_index: index,
10078 parallel_group_mode: None,
10079 selection_member_key: None,
10080 selection_member_index: None,
10081 selection_member_base_sequence: None,
10082 selection_member_size: None,
10083 selection_member_kind: None,
10084 }
10085}
10086
10087struct SelectionMemberMetadata {
10088 key: SelectionKey,
10089 index: usize,
10090 base_sequence: u64,
10091 size: usize,
10092 kind: String,
10093}
10094
10095fn selection_group_entry(
10096 base_sequence: u64,
10097 size: usize,
10098 index: usize,
10099 kind: &str,
10100 member: &SelectionMemberMetadata,
10101) -> ParallelGroupMetadata {
10102 ParallelGroupMetadata {
10103 parallel_group_id: format!("select-calls:{base_sequence}:{size}"),
10104 parallel_group_kind: kind.to_string(),
10105 parallel_group_base_sequence: base_sequence,
10106 parallel_group_size: size,
10107 parallel_group_index: index,
10108 parallel_group_mode: Some("select".to_string()),
10109 selection_member_key: Some(member.key.clone()),
10110 selection_member_index: Some(member.index),
10111 selection_member_base_sequence: Some(member.base_sequence),
10112 selection_member_size: Some(member.size),
10113 selection_member_kind: Some(member.kind.clone()),
10114 }
10115}
10116
10117fn apply_parallel_group_path(
10118 command: &mut serde_json::Map<String, Value>,
10119 path: &[ParallelGroupMetadata],
10120) {
10121 let Some(inner) = path.last() else {
10122 return;
10123 };
10124 command.insert(
10125 "parallel_group_id".to_string(),
10126 json!(inner.parallel_group_id),
10127 );
10128 command.insert(
10129 "parallel_group_kind".to_string(),
10130 json!(inner.parallel_group_kind),
10131 );
10132 command.insert(
10133 "parallel_group_base_sequence".to_string(),
10134 json!(inner.parallel_group_base_sequence),
10135 );
10136 command.insert(
10137 "parallel_group_size".to_string(),
10138 json!(inner.parallel_group_size),
10139 );
10140 command.insert(
10141 "parallel_group_index".to_string(),
10142 json!(inner.parallel_group_index),
10143 );
10144 if let Some(mode) = &inner.parallel_group_mode {
10145 command.insert("parallel_group_mode".to_string(), json!(mode));
10146 }
10147 if let Some(key) = &inner.selection_member_key {
10148 command.insert("selection_member_key".to_string(), json!(key));
10149 }
10150 if let Some(index) = inner.selection_member_index {
10151 command.insert("selection_member_index".to_string(), json!(index));
10152 }
10153 if let Some(base_sequence) = inner.selection_member_base_sequence {
10154 command.insert(
10155 "selection_member_base_sequence".to_string(),
10156 json!(base_sequence),
10157 );
10158 }
10159 if let Some(size) = inner.selection_member_size {
10160 command.insert("selection_member_size".to_string(), json!(size));
10161 }
10162 if let Some(kind) = &inner.selection_member_kind {
10163 command.insert("selection_member_kind".to_string(), json!(kind));
10164 }
10165 command.insert("parallel_group_path".to_string(), json!(path));
10166}
10167
10168fn ensure_parallel_path_matches(
10169 sequence: u64,
10170 recorded: Option<&[ParallelGroupMetadata]>,
10171 expected: &[ParallelGroupMetadata],
10172) -> Result<()> {
10173 match (recorded, expected.is_empty()) {
10174 (None, true) => Ok(()),
10175 (Some(recorded), false) if recorded == expected => Ok(()),
10176 (None, false) => Err(invalid_recorded_history(
10177 "parallel_group_metadata_missing",
10178 sequence,
10179 &serde_json::to_string(expected).unwrap_or_default(),
10180 "<missing>",
10181 "recorded parallel member is missing its durable group path",
10182 )),
10183 (Some(recorded), true) => Err(invalid_recorded_history(
10184 "parallel_group_shape_mismatch",
10185 sequence,
10186 "sequential command",
10187 &serde_json::to_string(recorded).unwrap_or_default(),
10188 "recorded command belonged to a parallel group but current code schedules it sequentially",
10189 )),
10190 (Some(recorded), false) => Err(invalid_recorded_history(
10191 "parallel_group_shape_mismatch",
10192 sequence,
10193 &serde_json::to_string(recorded).unwrap_or_default(),
10194 &serde_json::to_string(expected).unwrap_or_default(),
10195 "recorded parallel-group identity or path changed during replay",
10196 )),
10197 }
10198}
10199
10200#[derive(Clone, Debug)]
10201enum ParallelShape {
10202 Leaf,
10203 Group(Vec<ParallelShape>),
10204}
10205
10206struct ParallelDescriptor {
10207 operation: ParallelOperation,
10208 offset: usize,
10209 member_path: Vec<usize>,
10210 group_path: Vec<ParallelGroupMetadata>,
10211}
10212
10213fn parallel_leaf_count(operations: &[ParallelOperation]) -> usize {
10214 operations
10215 .iter()
10216 .map(|operation| match operation {
10217 ParallelOperation::Group(children) => parallel_leaf_count(children),
10218 _ => 1,
10219 })
10220 .sum()
10221}
10222
10223fn parallel_operation_kind(operation: &ParallelOperation) -> Option<&'static str> {
10224 match operation {
10225 ParallelOperation::Activity { .. } => Some("activity"),
10226 ParallelOperation::ChildWorkflow { .. } => Some("child"),
10227 ParallelOperation::Timer(_) => Some("timer"),
10228 ParallelOperation::Signal(_) => Some("signal"),
10229 ParallelOperation::Condition { .. } => Some("condition"),
10230 ParallelOperation::Group(children) => parallel_group_kind(children),
10231 }
10232}
10233
10234fn parallel_group_kind(operations: &[ParallelOperation]) -> Option<&'static str> {
10235 let mut kind = None;
10236 for operation in operations {
10237 let Some(operation_kind) = parallel_operation_kind(operation) else {
10238 continue;
10239 };
10240 match kind {
10241 None => kind = Some(operation_kind),
10242 Some(current) if current == operation_kind => {}
10243 Some(_) => return Some("mixed"),
10244 }
10245 }
10246 kind
10247}
10248
10249fn validate_parallel_operations(
10250 operations: &[ParallelOperation],
10251 member_path: &mut Vec<usize>,
10252 root: bool,
10253) -> Result<()> {
10254 let leaves = parallel_leaf_count(operations);
10255 if leaves > MAX_PARALLEL_OPERATIONS {
10256 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10257 reason: "fan_out_limit_exceeded",
10258 member_path: member_path.clone(),
10259 message: format!(
10260 "group contains {leaves} durable leaves; the limit is {MAX_PARALLEL_OPERATIONS}"
10261 ),
10262 }));
10263 }
10264 if !root && operations.is_empty() {
10265 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10266 reason: "nested_group_empty",
10267 member_path: member_path.clone(),
10268 message: "a nested group must contain at least one durable leaf".to_string(),
10269 }));
10270 }
10271
10272 for (index, operation) in operations.iter().enumerate() {
10273 member_path.push(index);
10274 match operation {
10275 ParallelOperation::Activity {
10276 options, arguments, ..
10277 } => {
10278 options
10279 .validate()
10280 .map_err(|error| Error::InvalidActivityOptions(error))?;
10281 if let Err(error) = arguments {
10282 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10283 reason: "arguments_invalid",
10284 member_path: member_path.clone(),
10285 message: error.to_string(),
10286 }));
10287 }
10288 }
10289 ParallelOperation::ChildWorkflow {
10290 options, arguments, ..
10291 } => {
10292 validate_parallel_child_options(options)?;
10293 if let Err(error) = arguments {
10294 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10295 reason: "arguments_invalid",
10296 member_path: member_path.clone(),
10297 message: error.to_string(),
10298 }));
10299 }
10300 }
10301 ParallelOperation::Timer(duration)
10302 if duration.as_secs() == u64::MAX && duration.subsec_nanos() > 0 =>
10303 {
10304 return Err(Error::TimerDurationOverflow);
10305 }
10306 ParallelOperation::Timer(_) => {}
10307 ParallelOperation::Signal(signal_name) => {
10308 validate_user_signal_name(signal_name)?;
10309 if signal_name.trim().is_empty() {
10310 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10311 reason: "signal_name_empty",
10312 member_path: member_path.clone(),
10313 message: "signal wait name must not be empty".to_string(),
10314 }));
10315 }
10316 }
10317 ParallelOperation::Condition { options, .. } => {
10318 options.validate()?;
10319 }
10320 ParallelOperation::Group(children) => {
10321 validate_parallel_operations(children, member_path, false)?;
10322 }
10323 }
10324 member_path.pop();
10325 }
10326 Ok(())
10327}
10328
10329fn validate_parallel_child_options(options: &ChildWorkflowOptions) -> Result<()> {
10330 if options.task_queue.trim().is_empty() {
10331 return Err(Error::InvalidChildWorkflowOptions(
10332 "task_queue must not be empty".to_string(),
10333 ));
10334 }
10335 for (name, value) in [
10336 (
10337 "execution_timeout_seconds",
10338 options.execution_timeout_seconds,
10339 ),
10340 ("run_timeout_seconds", options.run_timeout_seconds),
10341 ] {
10342 if value == Some(0) {
10343 return Err(Error::InvalidChildWorkflowOptions(format!(
10344 "{name} must be at least 1"
10345 )));
10346 }
10347 }
10348 if options
10349 .retry_policy
10350 .as_ref()
10351 .is_some_and(|policy| policy.max_attempts == Some(0))
10352 {
10353 return Err(Error::InvalidChildWorkflowOptions(
10354 "retry_policy.max_attempts must be at least 1".to_string(),
10355 ));
10356 }
10357 Ok(())
10358}
10359
10360fn parallel_shape(operations: &[ParallelOperation]) -> ParallelShape {
10361 ParallelShape::Group(
10362 operations
10363 .iter()
10364 .map(|operation| match operation {
10365 ParallelOperation::Group(children) => parallel_shape(children),
10366 _ => ParallelShape::Leaf,
10367 })
10368 .collect(),
10369 )
10370}
10371
10372fn parallel_descriptors(
10373 operations: Vec<ParallelOperation>,
10374 base_sequence: u64,
10375) -> Result<Vec<ParallelDescriptor>> {
10376 let size = parallel_leaf_count(&operations);
10377 let kind = parallel_group_kind(&operations).unwrap_or("activity");
10378 let mut descriptors = Vec::with_capacity(size);
10379 let mut cursor = 0;
10380
10381 for (index, operation) in operations.into_iter().enumerate() {
10382 match operation {
10383 ParallelOperation::Group(children) => {
10384 let child_base = base_sequence
10385 .checked_add(u64::try_from(cursor).unwrap_or(u64::MAX))
10386 .ok_or(Error::TimerDurationOverflow)?;
10387 for mut descriptor in parallel_descriptors(children, child_base)? {
10388 let outer_index = cursor + descriptor.offset;
10389 descriptor.group_path.insert(
10390 0,
10391 parallel_group_entry(base_sequence, size, outer_index, kind),
10392 );
10393 descriptor.member_path.insert(0, index);
10394 descriptor.offset = outer_index;
10395 descriptors.push(descriptor);
10396 }
10397 cursor = descriptors.len();
10398 }
10399 operation => {
10400 descriptors.push(ParallelDescriptor {
10401 operation,
10402 offset: cursor,
10403 member_path: vec![index],
10404 group_path: vec![parallel_group_entry(base_sequence, size, cursor, kind)],
10405 });
10406 cursor += 1;
10407 }
10408 }
10409 }
10410 Ok(descriptors)
10411}
10412
10413enum ParallelLeafCall {
10414 Activity(ActivityCall),
10415 ChildWorkflow(ChildWorkflowCall),
10416 Timer(TimerCall),
10417 Signal(SignalCall),
10418 Condition(ConditionWaitCall),
10419}
10420
10421fn parallel_leaf_call(
10422 ctx: &WorkflowContext,
10423 operation: ParallelOperation,
10424 parallel_group_path: Vec<ParallelGroupMetadata>,
10425) -> ParallelLeafCall {
10426 match operation {
10427 ParallelOperation::Activity {
10428 activity_type,
10429 options,
10430 arguments,
10431 } => ParallelLeafCall::Activity(ActivityCall {
10432 ctx: ctx.clone(),
10433 activity_type,
10434 options,
10435 args: Some(arguments),
10436 scheduled: false,
10437 parallel_group_path,
10438 }),
10439 ParallelOperation::ChildWorkflow {
10440 workflow_type,
10441 options,
10442 arguments,
10443 } => ParallelLeafCall::ChildWorkflow(ChildWorkflowCall {
10444 ctx: ctx.clone(),
10445 workflow_type,
10446 options,
10447 args: Some(arguments),
10448 scheduled: false,
10449 matched_pending: false,
10450 parallel_group_path,
10451 }),
10452 ParallelOperation::Timer(duration) => {
10453 let delay_seconds = duration
10454 .as_secs()
10455 .checked_add(u64::from(duration.subsec_nanos() > 0));
10456 ParallelLeafCall::Timer(TimerCall {
10457 ctx: ctx.clone(),
10458 delay_seconds,
10459 scheduled: false,
10460 matched_pending: false,
10461 parallel_group_path,
10462 })
10463 }
10464 ParallelOperation::Signal(signal_name) => ParallelLeafCall::Signal(SignalCall {
10465 ctx: ctx.clone(),
10466 signal_name,
10467 runtime_reserved_allowed: false,
10468 opened_wait: false,
10469 matched_pending: false,
10470 parallel_group_path,
10471 }),
10472 ParallelOperation::Condition { options, predicate } => {
10473 ParallelLeafCall::Condition(ConditionWaitCall {
10474 ctx: ctx.clone(),
10475 options,
10476 predicate,
10477 occurrence_id: None,
10478 opened_wait: false,
10479 parallel_group_path,
10480 })
10481 }
10482 ParallelOperation::Group(_) => {
10483 unreachable!("parallel descriptors contain only durable leaves")
10484 }
10485 }
10486}
10487
10488impl ParallelLeafCall {
10489 fn poll_avro_value(&mut self, cx: &mut TaskContext<'_>) -> Poll<Result<ParallelAvroResult>> {
10490 match self {
10491 Self::Activity(call) => Pin::new(call)
10492 .poll_avro_value(cx)
10493 .map_ok(ParallelAvroResult::Activity),
10494 Self::ChildWorkflow(call) => Pin::new(call)
10495 .poll_avro_value(cx)
10496 .map_ok(ParallelAvroResult::ChildWorkflow),
10497 Self::Timer(call) => Pin::new(call)
10498 .poll(cx)
10499 .map_ok(|()| ParallelAvroResult::Timer),
10500 Self::Signal(call) => Pin::new(call)
10501 .poll_avro_value(cx)
10502 .map_ok(ParallelAvroResult::Signal),
10503 Self::Condition(call) => Pin::new(call)
10504 .poll(cx)
10505 .map_ok(ParallelAvroResult::Condition),
10506 }
10507 }
10508}
10509
10510struct ParallelLeaf {
10511 call: ParallelLeafCall,
10512 member_path: Vec<usize>,
10513 group_path: Vec<ParallelGroupMetadata>,
10514 result: Option<ParallelAvroResult>,
10515}
10516
10517pub struct ParallelCall {
10519 ctx: WorkflowContext,
10520 operations: Option<Vec<ParallelOperation>>,
10521 shape: Option<ParallelShape>,
10522 leaves: Vec<ParallelLeaf>,
10523}
10524
10525impl ParallelCall {
10526 fn new(ctx: WorkflowContext, operations: Vec<ParallelOperation>) -> Self {
10527 Self {
10528 ctx,
10529 operations: Some(operations),
10530 shape: None,
10531 leaves: Vec::new(),
10532 }
10533 }
10534
10535 fn initialize(&mut self) -> Result<()> {
10536 let operations = self.operations.take().unwrap_or_default();
10537 validate_parallel_operations(&operations, &mut Vec::new(), true)?;
10538 self.shape = Some(parallel_shape(&operations));
10539 if operations.is_empty() {
10540 return Ok(());
10541 }
10542
10543 let base_sequence = {
10544 let state = self
10545 .ctx
10546 .state
10547 .lock()
10548 .map_err(|_| Error::WorkflowStatePoisoned)?;
10549 if let Some(recorded) = state.recorded_commands.get(state.command_cursor) {
10550 recorded.sequence()
10551 } else {
10552 let last = state
10553 .recorded_commands
10554 .last()
10555 .map(RecordedCommand::sequence)
10556 .unwrap_or(0);
10557 last.checked_add(u64::try_from(state.commands.len()).unwrap_or(u64::MAX))
10558 .and_then(|sequence| sequence.checked_add(1))
10559 .ok_or_else(|| {
10560 Error::InvalidParallelGroup(ParallelGroupError {
10561 reason: "sequence_overflow",
10562 member_path: Vec::new(),
10563 message: "parallel group sequence identity overflowed u64".to_string(),
10564 })
10565 })?
10566 }
10567 };
10568
10569 self.leaves = parallel_descriptors(operations, base_sequence)?
10570 .into_iter()
10571 .map(|descriptor| {
10572 let call = parallel_leaf_call(
10573 &self.ctx,
10574 descriptor.operation,
10575 descriptor.group_path.clone(),
10576 );
10577 ParallelLeaf {
10578 call,
10579 member_path: descriptor.member_path,
10580 group_path: descriptor.group_path,
10581 result: None,
10582 }
10583 })
10584 .collect();
10585 Ok(())
10586 }
10587
10588 fn poll_avro_value(
10589 mut self: Pin<&mut Self>,
10590 cx: &mut TaskContext<'_>,
10591 ) -> Poll<Result<Vec<ParallelAvroResult>>> {
10592 if self.operations.is_some() {
10593 if let Err(error) = self.initialize() {
10594 return Poll::Ready(Err(error));
10595 }
10596 }
10597 if self.leaves.is_empty() {
10598 return Poll::Ready(Ok(Vec::new()));
10599 }
10600
10601 let mut failures = Vec::new();
10602 let mut pending = false;
10603 for (index, leaf) in self.leaves.iter_mut().enumerate() {
10604 if leaf.result.is_some() {
10605 continue;
10606 }
10607 match leaf.call.poll_avro_value(cx) {
10608 Poll::Ready(Ok(result)) => leaf.result = Some(result),
10609 Poll::Ready(Err(error)) => failures.push((index, error)),
10610 Poll::Pending => pending = true,
10611 }
10612 }
10613
10614 if !failures.is_empty() {
10615 if let Some(position) = failures
10616 .iter()
10617 .position(|(_, error)| workflow_task_integrity_error(error))
10618 {
10619 return Poll::Ready(Err(failures.remove(position).1));
10620 }
10621 failures.sort_by_key(|(index, _)| *index);
10622 let (failed_index, cause) = failures.remove(0);
10623 let failed = &self.leaves[failed_index];
10624 let completed = self
10625 .leaves
10626 .iter()
10627 .filter_map(|leaf| {
10628 leaf.result
10629 .clone()
10630 .and_then(|result| result.into_json_result().ok())
10631 .map(|result| ParallelCompletion {
10632 member_path: leaf.member_path.clone(),
10633 result,
10634 })
10635 })
10636 .collect();
10637 let group_id = failed
10638 .group_path
10639 .first()
10640 .map(|entry| entry.parallel_group_id.clone())
10641 .unwrap_or_default();
10642 return Poll::Ready(Err(Error::ParallelFailed(ParallelFailure {
10643 group_id,
10644 member_path: failed.member_path.clone(),
10645 group_path: failed.group_path.clone(),
10646 completed,
10647 cause: Box::new(cause),
10648 })));
10649 }
10650 if pending {
10651 return Poll::Pending;
10652 }
10653
10654 let mut flat_results = self
10655 .leaves
10656 .iter_mut()
10657 .map(|leaf| leaf.result.take().expect("completed parallel leaf"))
10658 .collect::<Vec<_>>()
10659 .into_iter();
10660 let results = parallel_results_for_shape(
10661 self.shape.as_ref().expect("initialized parallel shape"),
10662 &mut flat_results,
10663 );
10664 Poll::Ready(Ok(match results {
10665 ParallelAvroResult::Group(results) => results,
10666 ParallelAvroResult::Activity(_)
10667 | ParallelAvroResult::ChildWorkflow(_)
10668 | ParallelAvroResult::Timer
10669 | ParallelAvroResult::Signal(_)
10670 | ParallelAvroResult::Condition(_) => {
10671 unreachable!("root parallel shape is a group")
10672 }
10673 }))
10674 }
10675}
10676
10677fn parallel_results_for_shape(
10678 shape: &ParallelShape,
10679 flat_results: &mut impl Iterator<Item = ParallelAvroResult>,
10680) -> ParallelAvroResult {
10681 match shape {
10682 ParallelShape::Leaf => flat_results.next().expect("one result per parallel leaf"),
10683 ParallelShape::Group(children) => ParallelAvroResult::Group(
10684 children
10685 .iter()
10686 .map(|child| parallel_results_for_shape(child, flat_results))
10687 .collect(),
10688 ),
10689 }
10690}
10691
10692impl Future for ParallelCall {
10693 type Output = Result<Vec<ParallelResult>>;
10694
10695 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
10696 self.poll_avro_value(cx)
10697 .map_ok(|results| {
10698 results
10699 .into_iter()
10700 .map(ParallelAvroResult::into_json_result)
10701 .collect::<Result<Vec<_>>>()
10702 })
10703 .map_ok(|result| result)
10704 .flatten_result()
10705 }
10706}
10707
10708#[derive(Clone, Debug)]
10709struct SelectionMemberPlan {
10710 key: SelectionKey,
10711 index: usize,
10712 base_sequence: u64,
10713 size: usize,
10714 kind: String,
10715 shape: ParallelShape,
10716 leaf_start: usize,
10717}
10718
10719fn selection_operation_kind(operation: &ParallelOperation) -> &'static str {
10720 match operation {
10721 ParallelOperation::Activity { .. } => "activity",
10722 ParallelOperation::ChildWorkflow { .. } => "child",
10723 ParallelOperation::Timer(_) => "timer",
10724 ParallelOperation::Signal(_) => "signal",
10725 ParallelOperation::Condition { .. } => "condition",
10726 ParallelOperation::Group(_) => "group",
10727 }
10728}
10729
10730fn selection_operation_shape(operation: &ParallelOperation) -> ParallelShape {
10731 match operation {
10732 ParallelOperation::Group(children) => parallel_shape(children),
10733 _ => ParallelShape::Leaf,
10734 }
10735}
10736
10737fn selection_descriptors(
10738 operations: Vec<(SelectionKey, ParallelOperation)>,
10739 base_sequence: u64,
10740) -> Result<(Vec<ParallelDescriptor>, Vec<SelectionMemberPlan>)> {
10741 if operations.is_empty() {
10742 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10743 reason: "selection_empty",
10744 member_path: Vec::new(),
10745 message: "durable selection requires at least one operation".to_string(),
10746 }));
10747 }
10748 let operation_refs = operations
10749 .iter()
10750 .map(|(_, operation)| operation)
10751 .collect::<Vec<_>>();
10752 let total_size = operation_refs
10753 .iter()
10754 .map(|operation| match operation {
10755 ParallelOperation::Group(children) => parallel_leaf_count(children),
10756 _ => 1,
10757 })
10758 .sum::<usize>();
10759 if total_size > MAX_PARALLEL_OPERATIONS {
10760 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10761 reason: "fan_out_limit_exceeded",
10762 member_path: Vec::new(),
10763 message: format!(
10764 "selection contains {total_size} durable leaves; the limit is {MAX_PARALLEL_OPERATIONS}"
10765 ),
10766 }));
10767 }
10768 let group_kind = {
10769 let mut kind = None;
10770 for operation in &operation_refs {
10771 let operation_kind = parallel_operation_kind(operation).unwrap_or("mixed");
10772 match kind {
10773 None => kind = Some(operation_kind),
10774 Some(current) if current == operation_kind => {}
10775 Some(_) => {
10776 kind = Some("mixed");
10777 break;
10778 }
10779 }
10780 }
10781 kind.unwrap_or("mixed")
10782 };
10783
10784 let mut descriptors = Vec::with_capacity(total_size);
10785 let mut members = Vec::with_capacity(operations.len());
10786 let mut cursor = 0usize;
10787 let mut seen_keys: Vec<SelectionKey> = Vec::new();
10788 for (member_index, (key, operation)) in operations.into_iter().enumerate() {
10789 if matches!(&key, SelectionKey::Name(value) if value.is_empty()) {
10790 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10791 reason: "selection_key_invalid",
10792 member_path: vec![member_index],
10793 message: "selection member keys must be non-empty strings or non-negative integers"
10794 .to_string(),
10795 }));
10796 }
10797 if seen_keys.contains(&key) {
10798 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10799 reason: "selection_key_duplicate",
10800 member_path: vec![member_index],
10801 message: format!("selection member key {key:?} is duplicated"),
10802 }));
10803 }
10804 seen_keys.push(key.clone());
10805 let member_size = match &operation {
10806 ParallelOperation::Group(children) => parallel_leaf_count(children),
10807 _ => 1,
10808 };
10809 if member_size == 0 {
10810 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10811 reason: "selection_member_empty",
10812 member_path: vec![member_index],
10813 message: "a selection member must contain at least one durable leaf".to_string(),
10814 }));
10815 }
10816 let member_base = base_sequence
10817 .checked_add(u64::try_from(cursor).unwrap_or(u64::MAX))
10818 .ok_or(Error::TimerDurationOverflow)?;
10819 let member_kind = selection_operation_kind(&operation).to_string();
10820 let member_shape = selection_operation_shape(&operation);
10821 let leaf_start = descriptors.len();
10822 match operation {
10823 ParallelOperation::Group(children) => {
10824 validate_parallel_operations(&children, &mut vec![member_index], false)?;
10825 for mut descriptor in parallel_descriptors(children, member_base)? {
10826 let flat_index = cursor + descriptor.offset;
10827 descriptor.group_path.insert(
10828 0,
10829 selection_group_entry(
10830 base_sequence,
10831 total_size,
10832 flat_index,
10833 group_kind,
10834 &SelectionMemberMetadata {
10835 key: key.clone(),
10836 index: member_index,
10837 base_sequence: member_base,
10838 size: member_size,
10839 kind: member_kind.clone(),
10840 },
10841 ),
10842 );
10843 descriptor.member_path.insert(0, member_index);
10844 descriptor.offset = flat_index;
10845 descriptors.push(descriptor);
10846 }
10847 }
10848 operation => {
10849 validate_parallel_operations(
10850 std::slice::from_ref(&operation),
10851 &mut Vec::new(),
10852 true,
10853 )?;
10854 descriptors.push(ParallelDescriptor {
10855 operation,
10856 offset: cursor,
10857 member_path: vec![member_index],
10858 group_path: vec![selection_group_entry(
10859 base_sequence,
10860 total_size,
10861 cursor,
10862 group_kind,
10863 &SelectionMemberMetadata {
10864 key: key.clone(),
10865 index: member_index,
10866 base_sequence: member_base,
10867 size: member_size,
10868 kind: member_kind.clone(),
10869 },
10870 )],
10871 });
10872 }
10873 }
10874 members.push(SelectionMemberPlan {
10875 key,
10876 index: member_index,
10877 base_sequence: member_base,
10878 size: member_size,
10879 kind: member_kind,
10880 shape: member_shape,
10881 leaf_start,
10882 });
10883 cursor += member_size;
10884 }
10885 Ok((descriptors, members))
10886}
10887
10888struct SelectionLeaf {
10889 call: ParallelLeafCall,
10890 outcome: Option<Result<ParallelAvroResult>>,
10891}
10892
10893#[derive(Clone)]
10895pub struct DurableOperationHandle {
10896 ctx: WorkflowContext,
10897 pub key: SelectionKey,
10898 pub index: usize,
10899 pub kind: String,
10900 pub identity: String,
10901 pub base_sequence: u64,
10902 pub size: usize,
10903 pub selection_group_id: String,
10904 shape: ParallelShape,
10905}
10906
10907impl std::fmt::Debug for DurableOperationHandle {
10908 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10909 formatter
10910 .debug_struct("DurableOperationHandle")
10911 .field("key", &self.key)
10912 .field("index", &self.index)
10913 .field("kind", &self.kind)
10914 .field("identity", &self.identity)
10915 .field("base_sequence", &self.base_sequence)
10916 .field("size", &self.size)
10917 .field("selection_group_id", &self.selection_group_id)
10918 .finish()
10919 }
10920}
10921
10922impl DurableOperationHandle {
10923 pub fn await_result(&self) -> DurableOperationAwaitCall {
10925 DurableOperationAwaitCall {
10926 handle: self.clone(),
10927 }
10928 }
10929
10930 pub fn cancel(&self) -> CancelDurableOperationCall {
10934 CancelDurableOperationCall {
10935 handle: self.clone(),
10936 emitted: false,
10937 }
10938 }
10939}
10940
10941#[derive(Debug)]
10943pub struct SelectionResult {
10944 pub key: SelectionKey,
10945 pub index: usize,
10946 pub kind: String,
10947 pub identity: String,
10948 pub value: Option<ParallelResult>,
10949 pub failure: Option<Error>,
10950 pub winner: DurableOperationHandle,
10951 pub handles: Vec<DurableOperationHandle>,
10952}
10953
10954impl SelectionResult {
10955 pub fn succeeded(&self) -> bool {
10956 self.failure.is_none()
10957 }
10958
10959 pub fn handle(&self, key: &SelectionKey) -> Option<&DurableOperationHandle> {
10960 self.handles.iter().find(|handle| &handle.key == key)
10961 }
10962
10963 pub fn remaining(&self) -> Vec<&DurableOperationHandle> {
10964 self.handles
10965 .iter()
10966 .filter(|handle| handle.index != self.index)
10967 .collect()
10968 }
10969
10970 pub fn into_result(self) -> Result<ParallelResult> {
10971 match (self.value, self.failure) {
10972 (Some(value), None) => Ok(value),
10973 (_, Some(error)) => Err(error),
10974 _ => Err(Error::WorkerLoop(
10975 "selection result contained neither a value nor a failure".to_string(),
10976 )),
10977 }
10978 }
10979}
10980
10981pub struct SelectCall {
10983 ctx: WorkflowContext,
10984 operations: Option<Vec<(SelectionKey, ParallelOperation)>>,
10985 members: Vec<SelectionMemberPlan>,
10986 leaves: Vec<SelectionLeaf>,
10987 group_id: Option<String>,
10988}
10989
10990impl SelectCall {
10991 fn new(ctx: WorkflowContext, operations: Vec<(SelectionKey, ParallelOperation)>) -> Self {
10992 Self {
10993 ctx,
10994 operations: Some(operations),
10995 members: Vec::new(),
10996 leaves: Vec::new(),
10997 group_id: None,
10998 }
10999 }
11000
11001 fn initialize(&mut self) -> Result<()> {
11002 let operations = self.operations.take().unwrap_or_default();
11003 let base_sequence = {
11004 let state = self
11005 .ctx
11006 .state
11007 .lock()
11008 .map_err(|_| Error::WorkflowStatePoisoned)?;
11009 if let Some(marker) = state.selection_markers.get(state.selection_marker_cursor) {
11010 marker.selection_group_base_sequence
11011 } else if let Some(recorded) = state.recorded_commands.get(state.command_cursor) {
11012 recorded.sequence()
11013 } else {
11014 let last = state
11015 .recorded_commands
11016 .last()
11017 .map(RecordedCommand::sequence)
11018 .unwrap_or(0);
11019 last.checked_add(u64::try_from(state.commands.len()).unwrap_or(u64::MAX))
11020 .and_then(|sequence| sequence.checked_add(1))
11021 .ok_or(Error::TimerDurationOverflow)?
11022 }
11023 };
11024 let (descriptors, members) = selection_descriptors(operations, base_sequence)?;
11025 let group_id = format!("select-calls:{base_sequence}:{}", descriptors.len());
11026 self.leaves = descriptors
11027 .into_iter()
11028 .map(|descriptor| SelectionLeaf {
11029 call: parallel_leaf_call(&self.ctx, descriptor.operation, descriptor.group_path),
11030 outcome: None,
11031 })
11032 .collect();
11033 self.members = members;
11034 self.group_id = Some(group_id);
11035 Ok(())
11036 }
11037}
11038
11039impl Future for SelectCall {
11040 type Output = Result<SelectionResult>;
11041
11042 fn poll(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11043 if self.operations.is_some() {
11044 if let Err(error) = self.initialize() {
11045 return Poll::Ready(Err(error));
11046 }
11047 }
11048
11049 for leaf in &mut self.leaves {
11050 if leaf.outcome.is_some() {
11051 continue;
11052 }
11053 if let Poll::Ready(outcome) = leaf.call.poll_avro_value(cx) {
11054 if outcome
11055 .as_ref()
11056 .err()
11057 .is_some_and(workflow_task_integrity_error)
11058 {
11059 return Poll::Ready(outcome.map(|_| unreachable!()));
11060 }
11061 leaf.outcome = Some(outcome);
11062 }
11063 }
11064
11065 let all_members_terminal = self.leaves.iter().all(|leaf| leaf.outcome.is_some());
11066 let selection_member_range = self
11067 .members
11068 .first()
11069 .map(|member| member.base_sequence)
11070 .zip(self.leaves.len().try_into().ok())
11071 .map(|(base_sequence, size): (u64, u64)| {
11072 base_sequence..base_sequence.saturating_add(size)
11073 });
11074 let marker = {
11075 let mut state = match self.ctx.state.lock() {
11076 Ok(state) => state,
11077 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11078 };
11079 let marker = state
11080 .selection_markers
11081 .get(state.selection_marker_cursor)
11082 .cloned();
11083 if marker.is_none()
11084 && all_members_terminal
11085 && selection_member_range.as_ref().is_some_and(|member_range| {
11086 state
11087 .recorded_commands
11088 .iter()
11089 .any(|command| member_range.contains(&command.sequence()))
11090 })
11091 {
11092 state.matched_recorded_pending = true;
11098 }
11099 marker
11100 };
11101 let Some(marker) = marker else {
11102 return Poll::Pending;
11103 };
11104 if self.group_id.as_deref() != Some(marker.selection_group_id.as_str())
11105 || marker.selection_group_size != self.leaves.len()
11106 || self.members.first().map(|member| member.base_sequence)
11107 != Some(marker.selection_group_base_sequence)
11108 {
11109 return Poll::Ready(Err(invalid_recorded_history(
11110 "selection_group_shape_mismatch",
11111 marker.selection_group_base_sequence,
11112 self.group_id
11113 .as_deref()
11114 .unwrap_or("initialized selection group"),
11115 &marker.selection_group_id,
11116 "recorded selection group differs from current workflow code",
11117 )));
11118 }
11119 let Some(member_position) = self.members.iter().position(|member| {
11120 member.key == marker.member_key
11121 && member.index == marker.member_index
11122 && member.base_sequence == marker.member_base_sequence
11123 && member.size == marker.member_size
11124 && member.kind == marker.operation_kind
11125 }) else {
11126 return Poll::Ready(Err(invalid_recorded_history(
11127 "selection_member_shape_mismatch",
11128 marker.member_base_sequence,
11129 "winner member matching current workflow code",
11130 &format!("{:?}", marker.member_key),
11131 "recorded selection winner differs from the authored member identity",
11132 )));
11133 };
11134 let member = self.members[member_position].clone();
11135 let (handles, resolution_sequence) = {
11136 let mut state = match self.ctx.state.lock() {
11137 Ok(state) => state,
11138 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11139 };
11140 let identities = self
11141 .members
11142 .iter()
11143 .map(|candidate| {
11144 selection_operation_identity(
11145 &state,
11146 &candidate.kind,
11147 candidate.base_sequence,
11148 candidate.size,
11149 )
11150 })
11151 .collect::<Vec<_>>();
11152 if let Some((position, missing)) = identities
11153 .iter()
11154 .enumerate()
11155 .find(|(_, identity)| identity.is_empty())
11156 .map(|(position, identity)| (position, identity.clone()))
11157 {
11158 let candidate = &self.members[position];
11159 return Poll::Ready(Err(invalid_recorded_history(
11160 "selection_operation_identity_missing",
11161 candidate.base_sequence,
11162 &format!(
11163 "durable {} resource identity from scheduled/open history",
11164 candidate.kind
11165 ),
11166 &missing,
11167 "selection member history is missing its canonical durable identity",
11168 )));
11169 }
11170 let expected_winner_identity = &identities[member_position];
11171 let resolution_sequence = match validated_selection_resolution_sequence(
11172 &state,
11173 &marker,
11174 &member,
11175 expected_winner_identity,
11176 ) {
11177 Ok(sequence) => sequence,
11178 Err(error) => return Poll::Ready(Err(error)),
11179 };
11180 let handles = self
11181 .members
11182 .iter()
11183 .zip(identities)
11184 .map(|(member, identity)| DurableOperationHandle {
11185 ctx: self.ctx.clone(),
11186 key: member.key.clone(),
11187 index: member.index,
11188 kind: member.kind.clone(),
11189 identity,
11190 base_sequence: member.base_sequence,
11191 size: member.size,
11192 selection_group_id: marker.selection_group_id.clone(),
11193 shape: member.shape.clone(),
11194 })
11195 .collect::<Vec<_>>();
11196 if let Err(error) = validate_selection_cancellations_for_handles(&state, &handles) {
11197 return Poll::Ready(Err(error));
11198 }
11199 state.selection_marker_cursor += 1;
11200 (handles, resolution_sequence)
11201 };
11202
11203 let mut winner_failure = None;
11204 let mut flat_results = Vec::with_capacity(member.size);
11205 if marker.outcome == "failed" {
11206 let resolution_offset = match resolution_sequence
11207 .checked_sub(member.base_sequence)
11208 .and_then(|offset| usize::try_from(offset).ok())
11209 {
11210 Some(offset) if offset < member.size => offset,
11211 _ => {
11212 return Poll::Ready(Err(invalid_recorded_history(
11213 "selection_resolution_event_mismatch",
11214 member.base_sequence,
11215 "failure event within selected member bounds",
11216 &resolution_sequence.to_string(),
11217 "selection failure event is outside the authored member",
11218 )))
11219 }
11220 };
11221 let leaf = &mut self.leaves[member.leaf_start + resolution_offset];
11222 match leaf.outcome.take() {
11223 Some(Err(error)) => winner_failure = Some(error),
11224 _ => {
11225 return Poll::Ready(Err(invalid_recorded_history(
11226 "selection_winner_outcome_mismatch",
11227 member.base_sequence,
11228 "exact failed terminal history referenced by SelectionResolved",
11229 "missing or successful resolution event",
11230 "selection winner marker disagrees with terminal operation history",
11231 )))
11232 }
11233 }
11234 } else {
11235 for leaf in &mut self.leaves[member.leaf_start..member.leaf_start + member.size] {
11236 match leaf.outcome.take() {
11237 Some(Ok(result)) => flat_results.push(result),
11238 Some(Err(_)) => {
11239 return Poll::Ready(Err(invalid_recorded_history(
11240 "selection_winner_outcome_mismatch",
11241 member.base_sequence,
11242 "fully completed nested selection member",
11243 "failed durable leaf",
11244 "completed selection winner contains a failed leaf",
11245 )))
11246 }
11247 None => {
11248 return Poll::Ready(Err(invalid_recorded_history(
11249 "selection_winner_unresolved",
11250 member.base_sequence,
11251 "terminal history for every completed winner leaf",
11252 "pending member history",
11253 "completed SelectionResolved member has an unfinished durable barrier",
11254 )))
11255 }
11256 }
11257 }
11258 }
11259 let value = if winner_failure.is_none() {
11260 let mut flat_results = flat_results.into_iter();
11261 let value = parallel_results_for_shape(&member.shape, &mut flat_results);
11262 match value.into_json_result() {
11263 Ok(value) => Some(value),
11264 Err(error) => return Poll::Ready(Err(error)),
11265 }
11266 } else {
11267 None
11268 };
11269 let winner = handles[member_position].clone();
11270 Poll::Ready(Ok(SelectionResult {
11271 key: winner.key.clone(),
11272 index: winner.index,
11273 kind: winner.kind.clone(),
11274 identity: winner.identity.clone(),
11275 value,
11276 failure: winner_failure,
11277 winner,
11278 handles,
11279 }))
11280 }
11281}
11282
11283fn selection_operation_identity(
11284 state: &WorkflowState,
11285 kind: &str,
11286 base_sequence: u64,
11287 size: usize,
11288) -> String {
11289 if kind == "group" {
11290 return format!("group:{base_sequence}:{size}");
11291 }
11292 let fields: &[&str] = match kind {
11293 "activity" => &["activity_execution_id"],
11294 "child" => &["child_workflow_run_id"],
11295 "timer" => &["timer_id"],
11296 "signal" => &["signal_wait_id"],
11297 "condition" => &["condition_wait_id"],
11298 _ => &[],
11299 };
11300 for sequence in base_sequence..base_sequence.saturating_add(size as u64) {
11301 for event in state
11302 .history_events
11303 .iter()
11304 .filter(|event| durable_event_sequence(event) == Some(sequence))
11305 {
11306 for field in fields {
11307 if let Some(identity) = event.payload.get(*field).and_then(Value::as_str) {
11308 if !identity.is_empty() {
11309 return identity.to_string();
11310 }
11311 }
11312 }
11313 }
11314 }
11315 String::new()
11316}
11317
11318fn validated_selection_resolution_sequence(
11319 state: &WorkflowState,
11320 marker: &SelectionMarker,
11321 member: &SelectionMemberPlan,
11322 expected_identity: &str,
11323) -> Result<u64> {
11324 if expected_identity.is_empty() {
11325 return Err(invalid_recorded_history(
11326 "selection_operation_identity_missing",
11327 member.base_sequence,
11328 &format!(
11329 "durable {} resource identity from scheduled/open history",
11330 member.kind
11331 ),
11332 "missing operation identity",
11333 "selection member history is missing its canonical durable identity",
11334 ));
11335 }
11336 if marker.operation_identity != expected_identity {
11337 return Err(invalid_recorded_history(
11338 "selection_operation_identity_mismatch",
11339 member.base_sequence,
11340 expected_identity,
11341 &marker.operation_identity,
11342 "selection winner identity does not match durable scheduled/open history",
11343 ));
11344 }
11345
11346 let failure_types = [
11347 "ActivityFailed",
11348 "ActivityCancelled",
11349 "ActivityTimedOut",
11350 "ChildRunFailed",
11351 "ChildRunCancelled",
11352 "ChildRunTerminated",
11353 ];
11354 let success_types = [
11355 "ActivityCompleted",
11356 "ChildRunCompleted",
11357 "TimerFired",
11358 "SignalApplied",
11359 "ConditionWaitSatisfied",
11360 "ConditionWaitTimedOut",
11361 ];
11362 let terminal_types: &[&str] = if marker.outcome == "failed" {
11363 &failure_types
11364 } else {
11365 &success_types
11366 };
11367 let mut candidates = Vec::new();
11368 for event in state.history_events.iter() {
11369 let Some(sequence) = durable_event_sequence(event) else {
11370 continue;
11371 };
11372 if sequence < member.base_sequence
11373 || sequence >= member.base_sequence.saturating_add(member.size as u64)
11374 || !terminal_types.contains(&event.event_type.as_str())
11375 {
11376 continue;
11377 }
11378 let event_id = event
11379 .raw
11380 .get("id")
11381 .or_else(|| event.raw.get("event_id"))
11382 .and_then(Value::as_str)
11383 .filter(|value| !value.is_empty())
11384 .ok_or_else(|| {
11385 invalid_recorded_history(
11386 "selection_resolution_event_id_missing",
11387 member.base_sequence,
11388 "terminal selection history with a durable event id",
11389 &event.payload.to_string(),
11390 "selection terminal history cannot be bound to its winner marker",
11391 )
11392 })?;
11393 candidates.push((event_id.to_string(), event.event_type.clone(), sequence));
11394 }
11395 let resolution = if marker.outcome == "failed" {
11396 candidates.first()
11397 } else {
11398 candidates.last()
11399 };
11400 let Some((event_id, event_type, sequence)) = resolution else {
11401 return Err(invalid_recorded_history(
11402 "selection_resolution_event_missing",
11403 member.base_sequence,
11404 "terminal history for the selected member",
11405 &format!("{:?}", marker.member_key),
11406 "selection winner marker has no matching durable terminal event",
11407 ));
11408 };
11409 if event_id != &marker.resolution_event_id || event_type != &marker.resolution_event_type {
11410 return Err(invalid_recorded_history(
11411 "selection_resolution_event_mismatch",
11412 member.base_sequence,
11413 &format!("{event_type}:{event_id}"),
11414 &format!(
11415 "{}:{}",
11416 marker.resolution_event_type, marker.resolution_event_id
11417 ),
11418 "selection winner marker does not reference the event that made its member terminal",
11419 ));
11420 }
11421 Ok(*sequence)
11422}
11423
11424fn recorded_selection_member_outcome(
11425 state: &WorkflowState,
11426 handle: &DurableOperationHandle,
11427) -> Result<Option<ParallelResult>> {
11428 for event in state.history_events.iter() {
11429 let Some(sequence) = durable_event_sequence(event) else {
11430 continue;
11431 };
11432 if sequence < handle.base_sequence
11433 || sequence >= handle.base_sequence.saturating_add(handle.size as u64)
11434 || !matches!(
11435 event.event_type.as_str(),
11436 "ActivityFailed"
11437 | "ActivityCancelled"
11438 | "ActivityTimedOut"
11439 | "ChildRunFailed"
11440 | "ChildRunCancelled"
11441 | "ChildRunTerminated"
11442 )
11443 {
11444 continue;
11445 }
11446 let Some(command) = state
11447 .recorded_commands
11448 .iter()
11449 .find(|command| command.sequence() == sequence)
11450 else {
11451 continue;
11452 };
11453 match command {
11454 RecordedCommand::Activity {
11455 outcome: Some(Err(failure)),
11456 ..
11457 } => return Err(Error::ActivityFailed(failure.clone())),
11458 RecordedCommand::ChildWorkflow {
11459 outcome: Some(Err(failure)),
11460 ..
11461 } => return Err(Error::ChildWorkflowFailed(failure.clone())),
11462 _ => {}
11463 }
11464 }
11465
11466 let mut results = Vec::with_capacity(handle.size);
11467 for sequence in handle.base_sequence..handle.base_sequence.saturating_add(handle.size as u64) {
11468 let Some(command) = state
11469 .recorded_commands
11470 .iter()
11471 .find(|command| command.sequence() == sequence)
11472 else {
11473 return Ok(None);
11474 };
11475 let result = match command {
11476 RecordedCommand::Activity { outcome, .. } => match outcome {
11477 Some(Ok(value)) => ParallelAvroResult::Activity(value.clone()),
11478 Some(Err(failure)) => return Err(Error::ActivityFailed(failure.clone())),
11479 None => return Ok(None),
11480 },
11481 RecordedCommand::Timer { fired, .. } => {
11482 if !fired {
11483 return Ok(None);
11484 }
11485 ParallelAvroResult::Timer
11486 }
11487 RecordedCommand::ChildWorkflow { outcome, .. } => match outcome {
11488 Some(Ok(value)) => ParallelAvroResult::ChildWorkflow(value.clone()),
11489 Some(Err(failure)) => return Err(Error::ChildWorkflowFailed(failure.clone())),
11490 None => return Ok(None),
11491 },
11492 RecordedCommand::SignalWait { value, .. } => match value {
11493 Some(value) => ParallelAvroResult::Signal(value.clone()),
11494 None => return Ok(None),
11495 },
11496 RecordedCommand::ConditionWait { result, .. } => match result {
11497 Some(result) => ParallelAvroResult::Condition(*result),
11498 None => return Ok(None),
11499 },
11500 other => {
11501 return Err(command_mismatch(
11502 other,
11503 format!("selected {} member", handle.kind),
11504 ))
11505 }
11506 };
11507 results.push(result);
11508 }
11509 let mut results = results.into_iter();
11510 parallel_results_for_shape(&handle.shape, &mut results)
11511 .into_json_result()
11512 .map(Some)
11513}
11514
11515fn recorded_selection_member_is_terminal(
11516 state: &WorkflowState,
11517 handle: &DurableOperationHandle,
11518) -> bool {
11519 let mut completed = 0usize;
11520 let mut all_completed = true;
11521 for sequence in handle.base_sequence..handle.base_sequence.saturating_add(handle.size as u64) {
11522 let Some(command) = state
11523 .recorded_commands
11524 .iter()
11525 .find(|command| command.sequence() == sequence)
11526 else {
11527 all_completed = false;
11528 continue;
11529 };
11530 let terminal = match command {
11531 RecordedCommand::Activity {
11532 outcome: Some(Err(_)),
11533 ..
11534 }
11535 | RecordedCommand::ChildWorkflow {
11536 outcome: Some(Err(_)),
11537 ..
11538 } => return true,
11539 RecordedCommand::Activity { outcome, .. } => outcome.is_some(),
11540 RecordedCommand::ChildWorkflow { outcome, .. } => outcome.is_some(),
11541 RecordedCommand::Timer { fired, .. } => *fired,
11542 RecordedCommand::SignalWait { value, .. } => value.is_some(),
11543 RecordedCommand::ConditionWait { result, .. } => result.is_some(),
11544 RecordedCommand::SearchAttributes { .. }
11545 | RecordedCommand::SideEffect { .. }
11546 | RecordedCommand::VersionMarker { .. }
11547 | RecordedCommand::Memo { .. } => false,
11548 };
11549 if !terminal {
11550 all_completed = false;
11551 continue;
11552 }
11553 completed += 1;
11554 }
11555 all_completed && completed == handle.size
11556}
11557
11558fn selection_cancellation_for_handle(
11559 state: &WorkflowState,
11560 handle: &DurableOperationHandle,
11561) -> Result<bool> {
11562 let Some(marker) = state.cancelled_selection_members.iter().find(|recorded| {
11563 recorded.selection_group_id == handle.selection_group_id
11564 && recorded.member_base_sequence == handle.base_sequence
11565 }) else {
11566 return Ok(false);
11567 };
11568 validate_selection_cancellation_marker(marker, handle)?;
11569 Ok(true)
11570}
11571
11572fn validate_selection_cancellations_for_handles(
11573 state: &WorkflowState,
11574 handles: &[DurableOperationHandle],
11575) -> Result<()> {
11576 let Some(group_id) = handles
11577 .first()
11578 .map(|handle| handle.selection_group_id.as_str())
11579 else {
11580 return Ok(());
11581 };
11582 for marker in state
11583 .cancelled_selection_members
11584 .iter()
11585 .filter(|marker| marker.selection_group_id == group_id)
11586 {
11587 let Some(handle) = handles
11588 .iter()
11589 .find(|handle| handle.base_sequence == marker.member_base_sequence)
11590 else {
11591 return Err(invalid_recorded_history(
11592 "selection_cancellation_member_mismatch",
11593 marker.member_base_sequence,
11594 "SelectionOperationCancelled matching an authored selection handle",
11595 &format!("{marker:?}"),
11596 "selection cancellation member base does not name an authored member",
11597 ));
11598 };
11599 validate_selection_cancellation_marker(marker, handle)?;
11600 }
11601 Ok(())
11602}
11603
11604fn validate_selection_cancellation_marker(
11605 marker: &SelectionCancellation,
11606 handle: &DurableOperationHandle,
11607) -> Result<()> {
11608 if marker.selection_group_id != handle.selection_group_id
11609 || marker.member_key != handle.key
11610 || marker.member_index != handle.index
11611 || marker.member_base_sequence != handle.base_sequence
11612 || marker.member_size != handle.size
11613 || marker.operation_kind != handle.kind
11614 || marker.operation_identity != handle.identity
11615 {
11616 return Err(invalid_recorded_history(
11617 "selection_cancellation_member_mismatch",
11618 handle.base_sequence,
11619 "SelectionOperationCancelled matching the authored selection handle",
11620 &format!("{marker:?}"),
11621 "selection cancellation history targets different authored member metadata",
11622 ));
11623 }
11624 Ok(())
11625}
11626
11627pub struct DurableOperationAwaitCall {
11629 handle: DurableOperationHandle,
11630}
11631
11632impl Future for DurableOperationAwaitCall {
11633 type Output = Result<ParallelResult>;
11634
11635 fn poll(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11636 let state = match self.handle.ctx.state.lock() {
11637 Ok(state) => state,
11638 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11639 };
11640 match selection_cancellation_for_handle(&state, &self.handle) {
11641 Err(error) => return Poll::Ready(Err(error)),
11642 Ok(false) => {}
11643 Ok(true) => {
11644 return Poll::Ready(Err(Error::DurableOperationCancelled(
11645 DurableOperationCancelled {
11646 selection_group_id: self.handle.selection_group_id.clone(),
11647 member_key: self.handle.key.clone(),
11648 member_index: self.handle.index,
11649 operation_kind: self.handle.kind.clone(),
11650 operation_identity: self.handle.identity.clone(),
11651 },
11652 )));
11653 }
11654 }
11655 match recorded_selection_member_outcome(&state, &self.handle) {
11656 Ok(Some(result)) => Poll::Ready(Ok(result)),
11657 Ok(None) => Poll::Pending,
11658 Err(error) => Poll::Ready(Err(error)),
11659 }
11660 }
11661}
11662
11663pub struct CancelDurableOperationCall {
11665 handle: DurableOperationHandle,
11666 emitted: bool,
11667}
11668
11669impl Future for CancelDurableOperationCall {
11670 type Output = Result<()>;
11671
11672 fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11673 let ctx = self.handle.ctx.clone();
11674 let mut state = match ctx.state.lock() {
11675 Ok(state) => state,
11676 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11677 };
11678 match selection_cancellation_for_handle(&state, &self.handle) {
11679 Err(error) => return Poll::Ready(Err(error)),
11680 Ok(true) => return Poll::Ready(Ok(())),
11681 Ok(false) => {}
11682 }
11683 if recorded_selection_member_is_terminal(&state, &self.handle) {
11684 return Poll::Ready(Ok(()));
11685 }
11686 if !self.emitted {
11687 state.commands.push(json!({
11688 "type": "cancel_selection_operation",
11689 "selection_group_id": self.handle.selection_group_id,
11690 "member_key": self.handle.key,
11691 "member_index": self.handle.index,
11692 "member_base_sequence": self.handle.base_sequence,
11693 "member_size": self.handle.size,
11694 "operation_kind": self.handle.kind,
11695 "operation_identity": self.handle.identity,
11696 }));
11697 self.emitted = true;
11698 }
11699 Poll::Pending
11703 }
11704}
11705
11706trait PollNestedResultExt<T> {
11707 fn flatten_result(self) -> Poll<Result<T>>;
11708}
11709
11710impl<T> PollNestedResultExt<T> for Poll<Result<Result<T>>> {
11711 fn flatten_result(self) -> Poll<Result<T>> {
11712 match self {
11713 Poll::Ready(Ok(result)) => Poll::Ready(result),
11714 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
11715 Poll::Pending => Poll::Pending,
11716 }
11717 }
11718}
11719
11720struct SagaCompensation {
11721 activity_type: String,
11722 options: ActivityOptions,
11723 arguments: AvroValue,
11724 registration_order: usize,
11725}
11726
11727pub struct Saga {
11735 ctx: WorkflowContext,
11736 compensations: Vec<SagaCompensation>,
11737}
11738
11739impl Saga {
11740 fn new(ctx: WorkflowContext) -> Self {
11741 Self {
11742 ctx,
11743 compensations: Vec::new(),
11744 }
11745 }
11746
11747 pub fn add_compensation<T: Serialize>(
11748 &mut self,
11749 activity_type: impl Into<String>,
11750 args: T,
11751 ) -> Result<&mut Self> {
11752 self.add_compensation_with_options(activity_type, ActivityOptions::new(), args)
11753 }
11754
11755 pub fn add_compensation_with_options<T: Serialize>(
11756 &mut self,
11757 activity_type: impl Into<String>,
11758 options: ActivityOptions,
11759 args: T,
11760 ) -> Result<&mut Self> {
11761 let activity_type = activity_type.into();
11762 if activity_type.trim().is_empty() || activity_type.trim() != activity_type {
11763 return Err(Error::Codec(
11764 "saga compensation activity type must be non-empty without surrounding whitespace"
11765 .to_string(),
11766 ));
11767 }
11768 options.validate().map_err(Error::InvalidActivityOptions)?;
11769 let arguments = AvroValue::from_serialize(&args)?;
11770 let registration_order = self.compensations.len() + 1;
11771 self.compensations.push(SagaCompensation {
11772 activity_type,
11773 options,
11774 arguments,
11775 registration_order,
11776 });
11777 Ok(self)
11778 }
11779
11780 pub async fn compensate(mut self, initiating_failure: Error) -> Error {
11782 while let Some(compensation) = self.compensations.pop() {
11783 if let Err(compensation_failure) = self
11784 .ctx
11785 .activity_with_options(
11786 compensation.activity_type.clone(),
11787 compensation.options,
11788 compensation.arguments,
11789 )
11790 .await
11791 {
11792 if workflow_task_integrity_error(&compensation_failure) {
11793 return compensation_failure;
11794 }
11795 return Error::SagaCompensationFailed(SagaCompensationFailure {
11796 initiating_failure: Box::new(initiating_failure),
11797 compensation_failure: Box::new(compensation_failure),
11798 compensation_activity_type: compensation.activity_type,
11799 compensation_registration_order: compensation.registration_order,
11800 });
11801 }
11802 }
11803 initiating_failure
11804 }
11805
11806 pub async fn finish<T>(self, outcome: Result<T>) -> Result<T> {
11808 match outcome {
11809 Ok(value) => Ok(value),
11810 Err(error) => Err(self.compensate(error).await),
11811 }
11812 }
11813}
11814
11815pub struct ActivityCall {
11816 ctx: WorkflowContext,
11817 activity_type: String,
11818 options: ActivityOptions,
11819 args: Option<Result<AvroValue>>,
11820 scheduled: bool,
11821 parallel_group_path: Vec<ParallelGroupMetadata>,
11822}
11823
11824impl ActivityCall {
11825 fn poll_avro_value(
11826 mut self: Pin<&mut Self>,
11827 _cx: &mut TaskContext<'_>,
11828 ) -> Poll<Result<AvroValue>> {
11829 let ctx = self.ctx.clone();
11830 let mut state = match ctx.state.lock() {
11831 Ok(state) => state,
11832 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11833 };
11834
11835 if self.scheduled {
11836 return Poll::Pending;
11837 }
11838
11839 let options = match self.options.validate() {
11840 Ok(options) => options,
11841 Err(error) => {
11842 return Poll::Ready(Err(Error::InvalidActivityOptions(error)));
11843 }
11844 };
11845 let task_queue = options
11846 .task_queue
11847 .clone()
11848 .unwrap_or_else(|| state.task_queue.clone());
11849 let current_recorded_options = RecordedActivityOptions {
11850 task_queue: RecordedSnapshotValue::Known(Some(task_queue.clone())),
11851 execution_mode: RecordedSnapshotValue::Known(None),
11854 retry_policy: current_activity_retry_snapshot(&options),
11855 };
11856
11857 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
11858 let sequence = recorded.sequence();
11859 match recorded {
11860 RecordedCommand::Activity {
11861 activity_type,
11862 options: recorded_options,
11863 outcome,
11864 parallel_group_path,
11865 ..
11866 } => {
11867 if let Err(error) = ensure_parallel_path_matches(
11868 sequence,
11869 parallel_group_path.as_deref(),
11870 &self.parallel_group_path,
11871 ) {
11872 return Poll::Ready(Err(error));
11873 }
11874 if let Some(recorded_type) = activity_type {
11875 if recorded_type != self.activity_type {
11876 return Poll::Ready(Err(Error::NonDeterministicReplay(
11877 ReplayFailure::new(
11878 "recorded_command_detail_mismatch",
11879 Some(sequence),
11880 Some(format!("activity:{recorded_type}")),
11881 Some(format!("activity:{}", self.activity_type)),
11882 "recorded activity type differs from the current workflow command",
11883 ),
11884 )));
11885 }
11886 }
11887 if let Some(recorded_options) = recorded_options {
11888 if !recorded_options
11889 .task_queue
11890 .matches_current(¤t_recorded_options.task_queue)
11891 {
11892 return Poll::Ready(Err(Error::NonDeterministicReplay(
11893 ReplayFailure::new(
11894 "activity_task_queue_mismatch",
11895 Some(sequence),
11896 Some(activity_options_description(&recorded_options)),
11897 Some(activity_options_description(¤t_recorded_options)),
11898 "recorded activity task queue differs from the current workflow command",
11899 ),
11900 )));
11901 }
11902 if !recorded_options
11903 .execution_mode
11904 .matches_current(¤t_recorded_options.execution_mode)
11905 {
11906 return Poll::Ready(Err(Error::NonDeterministicReplay(
11907 ReplayFailure::new(
11908 "activity_execution_mode_mismatch",
11909 Some(sequence),
11910 Some(activity_options_description(&recorded_options)),
11911 Some(activity_options_description(¤t_recorded_options)),
11912 "recorded activity execution mode differs from the current workflow command",
11913 ),
11914 )));
11915 }
11916 if !recorded_options
11917 .retry_policy
11918 .matches_current(¤t_recorded_options.retry_policy)
11919 {
11920 return Poll::Ready(Err(Error::NonDeterministicReplay(
11921 ReplayFailure::new(
11922 "activity_retry_policy_mismatch",
11923 Some(sequence),
11924 Some(activity_options_description(&recorded_options)),
11925 Some(activity_options_description(¤t_recorded_options)),
11926 "recorded activity retry policy differs from the current workflow command",
11927 ),
11928 )));
11929 }
11930 }
11931 state.command_cursor += 1;
11932 if let Some(outcome) = outcome {
11933 return Poll::Ready(outcome.map_err(Error::ActivityFailed));
11934 }
11935 state.matched_recorded_pending = true;
11936 self.scheduled = true;
11937 return Poll::Pending;
11938 }
11939 other => {
11940 return Poll::Ready(Err(command_mismatch(
11941 &other,
11942 format!("activity:{}", self.activity_type),
11943 )));
11944 }
11945 }
11946 }
11947
11948 if !self.scheduled {
11949 let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
11950 Ok(args) => args,
11951 Err(error) => return Poll::Ready(Err(error)),
11952 };
11953 let arguments = normalize_avro_arguments(args);
11954 let envelope = match encode_typed_envelope(&arguments, &state.payload_codec) {
11955 Ok(envelope) => envelope,
11956 Err(error) => return Poll::Ready(Err(error)),
11957 };
11958
11959 let mut command = serde_json::Map::from_iter([
11960 ("type".to_string(), json!("schedule_activity")),
11961 (
11962 "activity_type".to_string(),
11963 json!(self.activity_type.clone()),
11964 ),
11965 ("queue".to_string(), json!(task_queue)),
11966 ("arguments".to_string(), envelope),
11967 ]);
11968 for (field, value) in [
11969 ("start_to_close_timeout", options.start_to_close_timeout),
11970 (
11971 "schedule_to_start_timeout",
11972 options.schedule_to_start_timeout,
11973 ),
11974 (
11975 "schedule_to_close_timeout",
11976 options.schedule_to_close_timeout,
11977 ),
11978 ("heartbeat_timeout", options.heartbeat_timeout),
11979 ] {
11980 if let Some(value) = value {
11981 command.insert(field.to_string(), json!(value));
11982 }
11983 }
11984 if let Some(retry_policy) = options.retry_policy {
11985 command.insert("retry_policy".to_string(), retry_policy);
11986 }
11987 apply_parallel_group_path(&mut command, &self.parallel_group_path);
11988 state.commands.push(Value::Object(command));
11989 self.scheduled = true;
11990 }
11991
11992 Poll::Pending
11993 }
11994}
11995
11996impl Future for ActivityCall {
11997 type Output = Result<Value>;
11998
11999 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12000 match self.poll_avro_value(cx) {
12001 Poll::Ready(Ok(value)) => Poll::Ready(value.into_json()),
12002 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
12003 Poll::Pending => Poll::Pending,
12004 }
12005 }
12006}
12007
12008pub struct TimerCall {
12010 ctx: WorkflowContext,
12011 delay_seconds: Option<u64>,
12012 scheduled: bool,
12013 matched_pending: bool,
12014 parallel_group_path: Vec<ParallelGroupMetadata>,
12015}
12016
12017impl Future for TimerCall {
12018 type Output = Result<()>;
12019
12020 fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12021 if self.matched_pending {
12022 return Poll::Pending;
12023 }
12024
12025 let ctx = self.ctx.clone();
12026 let Some(requested_delay) = self.delay_seconds else {
12027 return Poll::Ready(Err(Error::TimerDurationOverflow));
12028 };
12029 let mut state = match ctx.state.lock() {
12030 Ok(state) => state,
12031 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12032 };
12033
12034 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
12035 match recorded {
12036 RecordedCommand::Timer {
12037 sequence,
12038 delay_seconds,
12039 fired,
12040 parallel_group_path,
12041 ..
12042 } => {
12043 if let Err(error) = ensure_parallel_path_matches(
12044 sequence,
12045 parallel_group_path.as_deref(),
12046 &self.parallel_group_path,
12047 ) {
12048 return Poll::Ready(Err(error));
12049 }
12050 if delay_seconds != requested_delay {
12051 return Poll::Ready(Err(Error::NonDeterministicReplay(
12052 ReplayFailure::new(
12053 "timer_delay_mismatch",
12054 Some(sequence),
12055 Some(format!("timer:{delay_seconds}s")),
12056 Some(format!("timer:{requested_delay}s")),
12057 "recorded timer delay differs from the current workflow command",
12058 ),
12059 )));
12060 }
12061 state.command_cursor += 1;
12062 if fired {
12063 return Poll::Ready(Ok(()));
12064 }
12065 state.matched_recorded_pending = true;
12066 self.scheduled = true;
12067 self.matched_pending = true;
12068 return Poll::Pending;
12069 }
12070 other => return Poll::Ready(Err(command_mismatch(&other, "timer"))),
12071 }
12072 }
12073
12074 if !self.scheduled {
12075 let mut command = serde_json::Map::from_iter([
12076 ("type".to_string(), json!("start_timer")),
12077 ("delay_seconds".to_string(), json!(requested_delay)),
12078 ]);
12079 apply_parallel_group_path(&mut command, &self.parallel_group_path);
12080 state.commands.push(Value::Object(command));
12081 self.scheduled = true;
12082 }
12083
12084 Poll::Pending
12085 }
12086}
12087
12088pub struct ConditionWaitCall {
12090 ctx: WorkflowContext,
12091 options: ConditionWaitOptions,
12092 predicate: Box<dyn Fn() -> Result<bool> + Send + 'static>,
12093 occurrence_id: Option<String>,
12094 opened_wait: bool,
12095 parallel_group_path: Vec<ParallelGroupMetadata>,
12096}
12097
12098impl Future for ConditionWaitCall {
12099 type Output = Result<ConditionWaitResult>;
12100
12101 fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12102 if self.opened_wait {
12103 return Poll::Pending;
12104 }
12105
12106 let options = match self.options.validate() {
12107 Ok(options) => options,
12108 Err(error) => return Poll::Ready(Err(Error::InvalidConditionWaitOptions(error))),
12109 };
12110 let ctx = self.ctx.clone();
12111 let occurrence_id = match self.occurrence_id.as_ref() {
12112 Some(occurrence_id) => occurrence_id.clone(),
12113 None => {
12114 let mut state = match ctx.state.lock() {
12115 Ok(state) => state,
12116 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12117 };
12118 let ordinal = state.condition_wait_occurrence_counter;
12119 state.condition_wait_occurrence_counter = match ordinal.checked_add(1) {
12120 Some(next) => next,
12121 None => {
12122 return Poll::Ready(Err(Error::WorkerLoop(
12123 "condition wait occurrence counter overflowed".to_string(),
12124 )))
12125 }
12126 };
12127 let occurrence_id = format!("{CONDITION_WAIT_OCCURRENCE_PREFIX}{ordinal}");
12128 drop(state);
12129 self.occurrence_id = Some(occurrence_id.clone());
12130 occurrence_id
12131 }
12132 };
12133
12134 let recorded_result = {
12135 let mut state = match ctx.state.lock() {
12136 Ok(state) => state,
12137 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12138 };
12139 let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() else {
12140 drop(state);
12141 return self.poll_new_condition(options);
12142 };
12143 if !matches!(recorded, RecordedCommand::ConditionWait { .. }) {
12144 return Poll::Ready(Err(command_mismatch(&recorded, "condition wait")));
12145 }
12146
12147 let mut cursor = state.command_cursor;
12148 let mut result = None;
12149 loop {
12150 let Some(RecordedCommand::ConditionWait {
12151 sequence,
12152 occurrence_id: recorded_occurrence_id,
12153 condition_key,
12154 predicate_identity,
12155 timeout_seconds,
12156 result: recorded_result,
12157 parallel_group_path,
12158 ..
12159 }) = state.recorded_commands.get(cursor)
12160 else {
12161 break;
12162 };
12163
12164 if cursor > state.command_cursor && recorded_occurrence_id != &occurrence_id {
12165 break;
12166 }
12167 if let Err(error) = ensure_parallel_path_matches(
12168 *sequence,
12169 parallel_group_path.as_deref(),
12170 &self.parallel_group_path,
12171 ) {
12172 return Poll::Ready(Err(error));
12173 }
12174 if let Err(error) = validate_recorded_condition_wait(
12175 *sequence,
12176 recorded_occurrence_id,
12177 condition_key.as_deref(),
12178 predicate_identity,
12179 *timeout_seconds,
12180 &occurrence_id,
12181 &options,
12182 ) {
12183 return Poll::Ready(Err(error));
12184 }
12185 if result == Some(ConditionWaitResult::TimedOut) {
12186 return Poll::Ready(Err(Error::NonDeterministicReplay(ReplayFailure::new(
12187 "condition_wait_reopened_after_timeout",
12188 Some(*sequence),
12189 Some("timed-out condition is terminal".to_string()),
12190 Some("another physical wait-open".to_string()),
12191 "condition history reopened one logical wait after its durable timeout",
12192 ))));
12193 }
12194 result = *recorded_result;
12195 cursor += 1;
12196 }
12197 state.command_cursor = cursor;
12198 result
12199 };
12200
12201 if let Some(result) = recorded_result {
12202 return Poll::Ready(Ok(result));
12203 }
12204
12205 self.poll_open_condition(options)
12206 }
12207}
12208
12209impl ConditionWaitCall {
12210 fn poll_new_condition(
12211 self: Pin<&mut Self>,
12212 options: ValidatedConditionWaitOptions,
12213 ) -> Poll<Result<ConditionWaitResult>> {
12214 self.poll_open_condition(options)
12215 }
12216
12217 fn poll_open_condition(
12218 mut self: Pin<&mut Self>,
12219 options: ValidatedConditionWaitOptions,
12220 ) -> Poll<Result<ConditionWaitResult>> {
12221 let selection_member = self
12222 .parallel_group_path
12223 .first()
12224 .is_some_and(|entry| entry.parallel_group_mode.as_deref() == Some("select"));
12225 match (self.predicate)() {
12226 Ok(true) if !selection_member => {
12227 return Poll::Ready(Ok(ConditionWaitResult::Satisfied))
12228 }
12229 Ok(_) => {}
12230 Err(error) => return Poll::Ready(Err(error)),
12231 }
12232 if options.timeout_seconds == Some(0) && !selection_member {
12233 return Poll::Ready(Ok(ConditionWaitResult::TimedOut));
12234 }
12235
12236 let ctx = self.ctx.clone();
12237 let mut state = match ctx.state.lock() {
12238 Ok(state) => state,
12239 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12240 };
12241 let mut command = serde_json::Map::from_iter([
12242 ("type".to_string(), json!("open_condition_wait")),
12243 (
12244 "condition_wait_occurrence_id".to_string(),
12245 json!(self.occurrence_id.as_deref().unwrap_or_default()),
12246 ),
12247 ("condition_key".to_string(), json!(options.condition_key)),
12248 (
12249 "condition_definition_fingerprint".to_string(),
12250 json!(options.predicate_identity),
12251 ),
12252 ]);
12253 if let Some(timeout_seconds) = options.timeout_seconds {
12254 command.insert("timeout_seconds".to_string(), json!(timeout_seconds));
12255 }
12256 apply_parallel_group_path(&mut command, &self.parallel_group_path);
12257 state.commands.push(Value::Object(command));
12258 drop(state);
12259 self.opened_wait = true;
12260 Poll::Pending
12261 }
12262}
12263
12264fn validate_recorded_condition_wait(
12265 sequence: u64,
12266 recorded_occurrence_id: &str,
12267 recorded_key: Option<&str>,
12268 recorded_predicate_identity: &str,
12269 recorded_timeout_seconds: Option<u64>,
12270 current_occurrence_id: &str,
12271 current: &ValidatedConditionWaitOptions,
12272) -> Result<()> {
12273 if recorded_occurrence_id != current_occurrence_id {
12274 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12275 "condition_wait_occurrence_mismatch",
12276 Some(sequence),
12277 Some(recorded_occurrence_id.to_string()),
12278 Some(current_occurrence_id.to_string()),
12279 "recorded condition occurrence differs from the current authored wait position",
12280 )));
12281 }
12282 if recorded_key != Some(current.condition_key.as_str()) {
12283 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12284 "condition_wait_key_mismatch",
12285 Some(sequence),
12286 recorded_key.map(str::to_string),
12287 Some(current.condition_key.clone()),
12288 "recorded condition identity differs from the current workflow wait",
12289 )));
12290 }
12291 if recorded_predicate_identity != current.predicate_identity {
12292 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12293 "condition_wait_predicate_mismatch",
12294 Some(sequence),
12295 Some(recorded_predicate_identity.to_string()),
12296 Some(current.predicate_identity.clone()),
12297 "recorded condition predicate behavior differs from current workflow code",
12298 )));
12299 }
12300 if recorded_timeout_seconds != current.timeout_seconds {
12301 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12302 "condition_wait_timeout_mismatch",
12303 Some(sequence),
12304 recorded_timeout_seconds.map(|seconds| format!("{seconds}s")),
12305 current.timeout_seconds.map(|seconds| format!("{seconds}s")),
12306 "recorded condition timeout differs from the current workflow wait",
12307 )));
12308 }
12309 Ok(())
12310}
12311
12312pub struct ChildWorkflowCall {
12314 ctx: WorkflowContext,
12315 workflow_type: String,
12316 options: ChildWorkflowOptions,
12317 args: Option<Result<AvroValue>>,
12318 scheduled: bool,
12319 matched_pending: bool,
12320 parallel_group_path: Vec<ParallelGroupMetadata>,
12321}
12322
12323impl ChildWorkflowCall {
12324 fn poll_avro_value(
12325 mut self: Pin<&mut Self>,
12326 _cx: &mut TaskContext<'_>,
12327 ) -> Poll<Result<ChildWorkflowAvroResult>> {
12328 if self.matched_pending {
12329 return Poll::Pending;
12330 }
12331
12332 let ctx = self.ctx.clone();
12333 let mut state = match ctx.state.lock() {
12334 Ok(state) => state,
12335 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12336 };
12337
12338 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
12339 let sequence = recorded.sequence();
12340 match recorded {
12341 RecordedCommand::ChildWorkflow {
12342 workflow_type,
12343 outcome,
12344 parallel_group_path,
12345 ..
12346 } => {
12347 if let Err(error) = ensure_parallel_path_matches(
12348 sequence,
12349 parallel_group_path.as_deref(),
12350 &self.parallel_group_path,
12351 ) {
12352 return Poll::Ready(Err(error));
12353 }
12354 if let Some(recorded_type) = workflow_type {
12355 if recorded_type != self.workflow_type {
12356 return Poll::Ready(Err(Error::NonDeterministicReplay(
12357 ReplayFailure::new(
12358 "recorded_command_detail_mismatch",
12359 Some(sequence),
12360 Some(format!("child workflow:{recorded_type}")),
12361 Some(format!("child workflow:{}", self.workflow_type)),
12362 "recorded child workflow type differs from the current workflow command",
12363 ),
12364 )));
12365 }
12366 }
12367 state.command_cursor += 1;
12368 if let Some(outcome) = outcome {
12369 return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed));
12370 }
12371 state.matched_recorded_pending = true;
12372 self.scheduled = true;
12373 self.matched_pending = true;
12374 return Poll::Pending;
12375 }
12376 other => {
12377 return Poll::Ready(Err(command_mismatch(
12378 &other,
12379 format!("child workflow:{}", self.workflow_type),
12380 )));
12381 }
12382 }
12383 }
12384
12385 if !self.scheduled {
12386 if self.options.task_queue.trim().is_empty() {
12387 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
12388 "task_queue must not be empty".to_string(),
12389 )));
12390 }
12391 for (name, value) in [
12392 (
12393 "execution_timeout_seconds",
12394 self.options.execution_timeout_seconds,
12395 ),
12396 ("run_timeout_seconds", self.options.run_timeout_seconds),
12397 ] {
12398 if value == Some(0) {
12399 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(format!(
12400 "{name} must be at least 1"
12401 ))));
12402 }
12403 }
12404
12405 let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
12406 Ok(args) => args,
12407 Err(error) => return Poll::Ready(Err(error)),
12408 };
12409 let arguments = match encode_typed_envelope(
12410 &normalize_avro_arguments(args),
12411 &state.payload_codec,
12412 ) {
12413 Ok(arguments) => arguments,
12414 Err(error) => return Poll::Ready(Err(error)),
12415 };
12416 let mut command = json!({
12417 "type": "start_child_workflow",
12418 "workflow_type": self.workflow_type,
12419 "queue": self.options.task_queue,
12420 "parent_close_policy": self.options.parent_close_policy.as_str(),
12421 "arguments": arguments,
12422 });
12423 let object = command
12424 .as_object_mut()
12425 .expect("child workflow command is always an object");
12426 if let Some(policy) = &self.options.retry_policy {
12427 let mut retry_policy = serde_json::Map::new();
12428 if let Some(max_attempts) = policy.max_attempts {
12429 if max_attempts == 0 {
12430 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
12431 "retry_policy.max_attempts must be at least 1".to_string(),
12432 )));
12433 }
12434 retry_policy.insert("max_attempts".to_string(), json!(max_attempts));
12435 }
12436 if !policy.backoff_seconds.is_empty() {
12437 retry_policy
12438 .insert("backoff_seconds".to_string(), json!(policy.backoff_seconds));
12439 }
12440 if !policy.non_retryable_error_types.is_empty() {
12441 retry_policy.insert(
12442 "non_retryable_error_types".to_string(),
12443 json!(policy.non_retryable_error_types),
12444 );
12445 }
12446 if retry_policy.is_empty() {
12447 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
12448 "retry_policy must configure at least one field".to_string(),
12449 )));
12450 }
12451 object.insert("retry_policy".to_string(), Value::Object(retry_policy));
12452 }
12453 if let Some(seconds) = self.options.execution_timeout_seconds {
12454 object.insert("execution_timeout_seconds".to_string(), json!(seconds));
12455 }
12456 if let Some(seconds) = self.options.run_timeout_seconds {
12457 object.insert("run_timeout_seconds".to_string(), json!(seconds));
12458 }
12459 apply_parallel_group_path(object, &self.parallel_group_path);
12460 state.commands.push(command);
12461 self.scheduled = true;
12462 }
12463
12464 Poll::Pending
12465 }
12466}
12467
12468impl Future for ChildWorkflowCall {
12469 type Output = Result<ChildWorkflowResult>;
12470
12471 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12472 match self.poll_avro_value(cx) {
12473 Poll::Ready(Ok(result)) => match result.result.into_json() {
12474 Ok(projected) => Poll::Ready(Ok(ChildWorkflowResult {
12475 parent: result.parent,
12476 child: result.child,
12477 child_workflow_type: result.child_workflow_type,
12478 result: projected,
12479 })),
12480 Err(error) => Poll::Ready(Err(error)),
12481 },
12482 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
12483 Poll::Pending => Poll::Pending,
12484 }
12485 }
12486}
12487
12488fn command_mismatch(recorded: &RecordedCommand, actual: impl Into<String>) -> Error {
12489 Error::NonDeterministicReplay(ReplayFailure::new(
12490 "recorded_command_mismatch",
12491 Some(recorded.sequence()),
12492 Some(recorded.shape().to_string()),
12493 Some(actual.into()),
12494 "current workflow command does not match the recorded durable command sequence",
12495 ))
12496}
12497
12498pub struct SignalCall {
12499 ctx: WorkflowContext,
12500 signal_name: String,
12501 runtime_reserved_allowed: bool,
12502 opened_wait: bool,
12503 matched_pending: bool,
12504 parallel_group_path: Vec<ParallelGroupMetadata>,
12505}
12506
12507impl SignalCall {
12508 fn poll_avro_value(
12509 mut self: Pin<&mut Self>,
12510 _cx: &mut TaskContext<'_>,
12511 ) -> Poll<Result<Vec<AvroValue>>> {
12512 if self.matched_pending {
12513 return Poll::Pending;
12514 }
12515 if !self.runtime_reserved_allowed {
12516 if let Err(error) = validate_user_signal_name(&self.signal_name) {
12517 return Poll::Ready(Err(error));
12518 }
12519 }
12520
12521 let ctx = self.ctx.clone();
12522 let mut state = match ctx.state.lock() {
12523 Ok(state) => state,
12524 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12525 };
12526
12527 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
12528 match recorded {
12529 RecordedCommand::SignalWait {
12530 sequence,
12531 signal_name,
12532 value,
12533 parallel_group_path,
12534 } => {
12535 if let Err(error) = ensure_parallel_path_matches(
12536 sequence,
12537 parallel_group_path.as_deref(),
12538 &self.parallel_group_path,
12539 ) {
12540 return Poll::Ready(Err(error));
12541 }
12542 if signal_name != self.signal_name {
12543 return Poll::Ready(Err(Error::NonDeterministicReplay(
12544 ReplayFailure::new(
12545 "recorded_command_detail_mismatch",
12546 Some(sequence),
12547 Some(format!("signal wait:{signal_name}")),
12548 Some(format!("signal wait:{}", self.signal_name)),
12549 "recorded signal name differs from the current workflow command",
12550 ),
12551 )));
12552 }
12553
12554 state.command_cursor += 1;
12555 if let Some(value) = value {
12556 return Poll::Ready(Ok(value));
12557 }
12558 if state
12559 .resume_signal
12560 .as_ref()
12561 .is_some_and(|signal| signal.signal_name == self.signal_name)
12562 {
12563 let signal = state
12564 .resume_signal
12565 .take()
12566 .expect("matching resume signal is present");
12567 return Poll::Ready(Ok(signal.arguments));
12568 }
12569
12570 state.matched_recorded_pending = true;
12571 self.opened_wait = true;
12572 self.matched_pending = true;
12573 return Poll::Pending;
12574 }
12575 other => {
12576 return Poll::Ready(Err(command_mismatch(
12577 &other,
12578 format!("signal wait:{}", self.signal_name),
12579 )));
12580 }
12581 }
12582 }
12583
12584 if state
12585 .resume_signal
12586 .as_ref()
12587 .is_some_and(|signal| signal.signal_name == self.signal_name)
12588 {
12589 let signal = state
12590 .resume_signal
12591 .take()
12592 .expect("matching resume signal is present");
12593 return Poll::Ready(Ok(signal.arguments));
12594 }
12595
12596 if !self.opened_wait {
12597 let mut command = serde_json::Map::from_iter([
12598 ("type".to_string(), json!("open_signal_wait")),
12599 ("signal_name".to_string(), json!(self.signal_name)),
12600 ]);
12601 apply_parallel_group_path(&mut command, &self.parallel_group_path);
12602 state.commands.push(Value::Object(command));
12603 self.opened_wait = true;
12604 }
12605
12606 Poll::Pending
12607 }
12608}
12609
12610impl Future for SignalCall {
12611 type Output = Result<Vec<Value>>;
12612
12613 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12614 match self.poll_avro_value(cx) {
12615 Poll::Ready(Ok(values)) => Poll::Ready(
12616 values
12617 .into_iter()
12618 .map(AvroValue::into_json)
12619 .collect::<Result<Vec<_>>>(),
12620 ),
12621 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
12622 Poll::Pending => Poll::Pending,
12623 }
12624 }
12625}
12626
12627#[derive(Clone, Debug)]
12628pub struct ActivityContext {
12629 client: Client,
12630 pub task_id: String,
12631 pub activity_attempt_id: String,
12632 pub lease_owner: String,
12633 pub activity_type: String,
12634 pub attempt_number: u64,
12635 pub task_queue: String,
12636 pub worker_id: String,
12637}
12638
12639impl ActivityContext {
12640 pub async fn heartbeat<T: Serialize>(&self, details: T) -> Result<ActivityHeartbeatResponse> {
12641 self.client
12642 .heartbeat_activity_task(
12643 &self.task_id,
12644 &self.activity_attempt_id,
12645 &self.lease_owner,
12646 details,
12647 )
12648 .await
12649 }
12650}
12651
12652fn decode_task_avro_arguments(value: Option<&Value>, codec: &str) -> Result<AvroValue> {
12653 validate_payload_codec(codec)?;
12654 match value {
12655 Some(value) => Ok(normalize_avro_arguments(decode_wire_avro_value(
12656 value, codec,
12657 )?)),
12658 None => Ok(AvroValue::Array(Vec::new())),
12659 }
12660}
12661
12662fn decode_resume_signal(task: &WorkflowTask) -> Result<Option<ResumeSignal>> {
12663 let Some(signal_name) = task
12664 .signal_name
12665 .as_deref()
12666 .filter(|value| !value.is_empty())
12667 else {
12668 return Ok(None);
12669 };
12670 let decoded = decode_task_avro_arguments(task.signal_arguments.as_ref(), &task.payload_codec)?;
12671 let AvroValue::Array(arguments) = decoded else {
12672 unreachable!("normalize_avro_arguments always returns an array");
12673 };
12674
12675 Ok(Some(ResumeSignal {
12676 signal_name: signal_name.to_string(),
12677 arguments,
12678 }))
12679}
12680
12681fn validate_workflow_task_payloads(task: &WorkflowTask) -> Result<()> {
12682 validate_payload_codec(&task.payload_codec)?;
12683 validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)?;
12684 validate_optional_inbound_payload(task.signal_arguments.as_ref(), &task.payload_codec)?;
12685 for event in &task.history_events {
12686 validate_history_event_payloads(event, &task.payload_codec)?;
12687 }
12688 Ok(())
12689}
12690
12691fn validate_activity_task_payloads(task: &ActivityTask) -> Result<()> {
12692 validate_payload_codec(&task.payload_codec)?;
12693 validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)
12694}
12695
12696fn validate_query_task_payloads(task: &QueryTask) -> Result<()> {
12697 validate_payload_codec(&task.payload_codec)?;
12698 validate_optional_inbound_payload(task.workflow_arguments.as_ref(), &task.payload_codec)?;
12699 validate_optional_inbound_payload(task.query_arguments.as_ref(), &task.payload_codec)?;
12700 for event in &task.history_events {
12701 validate_history_event_payloads(event, &task.payload_codec)?;
12702 }
12703
12704 let Some(export) = task.history_export.as_ref() else {
12705 return Ok(());
12706 };
12707 let export_codec = match export.get("payloads") {
12708 Some(payloads) => declared_payload_codec(payloads, "codec")?,
12709 None => None,
12710 }
12711 .unwrap_or(&task.payload_codec);
12712 validate_payload_codec(export_codec)?;
12713
12714 if let Some(events) = export.get("history_events").and_then(Value::as_array) {
12715 for event in events {
12716 let event_type = event
12717 .get("event_type")
12718 .or_else(|| event.get("type"))
12719 .and_then(Value::as_str)
12720 .unwrap_or_default();
12721 if let Some(payload) = event.get("payload") {
12722 validate_history_payloads(event_type, payload, export_codec)?;
12723 }
12724 }
12725 }
12726 for signal in export
12727 .get("signals")
12728 .and_then(Value::as_array)
12729 .into_iter()
12730 .flatten()
12731 {
12732 let codec = declared_payload_codec(signal, "payload_codec")?.unwrap_or(export_codec);
12733 validate_payload_codec(codec)?;
12734 validate_optional_inbound_payload(signal.get("arguments"), codec)?;
12735 }
12736 for activity in export
12737 .get("activities")
12738 .and_then(Value::as_array)
12739 .into_iter()
12740 .flatten()
12741 {
12742 let codec = declared_payload_codec(activity, "payload_codec")?.unwrap_or(export_codec);
12743 validate_payload_codec(codec)?;
12744 validate_optional_inbound_payload(activity.get("arguments"), codec)?;
12745 validate_optional_inbound_payload(activity.get("result"), codec)?;
12746 }
12747 Ok(())
12748}
12749
12750fn validate_history_event_payloads(event: &HistoryEvent, fallback_codec: &str) -> Result<()> {
12751 validate_history_payloads(&event.event_type, &event.payload, fallback_codec)
12752}
12753
12754fn validate_history_payloads(
12755 event_type: &str,
12756 payload: &Value,
12757 fallback_codec: &str,
12758) -> Result<()> {
12759 let codec = declared_payload_codec(payload, "payload_codec")?.unwrap_or(fallback_codec);
12760 validate_payload_codec(codec)?;
12761 for field in history_payload_fields(event_type) {
12762 validate_optional_inbound_payload(payload.get(*field), codec)?;
12763 }
12764 Ok(())
12765}
12766
12767const SIGNAL_HISTORY_PAYLOAD_FIELDS: &[&str] = &["value", "input", "arguments"];
12768
12769fn history_payload_fields(event_type: &str) -> &'static [&'static str] {
12770 match event_type {
12771 "ActivityCompleted" => &["result"],
12772 "SignalReceived" | "SignalApplied" => SIGNAL_HISTORY_PAYLOAD_FIELDS,
12773 "UpdateAccepted" | "UpdateRejected" | "UpdateApplied" => &["arguments"],
12774 "UpdateCompleted" | "SideEffectRecorded" => &["result"],
12775 "ChildRunCompleted" => &["result", "output"],
12776 "WorkflowCompleted" => &["output"],
12777 "ServiceCallStarted"
12778 | "ServiceCallCompleted"
12779 | "ServiceCallFailed"
12780 | "ServiceCallCancelled" => &["request_payload", "response_payload"],
12781 _ => &[],
12782 }
12783}
12784
12785fn signal_history_payload(payload: &Value) -> Option<&Value> {
12786 SIGNAL_HISTORY_PAYLOAD_FIELDS
12787 .iter()
12788 .find_map(|field| payload.get(*field))
12789}
12790
12791fn declared_payload_codec<'a>(value: &'a Value, field: &str) -> Result<Option<&'a str>> {
12792 match value.get(field) {
12793 None => Ok(None),
12794 Some(Value::String(codec)) => Ok(Some(codec)),
12795 Some(_) => Err(invalid_payload_envelope()),
12796 }
12797}
12798
12799fn validate_optional_inbound_payload(value: Option<&Value>, codec: &str) -> Result<()> {
12800 validate_payload_codec(codec)?;
12801 if let Some(value) = value.filter(|value| !value.is_null()) {
12802 decode_wire_avro_value(value, codec)?;
12803 }
12804 Ok(())
12805}
12806
12807fn recorded_parallel_group_entry(payload: &Value, sequence: u64) -> Result<ParallelGroupMetadata> {
12808 let group_id = payload_string(payload, "parallel_group_id").ok_or_else(|| {
12809 invalid_recorded_history(
12810 "parallel_group_metadata_invalid",
12811 sequence,
12812 "non-empty parallel_group_id",
12813 &payload.to_string(),
12814 "parallel-group history is missing its stable identity",
12815 )
12816 })?;
12817 let kind = payload_string(payload, "parallel_group_kind").ok_or_else(|| {
12818 invalid_recorded_history(
12819 "parallel_group_metadata_invalid",
12820 sequence,
12821 "activity, child, timer, signal, condition, or mixed group kind",
12822 &payload.to_string(),
12823 "parallel-group history is missing its group kind",
12824 )
12825 })?;
12826 if !matches!(
12827 kind.as_str(),
12828 "activity" | "child" | "timer" | "signal" | "condition" | "mixed"
12829 ) {
12830 return Err(invalid_recorded_history(
12831 "parallel_group_metadata_invalid",
12832 sequence,
12833 "activity, child, timer, signal, condition, or mixed group kind",
12834 &kind,
12835 "parallel-group history contains an unsupported group kind",
12836 ));
12837 }
12838 let base_sequence = payload
12839 .get("parallel_group_base_sequence")
12840 .and_then(value_as_u64)
12841 .filter(|value| *value > 0)
12842 .ok_or_else(|| {
12843 invalid_recorded_history(
12844 "parallel_group_metadata_invalid",
12845 sequence,
12846 "positive parallel_group_base_sequence",
12847 &payload.to_string(),
12848 "parallel-group history contains an invalid base sequence",
12849 )
12850 })?;
12851 let size = payload
12852 .get("parallel_group_size")
12853 .and_then(value_as_u64)
12854 .and_then(|value| usize::try_from(value).ok())
12855 .filter(|value| (1..=MAX_PARALLEL_OPERATIONS).contains(value))
12856 .ok_or_else(|| {
12857 invalid_recorded_history(
12858 "parallel_group_metadata_invalid",
12859 sequence,
12860 "bounded positive parallel_group_size",
12861 &payload.to_string(),
12862 "parallel-group history contains an invalid group size",
12863 )
12864 })?;
12865 let index = payload
12866 .get("parallel_group_index")
12867 .and_then(value_as_u64)
12868 .and_then(|value| usize::try_from(value).ok())
12869 .filter(|value| *value < size)
12870 .ok_or_else(|| {
12871 invalid_recorded_history(
12872 "parallel_group_metadata_invalid",
12873 sequence,
12874 "parallel_group_index within group bounds",
12875 &payload.to_string(),
12876 "parallel-group history contains an invalid member index",
12877 )
12878 })?;
12879 if base_sequence.checked_add(u64::try_from(index).unwrap_or(u64::MAX)) != Some(sequence) {
12880 return Err(invalid_recorded_history(
12881 "parallel_group_metadata_invalid",
12882 sequence,
12883 "base sequence plus member index equals workflow sequence",
12884 &payload.to_string(),
12885 "parallel-group path does not preserve durable workflow position",
12886 ));
12887 }
12888 let mode = payload
12889 .get("parallel_group_mode")
12890 .and_then(Value::as_str)
12891 .unwrap_or("all");
12892 if !matches!(mode, "all" | "select") {
12893 return Err(invalid_recorded_history(
12894 "parallel_group_metadata_invalid",
12895 sequence,
12896 "parallel group mode all or select",
12897 mode,
12898 "parallel-group history contains an unsupported group mode",
12899 ));
12900 }
12901 let expected_id = if mode == "select" {
12902 format!("select-calls:{base_sequence}:{size}")
12903 } else {
12904 format!("{}:{base_sequence}:{size}", parallel_group_prefix(&kind))
12905 };
12906 if group_id != expected_id {
12907 return Err(invalid_recorded_history(
12908 "parallel_group_metadata_invalid",
12909 sequence,
12910 &expected_id,
12911 &group_id,
12912 "parallel-group history contains an incompatible stable group ID",
12913 ));
12914 }
12915 let selection_member_key = if mode == "select" {
12916 Some(selection_key_from_value(
12917 payload.get("selection_member_key"),
12918 sequence,
12919 )?)
12920 } else {
12921 None
12922 };
12923 let selection_member_index = if mode == "select" {
12924 Some(required_parallel_usize(
12925 payload,
12926 "selection_member_index",
12927 sequence,
12928 )?)
12929 } else {
12930 None
12931 };
12932 let selection_member_base_sequence = if mode == "select" {
12933 Some(
12934 payload
12935 .get("selection_member_base_sequence")
12936 .and_then(value_as_u64)
12937 .filter(|value| *value >= base_sequence)
12938 .ok_or_else(|| {
12939 invalid_recorded_history(
12940 "parallel_group_metadata_invalid",
12941 sequence,
12942 "selection member base within its group",
12943 &payload.to_string(),
12944 "selection history contains an invalid member base sequence",
12945 )
12946 })?,
12947 )
12948 } else {
12949 None
12950 };
12951 let selection_member_size = if mode == "select" {
12952 let member_size = required_parallel_usize(payload, "selection_member_size", sequence)?;
12953 if member_size == 0 {
12954 return Err(invalid_recorded_history(
12955 "parallel_group_metadata_invalid",
12956 sequence,
12957 "positive selection member size",
12958 &payload.to_string(),
12959 "selection history contains an invalid member size",
12960 ));
12961 }
12962 Some(member_size)
12963 } else {
12964 None
12965 };
12966 let selection_member_kind = if mode == "select" {
12967 let kind = payload_string(payload, "selection_member_kind").ok_or_else(|| {
12968 invalid_recorded_history(
12969 "parallel_group_metadata_invalid",
12970 sequence,
12971 "selection member operation kind",
12972 &payload.to_string(),
12973 "selection history is missing its authored member kind",
12974 )
12975 })?;
12976 if !matches!(
12977 kind.as_str(),
12978 "activity" | "child" | "timer" | "signal" | "condition" | "group"
12979 ) {
12980 return Err(invalid_recorded_history(
12981 "parallel_group_metadata_invalid",
12982 sequence,
12983 "activity, child, timer, signal, condition, or group selection member kind",
12984 &kind,
12985 "selection history contains an unsupported member kind",
12986 ));
12987 }
12988 Some(kind)
12989 } else {
12990 None
12991 };
12992 if let (Some(member_base), Some(member_size)) =
12993 (selection_member_base_sequence, selection_member_size)
12994 {
12995 let member_end = member_base
12996 .checked_add(u64::try_from(member_size).unwrap_or(u64::MAX))
12997 .ok_or_else(|| {
12998 invalid_recorded_history(
12999 "parallel_group_metadata_invalid",
13000 sequence,
13001 "bounded selection member range",
13002 &payload.to_string(),
13003 "selection member range overflowed",
13004 )
13005 })?;
13006 let group_end = base_sequence
13007 .checked_add(u64::try_from(size).unwrap_or(u64::MAX))
13008 .unwrap_or(u64::MAX);
13009 if sequence < member_base || sequence >= member_end || member_end > group_end {
13010 return Err(invalid_recorded_history(
13011 "parallel_group_metadata_invalid",
13012 sequence,
13013 "workflow sequence within one bounded selection member",
13014 &payload.to_string(),
13015 "selection member range does not contain its durable leaf",
13016 ));
13017 }
13018 }
13019 Ok(ParallelGroupMetadata {
13020 parallel_group_id: group_id,
13021 parallel_group_kind: kind,
13022 parallel_group_base_sequence: base_sequence,
13023 parallel_group_size: size,
13024 parallel_group_index: index,
13025 parallel_group_mode: (mode == "select").then(|| "select".to_string()),
13026 selection_member_key,
13027 selection_member_index,
13028 selection_member_base_sequence,
13029 selection_member_size,
13030 selection_member_kind,
13031 })
13032}
13033
13034fn required_parallel_usize(payload: &Value, field: &str, sequence: u64) -> Result<usize> {
13035 payload
13036 .get(field)
13037 .and_then(value_as_u64)
13038 .and_then(|value| usize::try_from(value).ok())
13039 .ok_or_else(|| {
13040 invalid_recorded_history(
13041 "parallel_group_metadata_invalid",
13042 sequence,
13043 &format!("non-negative integer {field}"),
13044 &payload.to_string(),
13045 "selection history contains invalid member metadata",
13046 )
13047 })
13048}
13049
13050fn selection_key_from_value(value: Option<&Value>, sequence: u64) -> Result<SelectionKey> {
13051 match value {
13052 Some(Value::String(value)) if !value.is_empty() => Ok(SelectionKey::Name(value.clone())),
13053 Some(value) => value_as_u64(value)
13054 .and_then(|value| usize::try_from(value).ok())
13055 .map(SelectionKey::Index)
13056 .ok_or_else(|| {
13057 invalid_recorded_history(
13058 "selection_member_key_invalid",
13059 sequence,
13060 "non-empty string or non-negative integer member key",
13061 &value.to_string(),
13062 "selection history contains an invalid member key",
13063 )
13064 }),
13065 None => Err(invalid_recorded_history(
13066 "selection_member_key_missing",
13067 sequence,
13068 "selection_member_key",
13069 "<missing>",
13070 "selection history is missing its stable member key",
13071 )),
13072 }
13073}
13074
13075fn recorded_parallel_group_path(
13076 events: &[&HistoryEvent],
13077 sequence: u64,
13078) -> Result<Option<Vec<ParallelGroupMetadata>>> {
13079 let mut recorded: Option<Vec<ParallelGroupMetadata>> = None;
13080 for event in events {
13081 let payload = &event.payload;
13082 let has_metadata = payload.get("parallel_group_path").is_some()
13083 || payload.get("parallel_group_id").is_some()
13084 || payload.get("parallel_group_kind").is_some()
13085 || payload.get("parallel_group_base_sequence").is_some()
13086 || payload.get("parallel_group_size").is_some()
13087 || payload.get("parallel_group_index").is_some()
13088 || payload.get("parallel_group_mode").is_some()
13089 || payload.get("selection_member_key").is_some();
13090 if !has_metadata {
13091 continue;
13092 }
13093
13094 let top_level = recorded_parallel_group_entry(payload, sequence)?;
13095 let path = match payload.get("parallel_group_path") {
13096 None => vec![top_level.clone()],
13097 Some(Value::Array(entries)) if !entries.is_empty() => entries
13098 .iter()
13099 .map(|entry| recorded_parallel_group_entry(entry, sequence))
13100 .collect::<Result<Vec<_>>>()?,
13101 Some(value) => {
13102 return Err(invalid_recorded_history(
13103 "parallel_group_metadata_invalid",
13104 sequence,
13105 "non-empty parallel_group_path list",
13106 &value.to_string(),
13107 "parallel-group history contains an invalid group path",
13108 ));
13109 }
13110 };
13111 if path.last() != Some(&top_level) {
13112 return Err(invalid_recorded_history(
13113 "parallel_group_metadata_invalid",
13114 sequence,
13115 &serde_json::to_string(&path.last()).unwrap_or_default(),
13116 &serde_json::to_string(&top_level).unwrap_or_default(),
13117 "parallel-group top-level fields do not match the innermost path entry",
13118 ));
13119 }
13120 if recorded.as_ref().is_some_and(|existing| existing != &path) {
13121 return Err(invalid_recorded_history(
13122 "parallel_group_history_conflict",
13123 sequence,
13124 &serde_json::to_string(&recorded.as_ref()).unwrap_or_default(),
13125 &serde_json::to_string(&path).unwrap_or_default(),
13126 "parallel-group metadata changed between scheduling and resolution history",
13127 ));
13128 }
13129 recorded = Some(path);
13130 }
13131 Ok(recorded)
13132}
13133
13134fn recorded_commands(
13135 events: &[HistoryEvent],
13136 fallback_codec: &str,
13137 parent: WorkflowIdentity,
13138) -> Result<Vec<RecordedCommand>> {
13139 let mut events_by_sequence: BTreeMap<u64, Vec<&HistoryEvent>> = BTreeMap::new();
13140 let mut last_new_sequence = None;
13141
13142 for event in events {
13143 let is_activity = matches!(
13144 event.event_type.as_str(),
13145 "ActivityScheduled"
13146 | "ActivityStarted"
13147 | "ActivityHeartbeatRecorded"
13148 | "ActivityRetryScheduled"
13149 | "ActivityCompleted"
13150 | "ActivityFailed"
13151 | "ActivityCancelled"
13152 | "ActivityTimedOut"
13153 );
13154 let is_workflow_timer = matches!(
13155 event.event_type.as_str(),
13156 "TimerScheduled" | "TimerCancelled" | "TimerFired"
13157 ) && !is_internal_timer_event(event);
13158 let is_child_workflow = matches!(
13159 event.event_type.as_str(),
13160 "ChildWorkflowScheduled"
13161 | "ChildRunCompleted"
13162 | "ChildRunFailed"
13163 | "ChildRunCancelled"
13164 | "ChildRunTerminated"
13165 );
13166 let is_signal_wait = is_recorded_signal_wait_event(event);
13167 let is_condition_wait = is_recorded_condition_wait_event(event);
13168 let is_search_attributes = event.event_type == "SearchAttributesUpserted";
13169 let is_side_effect = event.event_type == "SideEffectRecorded";
13170 let is_version_marker = event.event_type == "VersionMarkerRecorded";
13171 let is_memo = event.event_type == "MemoUpserted";
13172 if !is_activity
13173 && !is_workflow_timer
13174 && !is_child_workflow
13175 && !is_signal_wait
13176 && !is_condition_wait
13177 && !is_search_attributes
13178 && !is_side_effect
13179 && !is_version_marker
13180 && !is_memo
13181 {
13182 continue;
13183 }
13184
13185 let sequence = durable_event_sequence(event).ok_or_else(|| {
13186 Error::NonDeterministicReplay(ReplayFailure::new(
13187 "durable_command_sequence_missing",
13188 None,
13189 Some("positive workflow sequence".to_string()),
13190 Some(event.event_type.clone()),
13191 "durable command history event has no workflow sequence",
13192 ))
13193 })?;
13194 if sequence == 0 {
13195 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
13196 "durable_command_sequence_invalid",
13197 Some(sequence),
13198 Some("positive workflow sequence".to_string()),
13199 Some(sequence.to_string()),
13200 "durable command history uses an invalid workflow sequence",
13201 )));
13202 }
13203 if !events_by_sequence.contains_key(&sequence) {
13204 if let Some(previous) = last_new_sequence {
13205 if sequence < previous {
13206 return Err(invalid_recorded_history(
13207 "durable_command_sequence_mismatch",
13208 sequence,
13209 &format!("workflow sequence greater than {previous}"),
13210 &sequence.to_string(),
13211 "durable commands are not strictly ordered by their recorded workflow sequence",
13212 ));
13213 }
13214 }
13215 last_new_sequence = Some(sequence);
13216 }
13217 events_by_sequence.entry(sequence).or_default().push(event);
13218 }
13219
13220 let commands: Vec<RecordedCommand> = events_by_sequence
13221 .into_iter()
13222 .map(|(sequence, sequence_events)| {
13223 let activity_events: Vec<_> = sequence_events
13224 .iter()
13225 .copied()
13226 .filter(|event| event.event_type.starts_with("Activity"))
13227 .collect();
13228 let timer_events: Vec<_> = sequence_events
13229 .iter()
13230 .copied()
13231 .filter(|event| event.event_type.starts_with("Timer"))
13232 .collect();
13233 let child_events: Vec<_> = sequence_events
13234 .iter()
13235 .copied()
13236 .filter(|event| {
13237 event.event_type == "ChildWorkflowScheduled"
13238 || event.event_type.starts_with("ChildRun")
13239 })
13240 .collect();
13241 let signal_wait_events: Vec<_> = sequence_events
13242 .iter()
13243 .copied()
13244 .filter(|event| is_recorded_signal_wait_event(event))
13245 .collect();
13246 let condition_wait_events: Vec<_> = sequence_events
13247 .iter()
13248 .copied()
13249 .filter(|event| is_recorded_condition_wait_event(event))
13250 .collect();
13251 let search_attribute_events: Vec<_> = sequence_events
13252 .iter()
13253 .copied()
13254 .filter(|event| event.event_type == "SearchAttributesUpserted")
13255 .collect();
13256 let side_effect_events: Vec<_> = sequence_events
13257 .iter()
13258 .copied()
13259 .filter(|event| event.event_type == "SideEffectRecorded")
13260 .collect();
13261 let version_marker_events: Vec<_> = sequence_events
13262 .iter()
13263 .copied()
13264 .filter(|event| event.event_type == "VersionMarkerRecorded")
13265 .collect();
13266 let memo_events: Vec<_> = sequence_events
13267 .iter()
13268 .copied()
13269 .filter(|event| event.event_type == "MemoUpserted")
13270 .collect();
13271
13272 let command_kind_count = usize::from(!activity_events.is_empty())
13273 + usize::from(!timer_events.is_empty())
13274 + usize::from(!child_events.is_empty())
13275 + usize::from(!signal_wait_events.is_empty())
13276 + usize::from(!condition_wait_events.is_empty())
13277 + usize::from(!search_attribute_events.is_empty())
13278 + usize::from(!side_effect_events.is_empty())
13279 + usize::from(!version_marker_events.is_empty())
13280 + usize::from(!memo_events.is_empty());
13281 if command_kind_count > 1 {
13282 let actual = [
13283 (!activity_events.is_empty()).then_some("activity"),
13284 (!timer_events.is_empty()).then_some("timer"),
13285 (!child_events.is_empty()).then_some("child workflow"),
13286 (!signal_wait_events.is_empty()).then_some("signal wait"),
13287 (!condition_wait_events.is_empty()).then_some("condition wait"),
13288 (!search_attribute_events.is_empty()).then_some("search-attribute update"),
13289 (!side_effect_events.is_empty()).then_some("side effect"),
13290 (!version_marker_events.is_empty()).then_some("version marker"),
13291 (!memo_events.is_empty()).then_some("memo upsert"),
13292 ]
13293 .into_iter()
13294 .flatten()
13295 .collect::<Vec<_>>()
13296 .join(" and ");
13297 return Err(invalid_recorded_history(
13298 "durable_command_sequence_collision",
13299 sequence,
13300 "one durable command kind",
13301 &actual,
13302 "one workflow sequence records more than one durable command kind",
13303 ));
13304 }
13305
13306 if !activity_events.is_empty() {
13307 let parallel_group_path =
13308 recorded_parallel_group_path(&activity_events, sequence)?;
13309 let scheduled_count = activity_events
13310 .iter()
13311 .filter(|event| event.event_type == "ActivityScheduled")
13312 .count();
13313 if scheduled_count > 1 {
13314 return Err(invalid_recorded_history(
13315 "duplicate_activity_schedule",
13316 sequence,
13317 "at most one ActivityScheduled event",
13318 "multiple ActivityScheduled events",
13319 "activity history schedules more than one command at one workflow sequence",
13320 ));
13321 }
13322 let activity_type = activity_events.iter().find_map(|event| {
13323 event
13324 .payload
13325 .get("activity_type")
13326 .or_else(|| event.payload.get("activity_name"))
13327 .and_then(Value::as_str)
13328 .map(str::to_string)
13329 });
13330 if activity_events.iter().filter_map(|event| {
13331 event
13332 .payload
13333 .get("activity_type")
13334 .or_else(|| event.payload.get("activity_name"))
13335 .and_then(Value::as_str)
13336 }).any(|candidate| Some(candidate) != activity_type.as_deref()) {
13337 return Err(invalid_recorded_history(
13338 "activity_identity_mismatch",
13339 sequence,
13340 activity_type.as_deref().unwrap_or("one activity identity"),
13341 "conflicting activity identities",
13342 "activity lifecycle events at one workflow sequence disagree on identity",
13343 ));
13344 }
13345 let terminal: Vec<_> = activity_events
13346 .iter()
13347 .copied()
13348 .filter(|event| {
13349 matches!(
13350 event.event_type.as_str(),
13351 "ActivityCompleted"
13352 | "ActivityFailed"
13353 | "ActivityCancelled"
13354 | "ActivityTimedOut"
13355 )
13356 })
13357 .collect();
13358 let duplicate_delivery = terminal.first().is_some_and(|first| {
13359 terminal.iter().all(|event| {
13360 event.event_type == first.event_type && event.payload == first.payload
13361 })
13362 });
13363 if terminal.len() > 1 && !duplicate_delivery {
13364 return Err(invalid_recorded_history(
13365 "duplicate_activity_terminal_event",
13366 sequence,
13367 "at most one terminal activity event",
13368 "multiple terminal activity events",
13369 "activity history settles one command more than once",
13370 ));
13371 }
13372 let outcome = terminal
13373 .first()
13374 .map(|event| activity_outcome(event, fallback_codec, activity_type.clone()))
13375 .transpose()?;
13376 let options = activity_events
13377 .iter()
13378 .find(|event| event.event_type == "ActivityScheduled")
13379 .and_then(|event| event.payload.get("activity"))
13380 .and_then(Value::as_object)
13381 .map(|activity| RecordedActivityOptions {
13382 task_queue: recorded_optional_string(activity, "queue"),
13383 execution_mode: recorded_optional_string(activity, "execution_mode"),
13384 retry_policy: recorded_activity_retry_snapshot(
13385 activity.get("retry_policy"),
13386 ),
13387 });
13388 return Ok(RecordedCommand::Activity {
13389 sequence,
13390 activity_type,
13391 options,
13392 outcome,
13393 parallel_group_path,
13394 });
13395 }
13396
13397 if !child_events.is_empty() {
13398 let parallel_group_path = recorded_parallel_group_path(&child_events, sequence)?;
13399 let scheduled: Vec<_> = child_events
13400 .iter()
13401 .copied()
13402 .filter(|event| event.event_type == "ChildWorkflowScheduled")
13403 .collect();
13404 if scheduled.len() != 1 {
13405 return Err(invalid_recorded_history(
13406 "child_workflow_schedule_missing_or_duplicate",
13407 sequence,
13408 "one ChildWorkflowScheduled event",
13409 &format!("{} ChildWorkflowScheduled events", scheduled.len()),
13410 "child workflow replay requires exactly one recorded schedule event",
13411 ));
13412 }
13413 let workflow_type = child_events.iter().find_map(|event| {
13414 event
13415 .payload
13416 .get("child_workflow_type")
13417 .or_else(|| event.payload.get("workflow_type"))
13418 .and_then(Value::as_str)
13419 .filter(|value| !value.is_empty())
13420 .map(str::to_string)
13421 });
13422 if child_events
13423 .iter()
13424 .filter_map(|event| {
13425 event
13426 .payload
13427 .get("child_workflow_type")
13428 .or_else(|| event.payload.get("workflow_type"))
13429 .and_then(Value::as_str)
13430 })
13431 .any(|candidate| Some(candidate) != workflow_type.as_deref())
13432 {
13433 return Err(invalid_recorded_history(
13434 "child_workflow_identity_mismatch",
13435 sequence,
13436 workflow_type
13437 .as_deref()
13438 .unwrap_or("one child workflow type"),
13439 "conflicting child workflow types",
13440 "child workflow lifecycle events at one sequence disagree on type",
13441 ));
13442 }
13443 let mut outcomes = child_workflow_outcomes(
13444 &child_events.iter().map(|event| (*event).clone()).collect::<Vec<_>>(),
13445 fallback_codec,
13446 parent.clone(),
13447 )?;
13448 let terminal_events = child_events
13449 .iter()
13450 .copied()
13451 .filter(|event| event.event_type.starts_with("ChildRun"))
13452 .collect::<Vec<_>>();
13453 let duplicate_delivery = terminal_events.first().is_some_and(|first| {
13454 terminal_events.iter().all(|event| {
13455 event.event_type == first.event_type && event.payload == first.payload
13456 })
13457 });
13458 if outcomes.len() > 1 && !duplicate_delivery {
13459 return Err(invalid_recorded_history(
13460 "duplicate_child_workflow_terminal_event",
13461 sequence,
13462 "at most one terminal child event",
13463 "multiple terminal child events",
13464 "child workflow history settles one command more than once",
13465 ));
13466 }
13467 return Ok(RecordedCommand::ChildWorkflow {
13468 sequence,
13469 workflow_type,
13470 outcome: outcomes.pop(),
13471 parallel_group_path,
13472 });
13473 }
13474
13475 if !signal_wait_events.is_empty() {
13476 let opened: Vec<_> = signal_wait_events
13477 .iter()
13478 .copied()
13479 .filter(|event| event.event_type == "SignalWaitOpened")
13480 .collect();
13481 if opened.len() != 1 {
13482 return Err(invalid_recorded_history(
13483 "signal_wait_open_missing_or_duplicate",
13484 sequence,
13485 "one SignalWaitOpened event",
13486 &format!("{} SignalWaitOpened events", opened.len()),
13487 "signal replay requires exactly one canonical wait-open event",
13488 ));
13489 }
13490
13491 let applied: Vec<_> = signal_wait_events
13492 .iter()
13493 .copied()
13494 .filter(|event| event.event_type == "SignalApplied")
13495 .collect();
13496 if applied.len() > 1 {
13497 return Err(invalid_recorded_history(
13498 "duplicate_signal_wait_apply",
13499 sequence,
13500 "at most one SignalApplied event",
13501 "multiple SignalApplied events",
13502 "signal history applies one durable wait more than once",
13503 ));
13504 }
13505
13506 let signal_names = signal_wait_events
13507 .iter()
13508 .map(|event| required_signal_wait_name(event, sequence))
13509 .collect::<Result<Vec<_>>>()?;
13510 let signal_name = signal_names
13511 .first()
13512 .expect("signal wait events are not empty")
13513 .clone();
13514 if signal_names.iter().any(|candidate| candidate != &signal_name) {
13515 return Err(invalid_recorded_history(
13516 "signal_wait_identity_mismatch",
13517 sequence,
13518 &signal_name,
13519 "conflicting signal names",
13520 "signal wait lifecycle events at one workflow sequence disagree on identity",
13521 ));
13522 }
13523 let value = applied
13524 .first()
13525 .map(|event| decode_signal_event_arguments(event, fallback_codec))
13526 .transpose()?;
13527 return Ok(RecordedCommand::SignalWait {
13528 sequence,
13529 signal_name,
13530 value,
13531 parallel_group_path: recorded_parallel_group_path(
13532 &signal_wait_events,
13533 sequence,
13534 )?,
13535 });
13536 }
13537
13538 if !condition_wait_events.is_empty() {
13539 return recorded_condition_wait(
13540 sequence,
13541 &condition_wait_events,
13542 events,
13543 );
13544 }
13545
13546 if !search_attribute_events.is_empty() {
13547 if search_attribute_events.len() != 1 {
13548 return Err(invalid_recorded_history(
13549 "duplicate_search_attribute_update",
13550 sequence,
13551 "one SearchAttributesUpserted event",
13552 &format!(
13553 "{} SearchAttributesUpserted events",
13554 search_attribute_events.len()
13555 ),
13556 "search-attribute history records one workflow command more than once",
13557 ));
13558 }
13559 let payload = &search_attribute_events[0].payload;
13560 let attributes = payload
13561 .get("attributes")
13562 .filter(|value| value.as_object().is_some_and(|values| !values.is_empty()))
13563 .cloned()
13564 .ok_or_else(|| {
13565 invalid_recorded_history(
13566 "search_attribute_update_missing",
13567 sequence,
13568 "non-empty attributes object",
13569 "missing or invalid attributes",
13570 "search-attribute history is missing its recorded mutation",
13571 )
13572 })?;
13573 let attribute_types =
13574 recorded_search_attribute_types(payload, &attributes, sequence)?;
13575 return Ok(RecordedCommand::SearchAttributes {
13576 sequence,
13577 attributes,
13578 attribute_types,
13579 });
13580 }
13581
13582 if !side_effect_events.is_empty() {
13583 if side_effect_events.len() != 1 {
13584 return Err(invalid_recorded_history(
13585 "duplicate_side_effect_record",
13586 sequence,
13587 "one SideEffectRecorded event",
13588 &format!("{} SideEffectRecorded events", side_effect_events.len()),
13589 "side-effect history records one workflow command more than once",
13590 ));
13591 }
13592 let event = side_effect_events[0];
13593 let result = event.payload.get("result").ok_or_else(|| {
13594 invalid_recorded_history(
13595 "side_effect_result_missing",
13596 sequence,
13597 "recorded result payload",
13598 "missing result",
13599 "side-effect history is missing its recorded value",
13600 )
13601 })?;
13602 let has_published_envelope = result.as_str().is_some()
13603 || result.as_object().is_some_and(|envelope| {
13604 envelope.get("codec").and_then(Value::as_str).is_some()
13605 && envelope.get("blob").and_then(Value::as_str).is_some()
13606 });
13607 if !has_published_envelope {
13608 return Err(invalid_recorded_history(
13609 "side_effect_payload_malformed",
13610 sequence,
13611 "payload blob or {codec, blob} envelope",
13612 &result.to_string(),
13613 "side-effect history result does not use a published payload envelope",
13614 ));
13615 }
13616 let codec = event
13617 .payload
13618 .get("payload_codec")
13619 .and_then(Value::as_str)
13620 .unwrap_or(fallback_codec);
13621 let value = decode_wire_avro_value(result, codec).map_err(|error| {
13622 if error.to_string().contains("unsupported_payload_codec") {
13623 return error;
13624 }
13625
13626 invalid_recorded_history(
13627 "side_effect_payload_incompatible",
13628 sequence,
13629 &format!("valid {codec} payload envelope"),
13630 &error.to_string(),
13631 "side-effect history payload cannot be decoded with its recorded codec",
13632 )
13633 })?;
13634 return Ok(RecordedCommand::SideEffect { sequence, value });
13635 }
13636
13637 if !version_marker_events.is_empty() {
13638 if version_marker_events.len() != 1 {
13639 return Err(invalid_recorded_history(
13640 "duplicate_version_marker_record",
13641 sequence,
13642 "one VersionMarkerRecorded event",
13643 &format!("{} VersionMarkerRecorded events", version_marker_events.len()),
13644 "version-marker history records one workflow command more than once",
13645 ));
13646 }
13647 let payload = &version_marker_events[0].payload;
13648 let change_id = payload
13649 .get("change_id")
13650 .and_then(Value::as_str)
13651 .filter(|value| !value.is_empty())
13652 .map(str::to_string)
13653 .ok_or_else(|| {
13654 invalid_recorded_history(
13655 "version_marker_field_missing",
13656 sequence,
13657 "non-empty change_id",
13658 "missing or invalid change_id",
13659 "version-marker history is missing its stable change ID",
13660 )
13661 })?;
13662 let version = required_version_i32(payload, "version", sequence)?;
13663 let min_supported = required_version_i32(payload, "min_supported", sequence)?;
13664 let max_supported = required_version_i32(payload, "max_supported", sequence)?;
13665 if min_supported > max_supported || version < min_supported || version > max_supported {
13666 return Err(invalid_recorded_history(
13667 "version_marker_history_range_invalid",
13668 sequence,
13669 "min_supported <= version <= max_supported",
13670 &format!("{min_supported} <= {version} <= {max_supported}"),
13671 "recorded version marker contains an internally incompatible range",
13672 ));
13673 }
13674 return Ok(RecordedCommand::VersionMarker {
13675 sequence,
13676 change_id,
13677 version,
13678 });
13679 }
13680
13681 if !memo_events.is_empty() {
13682 if memo_events.len() != 1 {
13683 return Err(invalid_recorded_history(
13684 "duplicate_memo_upsert_record",
13685 sequence,
13686 "one MemoUpserted event",
13687 &format!("{} MemoUpserted events", memo_events.len()),
13688 "memo history records one workflow update more than once",
13689 ));
13690 }
13691 let payload = &memo_events[0].payload;
13692 let entries = payload.get("entries").cloned().ok_or_else(|| {
13693 invalid_recorded_history(
13694 "memo_entries_missing",
13695 sequence,
13696 "memo entries object",
13697 "missing entries",
13698 "MemoUpserted history is missing replay identity entries",
13699 )
13700 })?;
13701 let entries = decode_memo_history_map(&entries, true).map_err(|error| {
13702 invalid_recorded_history(
13703 "memo_entries_invalid",
13704 sequence,
13705 "valid canonical memo entries",
13706 &error.to_string(),
13707 "MemoUpserted history contains invalid replay identity entries",
13708 )
13709 })?;
13710 let merged = payload.get("merged").cloned().ok_or_else(|| {
13711 invalid_recorded_history(
13712 "memo_merged_projection_missing",
13713 sequence,
13714 "merged memo projection",
13715 "missing merged",
13716 "MemoUpserted history is missing its merged projection",
13717 )
13718 })?;
13719 decode_memo_history_map(&merged, false).map_err(|error| {
13720 invalid_recorded_history(
13721 "memo_merged_projection_invalid",
13722 sequence,
13723 "valid merged memo projection",
13724 &error.to_string(),
13725 "MemoUpserted history contains an invalid merged projection",
13726 )
13727 })?;
13728
13729 return Ok(RecordedCommand::Memo { sequence, entries });
13730 }
13731 let scheduled: Vec<_> = timer_events
13732 .iter()
13733 .copied()
13734 .filter(|event| event.event_type == "TimerScheduled")
13735 .collect();
13736 let fired: Vec<_> = timer_events
13737 .iter()
13738 .copied()
13739 .filter(|event| event.event_type == "TimerFired")
13740 .collect();
13741 if scheduled.len() != 1 {
13742 return Err(invalid_recorded_history(
13743 "timer_schedule_missing_or_duplicate",
13744 sequence,
13745 "one TimerScheduled event",
13746 &format!("{} TimerScheduled events", scheduled.len()),
13747 "timer replay requires exactly one recorded schedule event",
13748 ));
13749 }
13750 if fired.len() > 1 {
13751 return Err(invalid_recorded_history(
13752 "duplicate_timer_fire",
13753 sequence,
13754 "at most one TimerFired event",
13755 "multiple TimerFired events",
13756 "timer history contains more than one fire event for a workflow sequence",
13757 ));
13758 }
13759
13760 let scheduled = scheduled[0];
13761 let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
13762 let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
13763 if let Some(fired) = fired.first() {
13764 let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
13765 if fired_timer_id != timer_id {
13766 return Err(invalid_recorded_history(
13767 "timer_identity_mismatch",
13768 sequence,
13769 &timer_id,
13770 &fired_timer_id,
13771 "TimerFired does not correspond to the recorded TimerScheduled event",
13772 ));
13773 }
13774 let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
13775 if fired_delay != delay_seconds {
13776 return Err(invalid_recorded_history(
13777 "timer_history_delay_mismatch",
13778 sequence,
13779 &delay_seconds.to_string(),
13780 &fired_delay.to_string(),
13781 "TimerScheduled and TimerFired record different delays",
13782 ));
13783 }
13784 }
13785
13786 Ok(RecordedCommand::Timer {
13787 sequence,
13788 delay_seconds,
13789 fired: !fired.is_empty(),
13790 parallel_group_path: recorded_parallel_group_path(&timer_events, sequence)?,
13791 })
13792 })
13793 .collect::<Result<_>>()?;
13794
13795 let mut marker_sequences = HashMap::new();
13796 for command in &commands {
13797 if let RecordedCommand::VersionMarker {
13798 sequence,
13799 change_id,
13800 ..
13801 } = command
13802 {
13803 if let Some(first_sequence) = marker_sequences.insert(change_id.clone(), *sequence) {
13804 return Err(invalid_recorded_history(
13805 "duplicate_version_marker",
13806 *sequence,
13807 &format!("one marker for change ID {change_id:?}"),
13808 &format!("markers at sequences {first_sequence} and {sequence}"),
13809 "workflow history contains duplicate markers for one stable change ID",
13810 ));
13811 }
13812 }
13813 }
13814
13815 Ok(commands)
13816}
13817
13818fn required_version_i32(payload: &Value, field: &str, sequence: u64) -> Result<i32> {
13819 payload
13820 .get(field)
13821 .and_then(Value::as_i64)
13822 .and_then(|value| i32::try_from(value).ok())
13823 .ok_or_else(|| {
13824 invalid_recorded_history(
13825 "version_marker_field_missing",
13826 sequence,
13827 &format!("integer {field}"),
13828 "missing or out-of-range integer",
13829 "version-marker history is missing a required integer field",
13830 )
13831 })
13832}
13833
13834fn durable_event_sequence(event: &HistoryEvent) -> Option<u64> {
13835 event
13836 .payload
13837 .get("sequence")
13838 .or_else(|| event.payload.get("workflow_sequence"))
13839 .or_else(|| event.raw.get("sequence"))
13840 .or_else(|| event.raw.get("workflow_sequence"))
13841 .and_then(value_as_u64)
13842}
13843
13844fn is_internal_timer_event(event: &HistoryEvent) -> bool {
13845 matches!(
13846 event
13847 .payload
13848 .get("timer_kind")
13849 .or_else(|| event.raw.get("timer_kind"))
13850 .and_then(Value::as_str),
13851 Some("condition_timeout" | "signal_timeout")
13852 )
13853}
13854
13855fn is_recorded_condition_wait_event(event: &HistoryEvent) -> bool {
13856 matches!(
13857 event.event_type.as_str(),
13858 "ConditionWaitOpened" | "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
13859 )
13860}
13861
13862fn recorded_condition_wait(
13863 sequence: u64,
13864 condition_events: &[&HistoryEvent],
13865 all_events: &[HistoryEvent],
13866) -> Result<RecordedCommand> {
13867 let opened = condition_events
13868 .iter()
13869 .copied()
13870 .filter(|event| event.event_type == "ConditionWaitOpened")
13871 .collect::<Vec<_>>();
13872 if opened.len() != 1 {
13873 return Err(invalid_recorded_history(
13874 "condition_wait_open_missing_or_duplicate",
13875 sequence,
13876 "one ConditionWaitOpened event",
13877 &format!("{} ConditionWaitOpened events", opened.len()),
13878 "condition replay requires exactly one canonical wait-open event",
13879 ));
13880 }
13881 let terminal = condition_events
13882 .iter()
13883 .copied()
13884 .filter(|event| {
13885 matches!(
13886 event.event_type.as_str(),
13887 "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
13888 )
13889 })
13890 .collect::<Vec<_>>();
13891 if terminal.len() > 1 {
13892 return Err(invalid_recorded_history(
13893 "duplicate_condition_wait_terminal_event",
13894 sequence,
13895 "at most one condition terminal event",
13896 "multiple condition terminal events",
13897 "condition history settles one durable wait more than once",
13898 ));
13899 }
13900
13901 let opened = opened[0];
13902 let condition_wait_id = required_condition_wait_id(opened, sequence)?;
13903 let occurrence_id = required_condition_wait_occurrence_id(opened, sequence)?;
13904 for event in condition_events
13905 .iter()
13906 .copied()
13907 .filter(|event| !std::ptr::eq(*event, opened))
13908 {
13909 let event_wait_id = required_condition_wait_id(event, sequence)?;
13910 if event_wait_id != condition_wait_id {
13911 return Err(invalid_recorded_history(
13912 "condition_wait_id_mismatch",
13913 sequence,
13914 &condition_wait_id,
13915 &event_wait_id,
13916 "condition lifecycle events at one sequence disagree on wait identity",
13917 ));
13918 }
13919 let event_occurrence_id = required_condition_wait_occurrence_id(event, sequence)?;
13920 if event_occurrence_id != occurrence_id {
13921 return Err(invalid_recorded_history(
13922 "condition_wait_occurrence_history_mismatch",
13923 sequence,
13924 &occurrence_id,
13925 &event_occurrence_id,
13926 "condition lifecycle events at one sequence disagree on authored occurrence identity",
13927 ));
13928 }
13929 }
13930
13931 let condition_key = optional_non_empty_history_string(opened, "condition_key");
13932 let predicate_identity = opened
13933 .payload
13934 .get("condition_definition_fingerprint")
13935 .and_then(Value::as_str)
13936 .filter(|value| !value.is_empty())
13937 .map(str::to_string)
13938 .ok_or_else(|| {
13939 invalid_recorded_history(
13940 "condition_wait_predicate_fingerprint_missing",
13941 sequence,
13942 "non-empty condition_definition_fingerprint",
13943 &opened.event_type,
13944 "canonical condition history is missing its predicate identity",
13945 )
13946 })?;
13947 let timeout_seconds = optional_history_u64(opened, "timeout_seconds", sequence)?;
13948 for event in condition_events
13949 .iter()
13950 .copied()
13951 .filter(|event| !std::ptr::eq(*event, opened))
13952 {
13953 for (field, opened_value) in [
13954 ("condition_key", condition_key.as_deref()),
13955 (
13956 "condition_definition_fingerprint",
13957 Some(predicate_identity.as_str()),
13958 ),
13959 ] {
13960 if let Some(value) = optional_non_empty_history_string(event, field) {
13961 if opened_value.is_some_and(|opened_value| opened_value != value) {
13962 return Err(invalid_recorded_history(
13963 "condition_wait_definition_history_mismatch",
13964 sequence,
13965 opened_value.unwrap_or_default(),
13966 &value,
13967 "condition lifecycle events disagree on the recorded definition",
13968 ));
13969 }
13970 }
13971 }
13972 if let Some(event_timeout) = optional_history_u64(event, "timeout_seconds", sequence)? {
13973 if timeout_seconds.is_some_and(|opened_timeout| opened_timeout != event_timeout) {
13974 return Err(invalid_recorded_history(
13975 "condition_wait_definition_history_mismatch",
13976 sequence,
13977 &format!("{}s", timeout_seconds.unwrap_or_default()),
13978 &format!("{event_timeout}s"),
13979 "condition lifecycle events disagree on the recorded timeout",
13980 ));
13981 }
13982 }
13983 }
13984
13985 let timeout_timer_events = all_events
13986 .iter()
13987 .filter(|event| {
13988 matches!(
13989 event.event_type.as_str(),
13990 "TimerScheduled" | "TimerCancelled" | "TimerFired"
13991 ) && event.payload.get("timer_kind").and_then(Value::as_str)
13992 == Some("condition_timeout")
13993 && event
13994 .payload
13995 .get("condition_wait_id")
13996 .and_then(Value::as_str)
13997 == Some(condition_wait_id.as_str())
13998 })
13999 .collect::<Vec<_>>();
14000 let scheduled = timeout_timer_events
14001 .iter()
14002 .copied()
14003 .filter(|event| event.event_type == "TimerScheduled")
14004 .collect::<Vec<_>>();
14005 let fired = timeout_timer_events
14006 .iter()
14007 .copied()
14008 .filter(|event| event.event_type == "TimerFired")
14009 .collect::<Vec<_>>();
14010 if scheduled.len() > 1 || fired.len() > 1 || (!fired.is_empty() && scheduled.len() != 1) {
14011 return Err(invalid_recorded_history(
14012 "condition_wait_timeout_history_invalid",
14013 sequence,
14014 "one timeout schedule and at most one fire",
14015 &format!("{} schedules and {} fires", scheduled.len(), fired.len()),
14016 "condition timeout history has a missing or duplicate lifecycle event",
14017 ));
14018 }
14019 if let Some(scheduled) = scheduled.first() {
14020 let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
14021 let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
14022 if timeout_seconds.is_some_and(|timeout| timeout != delay_seconds) {
14023 return Err(invalid_recorded_history(
14024 "condition_wait_timeout_delay_mismatch",
14025 sequence,
14026 &format!("{}s", timeout_seconds.unwrap_or_default()),
14027 &format!("{delay_seconds}s"),
14028 "condition timeout timer differs from the wait definition",
14029 ));
14030 }
14031 if let Some(fired) = fired.first() {
14032 let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
14033 let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
14034 if fired_timer_id != timer_id || fired_delay != delay_seconds {
14035 return Err(invalid_recorded_history(
14036 "condition_wait_timeout_identity_mismatch",
14037 sequence,
14038 &format!("{timer_id}:{delay_seconds}s"),
14039 &format!("{fired_timer_id}:{fired_delay}s"),
14040 "condition timeout fire does not match its durable schedule",
14041 ));
14042 }
14043 }
14044 }
14045
14046 let result = terminal.first().map(|event| {
14047 if event.event_type == "ConditionWaitTimedOut" {
14048 ConditionWaitResult::TimedOut
14049 } else {
14050 ConditionWaitResult::Satisfied
14051 }
14052 });
14053 let result = if !fired.is_empty() {
14054 if result == Some(ConditionWaitResult::Satisfied) {
14055 return Err(invalid_recorded_history(
14056 "condition_wait_terminal_conflict",
14057 sequence,
14058 "one satisfied or timed-out outcome",
14059 "satisfied event and fired timeout",
14060 "condition history records conflicting terminal outcomes",
14061 ));
14062 }
14063 Some(ConditionWaitResult::TimedOut)
14064 } else {
14065 result
14066 };
14067
14068 Ok(RecordedCommand::ConditionWait {
14069 sequence,
14070 occurrence_id,
14071 condition_key,
14072 predicate_identity,
14073 timeout_seconds,
14074 result,
14075 parallel_group_path: recorded_parallel_group_path(condition_events, sequence)?,
14076 })
14077}
14078
14079fn required_condition_wait_occurrence_id(event: &HistoryEvent, sequence: u64) -> Result<String> {
14080 event
14081 .payload
14082 .get("condition_wait_occurrence_id")
14083 .and_then(Value::as_str)
14084 .filter(|value| !value.is_empty())
14085 .map(str::to_string)
14086 .ok_or_else(|| {
14087 invalid_recorded_history(
14088 "condition_wait_occurrence_id_missing",
14089 sequence,
14090 "non-empty condition_wait_occurrence_id",
14091 &event.event_type,
14092 "condition history is missing authored occurrence identity",
14093 )
14094 })
14095}
14096
14097fn required_condition_wait_id(event: &HistoryEvent, sequence: u64) -> Result<String> {
14098 event
14099 .payload
14100 .get("condition_wait_id")
14101 .and_then(Value::as_str)
14102 .filter(|value| !value.is_empty())
14103 .map(str::to_string)
14104 .ok_or_else(|| {
14105 invalid_recorded_history(
14106 "condition_wait_id_missing",
14107 sequence,
14108 "non-empty condition_wait_id",
14109 &event.event_type,
14110 "canonical condition history is missing its durable wait identity",
14111 )
14112 })
14113}
14114
14115fn optional_non_empty_history_string(event: &HistoryEvent, field: &str) -> Option<String> {
14116 event
14117 .payload
14118 .get(field)
14119 .and_then(Value::as_str)
14120 .filter(|value| !value.is_empty())
14121 .map(str::to_string)
14122}
14123
14124fn optional_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<Option<u64>> {
14125 match event.payload.get(field) {
14126 None | Some(Value::Null) => Ok(None),
14127 Some(value) => value_as_u64(value).map(Some).ok_or_else(|| {
14128 invalid_recorded_history(
14129 "condition_wait_definition_invalid",
14130 sequence,
14131 &format!("non-negative integer {field}"),
14132 &value.to_string(),
14133 "condition history contains an invalid numeric definition field",
14134 )
14135 }),
14136 }
14137}
14138
14139fn required_signal_wait_name(event: &HistoryEvent, sequence: u64) -> Result<String> {
14140 event
14141 .payload
14142 .get("signal_name")
14143 .or_else(|| event.raw.get("signal_name"))
14144 .and_then(Value::as_str)
14145 .filter(|value| !value.is_empty())
14146 .map(str::to_string)
14147 .ok_or_else(|| {
14148 invalid_recorded_history(
14149 "signal_wait_name_missing",
14150 sequence,
14151 "non-empty signal_name",
14152 &event.event_type,
14153 "canonical signal-wait history is missing its signal identity",
14154 )
14155 })
14156}
14157
14158fn is_recorded_signal_wait_event(event: &HistoryEvent) -> bool {
14159 matches!(
14160 event.event_type.as_str(),
14161 "SignalWaitOpened" | "SignalApplied"
14162 )
14163}
14164
14165fn required_history_string(event: &HistoryEvent, field: &str, sequence: u64) -> Result<String> {
14166 event
14167 .payload
14168 .get(field)
14169 .and_then(Value::as_str)
14170 .filter(|value| !value.is_empty())
14171 .map(str::to_string)
14172 .ok_or_else(|| {
14173 invalid_recorded_history(
14174 "timer_history_field_missing",
14175 sequence,
14176 field,
14177 &event.event_type,
14178 "timer history is missing a required identity field",
14179 )
14180 })
14181}
14182
14183fn required_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<u64> {
14184 event
14185 .payload
14186 .get(field)
14187 .and_then(value_as_u64)
14188 .ok_or_else(|| {
14189 invalid_recorded_history(
14190 "timer_history_field_missing",
14191 sequence,
14192 field,
14193 &event.event_type,
14194 "timer history is missing a required numeric field",
14195 )
14196 })
14197}
14198
14199fn recorded_search_attribute_types(
14200 payload: &Value,
14201 attributes: &Value,
14202 sequence: u64,
14203) -> Result<RecordedSnapshotValue<BTreeMap<String, String>>> {
14204 let Some(raw_types) = payload.get("attribute_types") else {
14205 return Ok(RecordedSnapshotValue::Unknown);
14209 };
14210 let Some(raw_types) = raw_types.as_object() else {
14211 return Err(invalid_recorded_history(
14212 "search_attribute_types_malformed",
14213 sequence,
14214 "canonical attribute type map",
14215 &raw_types.to_string(),
14216 "search-attribute history contains malformed type identity",
14217 ));
14218 };
14219 let attribute_keys = attributes
14220 .as_object()
14221 .expect("recorded search attributes were validated as an object");
14222 let mut types = BTreeMap::new();
14223 for (key, value) in raw_types {
14224 let Some(attribute_type) = value.as_str() else {
14225 return Err(invalid_recorded_history(
14226 "search_attribute_types_malformed",
14227 sequence,
14228 "canonical string type name",
14229 &value.to_string(),
14230 "search-attribute history contains a non-string type identity",
14231 ));
14232 };
14233 if !attribute_keys.contains_key(key)
14234 || !matches!(
14235 attribute_type,
14236 "string" | "keyword" | "keyword_list" | "int" | "float" | "bool" | "datetime"
14237 )
14238 {
14239 return Err(invalid_recorded_history(
14240 "search_attribute_types_malformed",
14241 sequence,
14242 "canonical types for keys present in attributes",
14243 &format!("{key}:{attribute_type}"),
14244 "search-attribute history contains unsupported or orphaned type identity",
14245 ));
14246 }
14247 types.insert(key.clone(), attribute_type.to_string());
14248 }
14249 Ok(RecordedSnapshotValue::Known(types))
14250}
14251
14252fn invalid_recorded_history(
14253 reason: &str,
14254 sequence: u64,
14255 expected: &str,
14256 actual: &str,
14257 message: &str,
14258) -> Error {
14259 Error::NonDeterministicReplay(ReplayFailure::new(
14260 reason,
14261 Some(sequence),
14262 Some(expected.to_string()),
14263 Some(actual.to_string()),
14264 message,
14265 ))
14266}
14267
14268type ActivityOutcome = std::result::Result<AvroValue, ActivityFailure>;
14269
14270fn activity_outcome(
14271 event: &HistoryEvent,
14272 fallback_codec: &str,
14273 recorded_activity_type: Option<String>,
14274) -> Result<ActivityOutcome> {
14275 if event.event_type == "ActivityCompleted" {
14276 let codec = event
14277 .payload
14278 .get("payload_codec")
14279 .and_then(Value::as_str)
14280 .unwrap_or(fallback_codec);
14281 return Ok(Ok(decode_wire_avro_value(
14282 event.payload.get("result").unwrap_or(&Value::Null),
14283 codec,
14284 )?));
14285 }
14286
14287 let payload = &event.payload;
14288 let (kind, fallback_reason, fallback_message) = match event.event_type.as_str() {
14289 "ActivityFailed" => (ActivityFailureKind::Failed, "activity", "activity failed"),
14290 "ActivityCancelled" => (
14291 ActivityFailureKind::Cancelled,
14292 "cancelled",
14293 "activity was cancelled",
14294 ),
14295 "ActivityTimedOut" => (
14296 ActivityFailureKind::TimedOut,
14297 "timeout",
14298 "activity timed out",
14299 ),
14300 _ => unreachable!("activity_outcome is called only for terminal activity events"),
14301 };
14302 let exception = payload
14303 .get("exception")
14304 .filter(|value| !value.is_null())
14305 .cloned();
14306 let failure_category = payload_string(payload, "failure_category");
14307 let timeout_kind = payload_string(payload, "timeout_kind");
14308 let reason = payload_string(payload, "reason").unwrap_or_else(|| match kind {
14309 ActivityFailureKind::Failed => failure_category
14310 .clone()
14311 .unwrap_or_else(|| fallback_reason.to_string()),
14312 ActivityFailureKind::Cancelled => fallback_reason.to_string(),
14313 ActivityFailureKind::TimedOut => timeout_kind
14314 .clone()
14315 .unwrap_or_else(|| fallback_reason.to_string()),
14316 });
14317 let message = payload_string(payload, "message")
14318 .or_else(|| {
14319 exception
14320 .as_ref()
14321 .and_then(|value| payload_string(value, "message"))
14322 })
14323 .unwrap_or_else(|| fallback_message.to_string());
14324
14325 Ok(Err(ActivityFailure {
14326 kind,
14327 reason,
14328 message,
14329 activity_execution_id: payload_string(payload, "activity_execution_id"),
14330 activity_attempt_id: payload_string(payload, "activity_attempt_id"),
14331 activity_type: payload_string(payload, "activity_type")
14332 .or_else(|| payload_string(payload, "activity_name"))
14333 .or(recorded_activity_type),
14334 activity_class: payload_string(payload, "activity_class"),
14335 attempt_number: payload.get("attempt_number").and_then(value_as_u64),
14336 failure_id: payload_string(payload, "failure_id"),
14337 failure_category,
14338 timeout_kind,
14339 non_retryable: payload
14340 .get("non_retryable")
14341 .and_then(Value::as_bool)
14342 .unwrap_or(false),
14343 exception_type: payload_string(payload, "exception_type").or_else(|| {
14344 exception
14345 .as_ref()
14346 .and_then(|value| payload_string(value, "type"))
14347 }),
14348 exception_class: payload_string(payload, "exception_class").or_else(|| {
14349 exception
14350 .as_ref()
14351 .and_then(|value| payload_string(value, "class"))
14352 }),
14353 code: payload
14354 .get("code")
14355 .filter(|value| !value.is_null())
14356 .cloned(),
14357 exception,
14358 }))
14359}
14360
14361type ChildWorkflowOutcome = std::result::Result<ChildWorkflowAvroResult, ChildWorkflowFailure>;
14362
14363fn child_workflow_outcomes(
14364 events: &[HistoryEvent],
14365 fallback_codec: &str,
14366 parent: WorkflowIdentity,
14367) -> Result<Vec<ChildWorkflowOutcome>> {
14368 let mut outcomes = Vec::new();
14369
14370 for event in events {
14371 let kind = match event.event_type.as_str() {
14372 "ChildRunCompleted" => None,
14373 "ChildRunFailed" => Some((
14374 ChildWorkflowFailureKind::Failed,
14375 "child_workflow",
14376 "child workflow failed",
14377 )),
14378 "ChildRunCancelled" => Some((
14379 ChildWorkflowFailureKind::Cancelled,
14380 "cancelled",
14381 "child workflow was cancelled",
14382 )),
14383 "ChildRunTerminated" => Some((
14384 ChildWorkflowFailureKind::Terminated,
14385 "terminated",
14386 "child workflow was terminated",
14387 )),
14388 _ => continue,
14389 };
14390 let payload = &event.payload;
14391 let child_workflow_id = payload_string(payload, "child_workflow_instance_id");
14392 let child_workflow_run_id = payload_string(payload, "child_workflow_run_id");
14393 let child_workflow_type = payload_string(payload, "child_workflow_type");
14394
14395 if let Some((kind, reason, fallback_message)) = kind {
14396 let exception = payload
14397 .get("exception")
14398 .filter(|value| !value.is_null())
14399 .cloned();
14400 let message = payload_string(payload, "message")
14401 .or_else(|| {
14402 exception
14403 .as_ref()
14404 .and_then(|value| payload_string(value, "message"))
14405 })
14406 .unwrap_or_else(|| fallback_message.to_string());
14407 let exception_type = payload_string(payload, "exception_type").or_else(|| {
14408 exception
14409 .as_ref()
14410 .and_then(|value| payload_string(value, "type"))
14411 });
14412 let exception_class = payload_string(payload, "exception_class").or_else(|| {
14413 exception
14414 .as_ref()
14415 .and_then(|value| payload_string(value, "class"))
14416 });
14417 outcomes.push(Err(ChildWorkflowFailure {
14418 kind,
14419 reason: reason.to_string(),
14420 message,
14421 parent_workflow_id: parent.workflow_id.clone(),
14422 parent_workflow_run_id: parent.run_id.clone(),
14423 child_workflow_id,
14424 child_workflow_run_id,
14425 child_workflow_type,
14426 failure_id: payload_string(payload, "failure_id"),
14427 failure_category: payload_string(payload, "failure_category"),
14428 exception_type,
14429 exception_class,
14430 non_retryable: payload
14431 .get("non_retryable")
14432 .and_then(Value::as_bool)
14433 .unwrap_or(false),
14434 code: payload
14435 .get("code")
14436 .filter(|value| !value.is_null())
14437 .cloned(),
14438 exception,
14439 }));
14440 continue;
14441 }
14442
14443 let codec = payload
14444 .get("payload_codec")
14445 .and_then(Value::as_str)
14446 .unwrap_or(fallback_codec);
14447 let result = payload
14448 .get("result")
14449 .or_else(|| payload.get("output"))
14450 .unwrap_or(&Value::Null);
14451 outcomes.push(Ok(ChildWorkflowAvroResult {
14452 parent: parent.clone(),
14453 child: WorkflowIdentity {
14454 workflow_id: child_workflow_id,
14455 run_id: child_workflow_run_id,
14456 },
14457 child_workflow_type,
14458 result: decode_wire_avro_value(result, codec)?,
14459 }));
14460 }
14461
14462 Ok(outcomes)
14463}
14464
14465fn payload_string(payload: &Value, key: &str) -> Option<String> {
14466 payload
14467 .get(key)
14468 .and_then(Value::as_str)
14469 .filter(|value| !value.is_empty())
14470 .map(str::to_string)
14471}
14472
14473fn recorded_activity_failure_boundary(
14474 ctx: &WorkflowContext,
14475 error: &Error,
14476) -> Option<(u64, String)> {
14477 let Error::ActivityFailed(failure) = error else {
14478 return None;
14479 };
14480 if failure.kind != ActivityFailureKind::Failed {
14481 return None;
14482 }
14483 let activity_execution_id = failure
14484 .activity_execution_id
14485 .as_deref()
14486 .filter(|value| !value.is_empty())?;
14487 let state = ctx.state.lock().ok()?;
14488 let mut matches = state.recorded_commands.iter().filter_map(|command| {
14489 let RecordedCommand::Activity {
14490 sequence,
14491 outcome: Some(Err(recorded_failure)),
14492 ..
14493 } = command
14494 else {
14495 return None;
14496 };
14497 (*sequence > 0 && recorded_failure == failure)
14498 .then_some((*sequence, activity_execution_id.to_string()))
14499 });
14500 let boundary = matches.next()?;
14501 matches.next().is_none().then_some(boundary)
14502}
14503
14504fn workflow_failure_command(
14505 error: &Error,
14506 failed_activity_boundary: Option<(u64, String)>,
14507) -> Value {
14508 let (exception_type, exception_class, properties) = match error {
14509 Error::ActivityFailed(failure) => (
14510 match failure.kind {
14511 ActivityFailureKind::Failed => "ActivityFailed",
14512 ActivityFailureKind::Cancelled => "ActivityCancelled",
14513 ActivityFailureKind::TimedOut => "ActivityTimedOut",
14514 },
14515 "durable_workflow::ActivityFailure",
14516 json!({
14517 "reason": failure.reason,
14518 "activity_execution_id": failure.activity_execution_id,
14519 "activity_attempt_id": failure.activity_attempt_id,
14520 "activity_type": failure.activity_type,
14521 "activity_class": failure.activity_class,
14522 "attempt_number": failure.attempt_number,
14523 "failure_id": failure.failure_id,
14524 "failure_category": failure.failure_category,
14525 "timeout_kind": failure.timeout_kind,
14526 "activity_non_retryable": failure.non_retryable,
14527 "activity_exception_type": failure.exception_type,
14528 "activity_exception_class": failure.exception_class,
14529 "activity_code": failure.code,
14530 "activity_exception": failure.exception,
14531 }),
14532 ),
14533 Error::ChildWorkflowFailed(failure) => (
14534 match failure.kind {
14535 ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
14536 ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
14537 ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
14538 },
14539 "durable_workflow::ChildWorkflowFailure",
14540 json!({
14541 "reason": failure.reason,
14542 "parent_workflow_id": failure.parent_workflow_id,
14543 "parent_workflow_run_id": failure.parent_workflow_run_id,
14544 "child_workflow_id": failure.child_workflow_id,
14545 "child_workflow_run_id": failure.child_workflow_run_id,
14546 "child_workflow_type": failure.child_workflow_type,
14547 "failure_id": failure.failure_id,
14548 "failure_category": failure.failure_category,
14549 "child_exception_type": failure.exception_type,
14550 "child_exception_class": failure.exception_class,
14551 "child_non_retryable": failure.non_retryable,
14552 "child_code": failure.code,
14553 "child_exception": failure.exception,
14554 }),
14555 ),
14556 Error::ParallelFailed(failure) => (
14557 "ParallelFailed",
14558 "durable_workflow::ParallelFailure",
14559 json!({
14560 "parallel_group_id": failure.group_id,
14561 "parallel_member_path": failure.member_path,
14562 "parallel_group_path": failure.group_path,
14563 "completed_members": failure.completed.iter().map(|completion| &completion.member_path).collect::<Vec<_>>(),
14564 "cause_type": workflow_error_type(&failure.cause),
14565 "cause_message": failure.cause.to_string(),
14566 }),
14567 ),
14568 Error::SagaCompensationFailed(failure) => (
14569 "SagaCompensationFailed",
14570 "durable_workflow::SagaCompensationFailure",
14571 json!({
14572 "initiating_failure_type": workflow_error_type(&failure.initiating_failure),
14573 "initiating_failure_message": failure.initiating_failure.to_string(),
14574 "compensation_activity_type": failure.compensation_activity_type,
14575 "compensation_registration_order": failure.compensation_registration_order,
14576 "compensation_failure_type": workflow_error_type(&failure.compensation_failure),
14577 "compensation_failure_message": failure.compensation_failure.to_string(),
14578 }),
14579 ),
14580 Error::WorkflowCancellationRequested(_) => (
14581 "WorkflowCancellationRequested",
14582 "durable_workflow::WorkflowCancellationRequested",
14583 json!({"reason": "cancelled"}),
14584 ),
14585 Error::NonDeterministicReplay(_) => (
14586 "NonDeterministicReplay",
14587 "durable_workflow::Error",
14588 Value::Null,
14589 ),
14590 _ => ("RustWorkflowError", "durable_workflow::Error", Value::Null),
14591 };
14592 let non_retryable = match error {
14593 Error::ActivityFailed(failure) => failure.non_retryable,
14594 Error::ChildWorkflowFailed(failure) => failure.non_retryable,
14595 Error::ParallelFailed(failure) => workflow_error_non_retryable(&failure.cause),
14596 Error::SagaCompensationFailed(failure) => {
14597 workflow_error_non_retryable(&failure.compensation_failure)
14598 }
14599 Error::WorkflowCancellationRequested(_) => true,
14600 Error::NonDeterministicReplay(_) => true,
14601 _ => false,
14602 };
14603
14604 let mut command = json!({
14605 "type": "fail_workflow",
14606 "message": error.to_string(),
14607 "exception_type": exception_type,
14608 "exception_class": exception_class,
14609 "non_retryable": non_retryable,
14610 "exception": {
14611 "type": exception_type,
14612 "class": exception_class,
14613 "message": error.to_string(),
14614 "properties": properties,
14615 }
14616 });
14617
14618 if let Some((sequence, activity_execution_id)) = failed_activity_boundary {
14619 command["failed_step_sequence"] = json!(sequence);
14620 command["failed_activity_execution_id"] = json!(activity_execution_id);
14621 }
14622
14623 command
14624}
14625
14626fn workflow_error_type(error: &Error) -> &'static str {
14627 match error {
14628 Error::ActivityFailed(failure) => match failure.kind {
14629 ActivityFailureKind::Failed => "ActivityFailed",
14630 ActivityFailureKind::Cancelled => "ActivityCancelled",
14631 ActivityFailureKind::TimedOut => "ActivityTimedOut",
14632 },
14633 Error::ChildWorkflowFailed(failure) => match failure.kind {
14634 ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
14635 ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
14636 ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
14637 },
14638 Error::ParallelFailed(_) => "ParallelFailed",
14639 Error::SagaCompensationFailed(_) => "SagaCompensationFailed",
14640 Error::WorkflowCancellationRequested(_) => "WorkflowCancellationRequested",
14641 Error::NonDeterministicReplay(_) => "NonDeterministicReplay",
14642 _ => "RustWorkflowError",
14643 }
14644}
14645
14646fn workflow_error_non_retryable(error: &Error) -> bool {
14647 match error {
14648 Error::ActivityFailed(failure) => failure.non_retryable,
14649 Error::ChildWorkflowFailed(failure) => failure.non_retryable,
14650 Error::ParallelFailed(failure) => workflow_error_non_retryable(&failure.cause),
14651 Error::SagaCompensationFailed(failure) => {
14652 workflow_error_non_retryable(&failure.compensation_failure)
14653 }
14654 Error::WorkflowCancellationRequested(_) | Error::NonDeterministicReplay(_) => true,
14655 _ => false,
14656 }
14657}
14658
14659fn workflow_task_integrity_error(error: &Error) -> bool {
14660 matches!(
14661 error,
14662 Error::NonDeterministicReplay(_)
14663 | Error::Protocol(_)
14664 | Error::MissingWorkflowCommandIdentity
14665 | Error::WorkflowStatePoisoned
14666 )
14667}
14668
14669fn decode_signal_event_arguments(
14670 event: &HistoryEvent,
14671 fallback_codec: &str,
14672) -> Result<Vec<AvroValue>> {
14673 let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
14674 validate_payload_codec(codec)?;
14675 let raw = signal_history_payload(&event.payload);
14676 let decoded = match raw.filter(|value| !value.is_null()) {
14677 Some(value) => decode_wire_avro_value(value, codec)?,
14678 None => AvroValue::Array(Vec::new()),
14679 };
14680 let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
14681 unreachable!("normalize_avro_arguments always returns an array");
14682 };
14683 Ok(arguments)
14684}
14685
14686fn decode_update_event_arguments(
14687 event: &HistoryEvent,
14688 fallback_codec: &str,
14689) -> Result<Vec<AvroValue>> {
14690 let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
14691 validate_payload_codec(codec)?;
14692 let decoded = match event
14693 .payload
14694 .get("arguments")
14695 .filter(|value| !value.is_null())
14696 {
14697 Some(value) => decode_wire_avro_value(value, codec)?,
14698 None => AvroValue::Array(Vec::new()),
14699 };
14700 let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
14701 unreachable!("normalize_avro_arguments always returns an array");
14702 };
14703 Ok(arguments)
14704}
14705
14706fn hydrate_query_history_from_export(task: &mut QueryTask) -> Result<()> {
14707 let Some(export_events) = task
14708 .history_export
14709 .as_ref()
14710 .and_then(|export| export.get("history_events"))
14711 .and_then(Value::as_array)
14712 else {
14713 return Ok(());
14714 };
14715
14716 if export_events.len() > task.history_events.len() {
14717 task.history_events = serde_json::from_value(Value::Array(export_events.clone()))?;
14718 }
14719
14720 Ok(())
14721}
14722
14723fn enrich_query_history_from_export(task: &mut QueryTask) -> Result<()> {
14724 let Some(export) = task.history_export.as_ref() else {
14725 return Ok(());
14726 };
14727 let signals = export
14728 .get("signals")
14729 .and_then(Value::as_array)
14730 .cloned()
14731 .unwrap_or_default();
14732 let activities = export
14733 .get("activities")
14734 .and_then(Value::as_array)
14735 .cloned()
14736 .unwrap_or_default();
14737 let export_codec = export
14738 .get("payloads")
14739 .and_then(|payloads| payloads.get("codec"))
14740 .and_then(Value::as_str)
14741 .unwrap_or(&task.payload_codec)
14742 .to_string();
14743 let mut signal_name_offsets: HashMap<String, usize> = HashMap::new();
14744
14745 for event in &mut task.history_events {
14746 if event.event_type == "ActivityCompleted" {
14747 let sequence = event
14748 .payload
14749 .get("sequence")
14750 .or_else(|| event.payload.get("workflow_sequence"))
14751 .and_then(value_as_u64);
14752 let Some(activity) = sequence.and_then(|sequence| {
14753 activities.iter().find(|activity| {
14754 activity.get("sequence").and_then(value_as_u64) == Some(sequence)
14755 })
14756 }) else {
14757 continue;
14758 };
14759 let Some(payload) = event.payload.as_object_mut() else {
14760 continue;
14761 };
14762 if missing_payload(payload.get("result")) {
14763 if let Some(result) = activity
14764 .get("result")
14765 .filter(|value| !missing_payload(Some(value)))
14766 {
14767 payload.insert("result".to_string(), result.clone());
14768 }
14769 }
14770 for field in ["payload_codec", "activity_type"] {
14771 if payload
14772 .get(field)
14773 .and_then(Value::as_str)
14774 .unwrap_or_default()
14775 .is_empty()
14776 {
14777 if let Some(value) = activity.get(field) {
14778 payload.insert(field.to_string(), value.clone());
14779 }
14780 }
14781 }
14782 continue;
14783 }
14784
14785 if event.event_type != "SignalReceived" && event.event_type != "SignalApplied" {
14786 continue;
14787 }
14788 let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
14789 let command_id = event
14790 .payload
14791 .get("workflow_command_id")
14792 .or_else(|| event.raw.get("workflow_command_id"))
14793 .and_then(Value::as_str);
14794 let signal_name = event
14795 .payload
14796 .get("signal_name")
14797 .and_then(Value::as_str)
14798 .unwrap_or_default()
14799 .to_string();
14800 let matched = signals
14801 .iter()
14802 .find(|signal| {
14803 signal_id.is_some() && signal.get("id").and_then(Value::as_str) == signal_id
14804 })
14805 .or_else(|| {
14806 signals.iter().find(|signal| {
14807 command_id.is_some()
14808 && signal.get("command_id").and_then(Value::as_str) == command_id
14809 })
14810 })
14811 .or_else(|| {
14812 let offset = signal_name_offsets.entry(signal_name.clone()).or_default();
14813 let signal = signals
14814 .iter()
14815 .filter(|signal| {
14816 signal.get("name").and_then(Value::as_str) == Some(signal_name.as_str())
14817 })
14818 .nth(*offset);
14819 if signal.is_some() {
14820 *offset += 1;
14821 }
14822 signal
14823 });
14824 let Some(signal) = matched else {
14825 continue;
14826 };
14827 let signal_codec = signal
14828 .get("payload_codec")
14829 .and_then(Value::as_str)
14830 .unwrap_or(&export_codec);
14831 let Some(payload) = event.payload.as_object_mut() else {
14832 continue;
14833 };
14834 if missing_payload(payload.get("arguments")) {
14835 if let Some(arguments) = signal
14836 .get("arguments")
14837 .filter(|value| !missing_payload(Some(value)))
14838 {
14839 let envelope = match arguments {
14840 Value::String(blob) => json!({"codec": signal_codec, "blob": blob}),
14841 other => other.clone(),
14842 };
14843 payload.insert("arguments".to_string(), envelope);
14844 }
14845 }
14846 if payload
14847 .get("payload_codec")
14848 .and_then(Value::as_str)
14849 .unwrap_or_default()
14850 .is_empty()
14851 {
14852 payload.insert("payload_codec".to_string(), json!(signal_codec));
14853 }
14854 }
14855
14856 Ok(())
14857}
14858
14859fn missing_payload(value: Option<&Value>) -> bool {
14860 match value {
14861 None | Some(Value::Null) => true,
14862 Some(Value::String(value)) => value.is_empty(),
14863 Some(_) => false,
14864 }
14865}
14866
14867fn query_signal_events(task: &QueryTask) -> Result<Vec<QuerySignal>> {
14868 let export_signals = task
14869 .history_export
14870 .as_ref()
14871 .and_then(|export| export.get("signals"))
14872 .and_then(Value::as_array)
14873 .cloned()
14874 .unwrap_or_default();
14875 let export_codec = task
14876 .history_export
14877 .as_ref()
14878 .and_then(|export| export.get("payloads"))
14879 .and_then(|payloads| payloads.get("codec"))
14880 .and_then(Value::as_str)
14881 .unwrap_or(&task.payload_codec);
14882 let mut name_offsets: HashMap<String, usize> = HashMap::new();
14883 let mut signals = Vec::new();
14884
14885 for event in &task.history_events {
14886 if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
14887 continue;
14888 }
14889
14890 let name = event
14891 .payload
14892 .get("signal_name")
14893 .and_then(Value::as_str)
14894 .unwrap_or_default();
14895 if name.is_empty() {
14896 continue;
14897 }
14898 let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
14899 let command_id = event
14900 .payload
14901 .get("workflow_command_id")
14902 .or_else(|| event.raw.get("workflow_command_id"))
14903 .and_then(Value::as_str);
14904 let matched_export = export_signals
14905 .iter()
14906 .find(|candidate| {
14907 signal_id.is_some() && candidate.get("id").and_then(Value::as_str) == signal_id
14908 })
14909 .or_else(|| {
14910 export_signals.iter().find(|candidate| {
14911 command_id.is_some()
14912 && candidate.get("command_id").and_then(Value::as_str) == command_id
14913 })
14914 })
14915 .or_else(|| {
14916 let offset = name_offsets.entry(name.to_string()).or_default();
14917 let candidate = export_signals
14918 .iter()
14919 .filter(|candidate| candidate.get("name").and_then(Value::as_str) == Some(name))
14920 .nth(*offset);
14921 if candidate.is_some() {
14922 *offset += 1;
14923 }
14924 candidate
14925 });
14926 let codec = event
14927 .payload
14928 .get("payload_codec")
14929 .and_then(Value::as_str)
14930 .or_else(|| {
14931 matched_export
14932 .and_then(|signal| signal.get("payload_codec"))
14933 .and_then(Value::as_str)
14934 })
14935 .unwrap_or(export_codec);
14936 let raw_arguments = signal_history_payload(&event.payload)
14937 .filter(|value| !value.is_null())
14938 .or_else(|| matched_export.and_then(|signal| signal.get("arguments")));
14939 let (arguments, avro_arguments) = decode_query_signal_arguments(raw_arguments, codec)?;
14940 let workflow_sequence = event
14941 .payload
14942 .get("workflow_sequence")
14943 .and_then(value_as_u64)
14944 .or_else(|| {
14945 matched_export
14946 .and_then(|signal| signal.get("workflow_sequence"))
14947 .and_then(value_as_u64)
14948 });
14949
14950 signals.push(QuerySignal {
14951 id: signal_id.map(str::to_string).or_else(|| {
14952 matched_export
14953 .and_then(|signal| signal.get("id"))
14954 .and_then(Value::as_str)
14955 .map(str::to_string)
14956 }),
14957 name: name.to_string(),
14958 arguments,
14959 avro_arguments,
14960 workflow_sequence,
14961 });
14962 }
14963
14964 if signals.is_empty() {
14965 for signal in export_signals {
14966 if signal.get("status").and_then(Value::as_str) == Some("rejected") {
14967 continue;
14968 }
14969 let Some(name) = signal.get("name").and_then(Value::as_str) else {
14970 continue;
14971 };
14972 let codec = signal
14973 .get("payload_codec")
14974 .and_then(Value::as_str)
14975 .unwrap_or(export_codec);
14976 let (arguments, avro_arguments) =
14977 decode_query_signal_arguments(signal.get("arguments"), codec)?;
14978 signals.push(QuerySignal {
14979 id: signal.get("id").and_then(Value::as_str).map(str::to_string),
14980 name: name.to_string(),
14981 arguments,
14982 avro_arguments,
14983 workflow_sequence: signal.get("workflow_sequence").and_then(value_as_u64),
14984 });
14985 }
14986 signals.sort_by_key(|signal| signal.workflow_sequence.unwrap_or(u64::MAX));
14987 }
14988
14989 Ok(signals)
14990}
14991
14992fn decode_query_signal_arguments(
14993 raw: Option<&Value>,
14994 codec: &str,
14995) -> Result<(Vec<Value>, Vec<AvroValue>)> {
14996 validate_payload_codec(codec)?;
14997 let decoded = match raw.filter(|value| !value.is_null()) {
14998 Some(value) => decode_wire_avro_value(value, codec)?,
14999 None => AvroValue::Array(Vec::new()),
15000 };
15001 let AvroValue::Array(avro_arguments) = normalize_avro_arguments(decoded) else {
15002 unreachable!("normalize_avro_arguments always returns an array");
15003 };
15004 let arguments = avro_arguments
15005 .iter()
15006 .cloned()
15007 .map(AvroValue::into_json)
15008 .collect::<Result<Vec<_>>>()?;
15009 Ok((arguments, avro_arguments))
15010}
15011
15012fn value_as_u64(value: &Value) -> Option<u64> {
15013 value
15014 .as_u64()
15015 .or_else(|| value.as_str().and_then(|value| value.parse().ok()))
15016}
15017
15018#[cfg(test)]
15019mod tests {
15020 use super::*;
15021 mod runtime_payloads;
15022 mod runtime_uploads;
15023 use std::{
15024 fs,
15025 io::{Read, Write},
15026 net::{SocketAddr, TcpListener, TcpStream},
15027 process::Command as ProcessCommand,
15028 sync::atomic::AtomicUsize,
15029 thread,
15030 };
15031
15032 #[derive(Clone, Copy, Debug)]
15033 enum InvalidTaskPayloadCodec {
15034 Missing,
15035 Null,
15036 NonString,
15037 }
15038
15039 impl InvalidTaskPayloadCodec {
15040 fn label(self) -> &'static str {
15041 match self {
15042 Self::Missing => "missing",
15043 Self::Null => "null",
15044 Self::NonString => "non-string",
15045 }
15046 }
15047
15048 fn apply(self, task: &mut Value) {
15049 let task = task.as_object_mut().expect("task fixture object");
15050 match self {
15051 Self::Missing => {
15052 task.remove("payload_codec");
15053 }
15054 Self::Null => {
15055 task.insert("payload_codec".to_string(), Value::Null);
15056 }
15057 Self::NonString => {
15058 task.insert("payload_codec".to_string(), json!(42));
15059 }
15060 }
15061 }
15062 }
15063
15064 fn fixture_envelope(value: Value) -> Value {
15065 encode_value_envelope(&value, DEFAULT_CODEC).expect("encode Avro test fixture")
15066 }
15067
15068 fn fixture_blob(value: Value) -> String {
15069 encode_payload(&value, DEFAULT_CODEC)
15070 .expect("encode Avro test fixture")
15071 .blob
15072 }
15073
15074 #[test]
15075 fn client_builder_rejects_the_sdk_owned_api_suffix() {
15076 for base_url in [
15077 "http://127.0.0.1:8080/api",
15078 "http://localhost:8080/api/",
15079 "https://runtime.example.test/namespaces/orders/api",
15080 ] {
15081 let error = Client::builder(base_url)
15082 .build()
15083 .expect_err("SDK-owned /api suffix must be rejected during build");
15084
15085 assert!(matches!(error, Error::InvalidBaseUrl), "{base_url}");
15086 assert!(
15087 error.to_string().contains("SDK appends /api automatically"),
15088 "the validation error must explain how to fix the endpoint"
15089 );
15090 }
15091 }
15092
15093 #[test]
15094 fn client_builder_preserves_self_hosted_and_managed_runtime_prefixes() {
15095 for (base_url, expected) in [
15096 ("http://127.0.0.1:8080", "http://127.0.0.1:8080"),
15097 (
15098 "http://localhost:8080/durable-workflow/",
15099 "http://localhost:8080/durable-workflow",
15100 ),
15101 (
15102 "https://runtime.example.test/namespaces/orders",
15103 "https://runtime.example.test/namespaces/orders",
15104 ),
15105 (
15106 "https://runtime.example.test/gateway/api/namespaces/orders",
15107 "https://runtime.example.test/gateway/api/namespaces/orders",
15108 ),
15109 (
15110 "https://api.example.test/runtime/orders/",
15111 "https://api.example.test/runtime/orders",
15112 ),
15113 ] {
15114 let client = Client::builder(base_url)
15115 .build()
15116 .expect("Server and Cloud runtime base URL must remain valid");
15117
15118 assert_eq!(client.base_url, expected);
15119 }
15120 }
15121
15122 #[test]
15123 fn workflow_completion_uses_the_additive_command_protocol_floor() {
15124 assert_eq!(
15125 workflow_completion_protocol_version(&[json!({"type": "complete_workflow"})]),
15126 WORKER_PROTOCOL_VERSION
15127 );
15128 assert_eq!(
15129 workflow_completion_protocol_version(&[json!({
15130 "type": "upsert_search_attributes",
15131 "attributes": {"OrderStatus": "waiting"},
15132 })]),
15133 SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION
15134 );
15135 assert_eq!(
15136 workflow_completion_protocol_version(&[json!({
15137 "type": "upsert_search_attributes",
15138 "attributes": {"OrderStatus": "waiting"},
15139 "attribute_types": {"OrderStatus": "keyword"},
15140 })]),
15141 TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION
15142 );
15143 assert_eq!(
15144 workflow_completion_protocol_version(&[
15145 json!({"type": "upsert_memo", "entries": {"status": "waiting"}}),
15146 json!({"type": "open_condition_wait", "condition_key": "ready"}),
15147 ]),
15148 MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION
15149 );
15150 assert_eq!(
15151 workflow_completion_protocol_version(&[
15152 json!({"type": "upsert_search_attributes", "attributes": {"State": "waiting"}}),
15153 json!({"type": "open_condition_wait", "condition_key": "ready"}),
15154 ]),
15155 CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION
15156 );
15157 assert_eq!(
15158 workflow_completion_protocol_version(&[json!({
15159 "type": "open_condition_wait",
15160 "condition_wait_occurrence_id": "rust:condition-wait:0",
15161 "condition_key": "ready",
15162 })]),
15163 CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
15164 );
15165 assert_eq!(
15166 workflow_completion_protocol_version_with_message_streams(
15167 &[json!({"type": "upsert_memo", "entries": {"status": "waiting"}})],
15168 true,
15169 ),
15170 MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION
15171 );
15172 assert_eq!(
15173 workflow_completion_protocol_version_with_message_streams(
15174 &[json!({
15175 "type": "open_condition_wait",
15176 "condition_wait_occurrence_id": "rust:condition-wait:0",
15177 "condition_key": "ready",
15178 })],
15179 true,
15180 ),
15181 CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
15182 );
15183 }
15184
15185 #[test]
15186 fn portable_worker_affinity_manifest_explicitly_refuses_unimplemented_features() {
15187 let manifest = portable_worker_affinity_capability_manifest();
15188
15189 for capability in ["local_activities", "worker_sessions", "sticky_execution"] {
15190 assert_eq!(manifest[capability]["supported"], json!(false));
15191 assert_eq!(
15192 manifest[capability]["minimum_protocol_version"],
15193 json!(PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION)
15194 );
15195 assert!(manifest[capability]["reason"]
15196 .as_str()
15197 .is_some_and(|reason| !reason.is_empty()));
15198 }
15199 }
15200
15201 fn typed_fidelity_probe() -> AvroValue {
15202 AvroValue::Map(BTreeMap::from([
15203 ("bytes".to_string(), AvroValue::Bytes(vec![0, 0xff])),
15204 ("empty".to_string(), AvroValue::Map(BTreeMap::new())),
15205 (
15206 "numeric".to_string(),
15207 AvroValue::Map(BTreeMap::from([
15208 ("0".to_string(), AvroValue::String("zero".to_string())),
15209 ("1".to_string(), AvroValue::String("one".to_string())),
15210 ])),
15211 ),
15212 (
15213 "nested".to_string(),
15214 AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([(
15215 "enabled".to_string(),
15216 AvroValue::Boolean(true),
15217 )]))]),
15218 ),
15219 (
15220 "projection_collisions".to_string(),
15221 AvroValue::Array(projection_collision_probe()),
15222 ),
15223 ]))
15224 }
15225
15226 fn projection_collision_probe() -> Vec<AvroValue> {
15227 vec![
15228 AvroValue::Map(BTreeMap::from([
15229 ("$type".to_string(), AvroValue::String("bytes".to_string())),
15230 (
15231 "base64".to_string(),
15232 AvroValue::String("ordinary user text".to_string()),
15233 ),
15234 ])),
15235 AvroValue::Map(BTreeMap::from([
15236 ("$type".to_string(), AvroValue::String("map".to_string())),
15237 (
15238 "entries".to_string(),
15239 AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([
15240 ("key".to_string(), AvroValue::String("ordinary".to_string())),
15241 (
15242 "value".to_string(),
15243 AvroValue::String("user map".to_string()),
15244 ),
15245 ]))]),
15246 ),
15247 ])),
15248 ]
15249 }
15250
15251 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
15252 struct TypedContract {
15253 nested: TypedNested,
15254 mode: TypedMode,
15255 optional: Option<String>,
15256 absent: Option<String>,
15257 items: Vec<i64>,
15258 labels: BTreeMap<String, String>,
15259 bytes: serde_bytes::ByteBuf,
15260 signed: i64,
15261 finite: f64,
15262 }
15263
15264 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
15265 struct TypedNested {
15266 enabled: bool,
15267 }
15268
15269 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
15270 enum TypedMode {
15271 Detailed { label: String },
15272 }
15273
15274 fn typed_contract() -> TypedContract {
15275 TypedContract {
15276 nested: TypedNested { enabled: true },
15277 mode: TypedMode::Detailed {
15278 label: "compiler-checked".to_string(),
15279 },
15280 optional: Some("present".to_string()),
15281 absent: None,
15282 items: vec![i64::MIN, 0, i64::MAX],
15283 labels: BTreeMap::from([
15284 ("language".to_string(), "rust".to_string()),
15285 ("wire".to_string(), "avro".to_string()),
15286 ]),
15287 bytes: serde_bytes::ByteBuf::from(vec![0, 0xff, 7]),
15288 signed: -9_223_372_036_854_775_000,
15289 finite: 12.5,
15290 }
15291 }
15292
15293 #[derive(Clone, Debug, Default, PartialEq)]
15294 struct ReplayCounterState {
15295 loaded: Option<String>,
15296 count: i64,
15297 finished: bool,
15298 }
15299
15300 fn replay_counter_worker() -> Worker {
15301 let client = Client::new("http://127.0.0.1:8080").expect("client");
15302 let mut worker = Worker::new(client, "rust-workers");
15303 worker.register_replayed_workflow(
15304 "replay-counter",
15305 ReplayCounterState::default,
15306 |ctx, _input, state| async move {
15307 let loaded = ctx.activity("load-counter", json!([])).await?;
15308 state.update(|current| {
15309 current.loaded = loaded.as_str().map(str::to_string);
15310 })?;
15311 for _ in 0..2 {
15312 let signal = ctx.wait_signal("increment").await?;
15313 let amount = signal.first().and_then(Value::as_i64).unwrap_or_default();
15314 state.update(|current| current.count += amount)?;
15315 }
15316 state.update(|current| current.finished = true)?;
15317 state.read(|current| Ok(json!(current.count)))?
15318 },
15319 );
15320 worker.register_replayed_query::<ReplayCounterState, _, _>(
15321 "replay-counter",
15322 "current",
15323 |_ctx, state, _args| async move {
15324 Ok(json!({
15325 "loaded": state.loaded,
15326 "count": state.count,
15327 "finished": state.finished,
15328 }))
15329 },
15330 );
15331 worker.register_replayed_query::<ReplayCounterState, _, _>(
15332 "replay-counter",
15333 "detached-mutation",
15334 |_ctx, state, _args| async move {
15335 let mut detached = (*state).clone();
15336 detached.count = 999;
15337 Ok(json!(detached.count))
15338 },
15339 );
15340 worker.register_replayed_query::<ReplayCounterState, _, _>(
15341 "replay-counter",
15342 "failed-mutation",
15343 |_ctx, state, _args| async move {
15344 let mut detached = (*state).clone();
15345 detached.count = 999;
15346 Err(Error::WorkerLoop("query refused".to_string()))
15347 },
15348 );
15349 worker
15350 }
15351
15352 fn replay_counter_query(
15353 query_name: &str,
15354 history_events: Value,
15355 run_status: &str,
15356 ) -> QueryTask {
15357 let arguments = fixture_envelope(json!([]));
15358 serde_json::from_value(json!({
15359 "query_task_id": format!("query-{query_name}"),
15360 "workflow_type": "replay-counter",
15361 "query_name": query_name,
15362 "payload_codec": DEFAULT_CODEC,
15363 "workflow_arguments": arguments.clone(),
15364 "query_arguments": arguments,
15365 "history_events": history_events,
15366 "run_status": run_status,
15367 }))
15368 .expect("query task")
15369 }
15370
15371 fn workflow_context(history: Vec<HistoryEvent>) -> WorkflowContext {
15372 workflow_context_with_codec(history, DEFAULT_CODEC)
15373 }
15374
15375 fn workflow_context_with_codec(
15376 history: Vec<HistoryEvent>,
15377 payload_codec: &str,
15378 ) -> WorkflowContext {
15379 WorkflowContext {
15380 state: Arc::new(Mutex::new(
15381 WorkflowState::new_with_identity(
15382 history,
15383 None,
15384 None,
15385 "rust-workers".to_string(),
15386 payload_codec.to_string(),
15387 None,
15388 )
15389 .expect("valid workflow history"),
15390 )),
15391 }
15392 }
15393
15394 fn history_event(event_type: &str, payload: Value) -> HistoryEvent {
15395 HistoryEvent {
15396 event_type: event_type.to_string(),
15397 payload,
15398 raw: HashMap::new(),
15399 }
15400 }
15401
15402 fn parallel_path_entry(
15403 kind: &str,
15404 base: u64,
15405 size: usize,
15406 index: usize,
15407 ) -> ParallelGroupMetadata {
15408 parallel_group_entry(base, size, index, kind)
15409 }
15410
15411 fn parallel_history_event(
15412 event_type: &str,
15413 sequence: u64,
15414 identity_field: &str,
15415 identity: &str,
15416 path: Vec<ParallelGroupMetadata>,
15417 result: Option<Value>,
15418 ) -> HistoryEvent {
15419 let mut payload = serde_json::Map::from_iter([
15420 ("sequence".to_string(), json!(sequence)),
15421 (identity_field.to_string(), json!(identity)),
15422 ]);
15423 let inner = path.last().expect("parallel history path");
15424 apply_parallel_group_path(&mut payload, std::slice::from_ref(inner));
15425 payload.insert("parallel_group_path".to_string(), json!(path));
15426 if let Some(result) = result {
15427 let field = if event_type == "ChildRunCompleted" {
15428 "result"
15429 } else {
15430 "result"
15431 };
15432 payload.insert(field.to_string(), fixture_envelope(result));
15433 payload.insert("payload_codec".to_string(), json!(DEFAULT_CODEC));
15434 }
15435 history_event(event_type, Value::Object(payload))
15436 }
15437
15438 fn nested_parallel_operations() -> Vec<ParallelOperation> {
15439 vec![
15440 ParallelOperation::activity("first", json!([])),
15441 ParallelOperation::group(vec![
15442 ParallelOperation::child_workflow(
15443 "second",
15444 ChildWorkflowOptions::new("child-workers"),
15445 json!([]),
15446 ),
15447 ParallelOperation::activity("third", json!([])),
15448 ]),
15449 ]
15450 }
15451
15452 fn nested_parallel_paths() -> [Vec<ParallelGroupMetadata>; 3] {
15453 let outer = [
15454 parallel_path_entry("mixed", 1, 3, 0),
15455 parallel_path_entry("mixed", 1, 3, 1),
15456 parallel_path_entry("mixed", 1, 3, 2),
15457 ];
15458 [
15459 vec![outer[0].clone()],
15460 vec![outer[1].clone(), parallel_path_entry("mixed", 2, 2, 0)],
15461 vec![outer[2].clone(), parallel_path_entry("mixed", 2, 2, 1)],
15462 ]
15463 }
15464
15465 #[test]
15466 fn parallel_schedules_every_nested_mixed_leaf_with_stable_metadata() {
15467 let ctx = workflow_context(Vec::new());
15468 let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15469 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15470
15471 assert!(matches!(
15472 call.as_mut().poll(&mut task_context),
15473 Poll::Pending
15474 ));
15475 let commands = ctx.take_commands().expect("parallel commands");
15476 assert_eq!(
15477 commands
15478 .iter()
15479 .map(|command| command["type"].as_str().unwrap_or_default())
15480 .collect::<Vec<_>>(),
15481 [
15482 "schedule_activity",
15483 "start_child_workflow",
15484 "schedule_activity"
15485 ]
15486 );
15487 let paths = nested_parallel_paths();
15488 for (command, path) in commands.iter().zip(paths) {
15489 assert_eq!(command["parallel_group_path"], json!(path));
15490 assert_eq!(
15491 command["parallel_group_id"],
15492 json!(path.last().expect("inner group").parallel_group_id)
15493 );
15494 }
15495 }
15496
15497 fn completed_nested_parallel_history() -> Vec<HistoryEvent> {
15498 let paths = nested_parallel_paths();
15499 let third = parallel_history_event(
15500 "ActivityCompleted",
15501 3,
15502 "activity_type",
15503 "third",
15504 paths[2].clone(),
15505 Some(json!("three")),
15506 );
15507 vec![
15508 parallel_history_event(
15509 "ActivityCompleted",
15510 1,
15511 "activity_type",
15512 "first",
15513 paths[0].clone(),
15514 Some(json!("one")),
15515 ),
15516 parallel_history_event(
15517 "ChildWorkflowScheduled",
15518 2,
15519 "child_workflow_type",
15520 "second",
15521 paths[1].clone(),
15522 None,
15523 ),
15524 parallel_history_event(
15525 "ChildRunCompleted",
15526 2,
15527 "child_workflow_type",
15528 "second",
15529 paths[1].clone(),
15530 Some(json!("two")),
15531 ),
15532 third.clone(),
15533 third,
15534 ]
15535 }
15536
15537 #[test]
15538 fn parallel_replay_rebuilds_input_order_and_tolerates_duplicate_delivery() {
15539 for _restart_or_completed_replay in 0..2 {
15540 let ctx = workflow_context(completed_nested_parallel_history());
15541 let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15542 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15543 let Poll::Ready(Ok(results)) = call.as_mut().poll(&mut task_context) else {
15544 panic!("completed nested parallel history must replay");
15545 };
15546 assert_eq!(
15547 results,
15548 vec![
15549 ParallelResult::Activity(json!("one")),
15550 ParallelResult::Group(vec![
15551 ParallelResult::ChildWorkflow(ChildWorkflowResult {
15552 parent: WorkflowIdentity {
15553 workflow_id: None,
15554 run_id: None,
15555 },
15556 child: WorkflowIdentity {
15557 workflow_id: None,
15558 run_id: None,
15559 },
15560 child_workflow_type: Some("second".to_string()),
15561 result: json!("two"),
15562 }),
15563 ParallelResult::Activity(json!("three")),
15564 ]),
15565 ]
15566 );
15567 assert!(ctx.take_commands().expect("commands").is_empty());
15568 ctx.ensure_history_consumed().expect("history consumed");
15569 }
15570 }
15571
15572 #[test]
15573 fn parallel_failure_keeps_typed_cause_path_and_late_completions() {
15574 let paths = nested_parallel_paths();
15575 let history = vec![
15576 parallel_history_event(
15577 "ActivityCompleted",
15578 1,
15579 "activity_type",
15580 "first",
15581 paths[0].clone(),
15582 Some(json!("one")),
15583 ),
15584 parallel_history_event(
15585 "ChildWorkflowScheduled",
15586 2,
15587 "child_workflow_type",
15588 "second",
15589 paths[1].clone(),
15590 None,
15591 ),
15592 parallel_history_event(
15593 "ChildRunFailed",
15594 2,
15595 "child_workflow_type",
15596 "second",
15597 paths[1].clone(),
15598 None,
15599 ),
15600 parallel_history_event(
15601 "ActivityCompleted",
15602 3,
15603 "activity_type",
15604 "third",
15605 paths[2].clone(),
15606 Some(json!("late")),
15607 ),
15608 ];
15609 let ctx = workflow_context(history);
15610 let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15611 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15612 let outcome = call.as_mut().poll(&mut task_context);
15613 let Poll::Ready(Err(Error::ParallelFailed(failure))) = outcome else {
15614 panic!("one failed child must return a typed partial failure: {outcome:?}");
15615 };
15616 assert_eq!(failure.member_path, [1, 0]);
15617 assert_eq!(failure.group_id, "parallel-calls:1:3");
15618 assert!(matches!(*failure.cause, Error::ChildWorkflowFailed(_)));
15619 assert_eq!(
15620 failure
15621 .completed
15622 .iter()
15623 .map(|completion| completion.member_path.clone())
15624 .collect::<Vec<_>>(),
15625 [vec![0], vec![1, 1]]
15626 );
15627 }
15628
15629 #[test]
15630 fn pending_parallel_history_restarts_without_rescheduling_any_leaf() {
15631 let paths = nested_parallel_paths();
15632 let history = vec![
15633 parallel_history_event(
15634 "ActivityScheduled",
15635 1,
15636 "activity_type",
15637 "first",
15638 paths[0].clone(),
15639 None,
15640 ),
15641 parallel_history_event(
15642 "ChildWorkflowScheduled",
15643 2,
15644 "child_workflow_type",
15645 "second",
15646 paths[1].clone(),
15647 None,
15648 ),
15649 parallel_history_event(
15650 "ActivityScheduled",
15651 3,
15652 "activity_type",
15653 "third",
15654 paths[2].clone(),
15655 None,
15656 ),
15657 ];
15658 for _restart in 0..2 {
15659 let ctx = workflow_context(history.clone());
15660 let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15661 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15662 let outcome = call.as_mut().poll(&mut task_context);
15663 assert!(matches!(outcome, Poll::Pending), "{outcome:?}");
15664 assert!(ctx.take_commands().expect("commands").is_empty());
15665 }
15666 }
15667
15668 fn selection_path(index: usize, key: &str) -> Vec<ParallelGroupMetadata> {
15669 vec![selection_group_entry(
15670 1,
15671 2,
15672 index,
15673 "activity",
15674 &SelectionMemberMetadata {
15675 key: SelectionKey::Name(key.to_string()),
15676 index,
15677 base_sequence: index as u64 + 1,
15678 size: 1,
15679 kind: "activity".to_string(),
15680 },
15681 )]
15682 }
15683
15684 fn selection_activity_event(
15685 event_type: &str,
15686 index: usize,
15687 key: &str,
15688 result: Option<Value>,
15689 ) -> HistoryEvent {
15690 let sequence = index as u64 + 1;
15691 let mut event = parallel_history_event(
15692 event_type,
15693 sequence,
15694 "activity_type",
15695 &format!("{key}-activity"),
15696 selection_path(index, key),
15697 result,
15698 );
15699 event.payload["activity_execution_id"] = json!(format!("activity-{key}"));
15700 event.raw.insert(
15701 "id".to_string(),
15702 json!(if event_type == "ActivityCompleted" {
15703 format!("event-{key}")
15704 } else {
15705 format!("{event_type}-{key}")
15706 }),
15707 );
15708 event
15709 }
15710
15711 fn selection_winner_marker() -> HistoryEvent {
15712 history_event(
15713 "SelectionResolved",
15714 json!({
15715 "selection_group_id": "select-calls:1:2",
15716 "selection_group_base_sequence": 1,
15717 "selection_group_size": 2,
15718 "member_key": "fast",
15719 "member_index": 1,
15720 "member_base_sequence": 2,
15721 "member_size": 1,
15722 "operation_kind": "activity",
15723 "operation_identity": "activity-fast",
15724 "outcome": "completed",
15725 "resolution_event_id": "event-fast",
15726 "resolution_event_type": "ActivityCompleted",
15727 }),
15728 )
15729 }
15730
15731 fn keyed_activity_selection(ctx: &WorkflowContext) -> SelectCall {
15732 ctx.select_keyed(vec![
15733 (
15734 "slow",
15735 ParallelOperation::activity_with_options(
15736 "slow-activity",
15737 ActivityOptions::new().task_queue("default"),
15738 json!([]),
15739 ),
15740 ),
15741 (
15742 "fast",
15743 ParallelOperation::activity_with_options(
15744 "fast-activity",
15745 ActivityOptions::new().task_queue("default"),
15746 json!([]),
15747 ),
15748 ),
15749 ])
15750 }
15751
15752 fn assert_persisted_selection_replay(history: Vec<HistoryEvent>) {
15753 let ctx = workflow_context(history);
15754 let mut call = Box::pin(keyed_activity_selection(&ctx));
15755 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15756 let selected = match call.as_mut().poll(&mut task_context) {
15757 Poll::Ready(Ok(selected)) => selected,
15758 Poll::Ready(Err(error)) => panic!("persisted selection winner must replay: {error:?}"),
15759 Poll::Pending => panic!("persisted selection winner must replay without pending"),
15760 };
15761 assert_eq!(selected.key, SelectionKey::Name("fast".to_string()));
15762 assert_eq!(
15763 selected.value,
15764 Some(ParallelResult::Activity(json!("winner-value")))
15765 );
15766 let slow = selected
15767 .handle(&SelectionKey::Name("slow".to_string()))
15768 .expect("slow handle")
15769 .clone();
15770 let mut await_slow = Box::pin(slow.await_result());
15771 assert!(matches!(
15772 await_slow.as_mut().poll(&mut task_context),
15773 Poll::Ready(Ok(ParallelResult::Activity(value))) if value == json!("loser-value")
15774 ));
15775 assert!(ctx.take_commands().expect("commands").is_empty());
15776 }
15777
15778 const SELECTION_COLD_REPLAY_HISTORY: &str = "DURABLE_WORKFLOW_SELECTION_COLD_REPLAY_HISTORY";
15779
15780 fn canonical_selection_history() -> Vec<HistoryEvent> {
15781 const FIXTURE: &[u8] =
15782 include_bytes!("../tests/fixtures/durable_selection_runtime_history.json");
15783 assert_eq!(
15784 format!("{:x}", Sha256::digest(FIXTURE)),
15785 "51fd8b9c16e978dcef536a5c727b9fdc0ae724d9afc17d9a7837d219f41ee3ba",
15786 );
15787 let fixture: Value = serde_json::from_slice(FIXTURE).expect("canonical selection fixture");
15788
15789 serde_json::from_value(fixture["history"].clone()).expect("canonical selection history")
15790 }
15791
15792 #[test]
15793 fn selection_fresh_process_entrypoint() {
15794 let Ok(path) = std::env::var(SELECTION_COLD_REPLAY_HISTORY) else {
15795 return;
15796 };
15797 let persisted = fs::read(path).expect("persisted selection history");
15798 assert_eq!(
15799 format!("{:x}", Sha256::digest(&persisted)),
15800 "51fd8b9c16e978dcef536a5c727b9fdc0ae724d9afc17d9a7837d219f41ee3ba",
15801 );
15802 let fixture: Value =
15803 serde_json::from_slice(&persisted).expect("valid persisted selection fixture");
15804 let history: Vec<HistoryEvent> = serde_json::from_value(fixture["history"].clone())
15805 .expect("valid persisted selection history");
15806
15807 assert_persisted_selection_replay(history);
15808 }
15809
15810 #[test]
15811 fn selection_starts_every_member_with_stable_keys_and_group_identity() {
15812 let ctx = workflow_context(Vec::new());
15813 let mut call = Box::pin(keyed_activity_selection(&ctx));
15814 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15815
15816 assert!(matches!(
15817 call.as_mut().poll(&mut task_context),
15818 Poll::Pending
15819 ));
15820 let commands = ctx.take_commands().expect("selection commands");
15821 assert_eq!(commands.len(), 2);
15822 assert_eq!(commands[0]["selection_member_key"], json!("slow"));
15823 assert_eq!(commands[1]["selection_member_key"], json!("fast"));
15824 assert!(commands.iter().all(|command| {
15825 command["parallel_group_id"] == json!("select-calls:1:2")
15826 && command["parallel_group_mode"] == json!("select")
15827 }));
15828 }
15829
15830 #[test]
15831 fn selection_key_domain_rejects_empty_authoring_and_malformed_history() {
15832 let ctx = workflow_context(Vec::new());
15833 let mut invalid = Box::pin(ctx.select_keyed(vec![(
15834 "",
15835 ParallelOperation::activity("invalid", json!([])),
15836 )]));
15837 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15838 assert!(matches!(
15839 invalid.as_mut().poll(&mut task_context),
15840 Poll::Ready(Err(Error::InvalidParallelGroup(ParallelGroupError {
15841 reason: "selection_key_invalid",
15842 ..
15843 })))
15844 ));
15845
15846 for invalid_key in [json!(""), json!(-1)] {
15847 let mut event = selection_activity_event("ActivityScheduled", 0, "slow", None);
15848 event.payload["selection_member_key"] = invalid_key.clone();
15849 event.payload["parallel_group_path"][0]["selection_member_key"] = invalid_key;
15850 assert!(matches!(
15851 WorkflowState::new_with_identity(
15852 vec![event],
15853 None,
15854 None,
15855 "rust-workers".to_string(),
15856 DEFAULT_CODEC.to_string(),
15857 None,
15858 ),
15859 Err(Error::NonDeterministicReplay(_))
15860 ));
15861 }
15862 }
15863
15864 #[test]
15865 fn selection_preserves_valid_named_and_numeric_keys() {
15866 let ctx = workflow_context(Vec::new());
15867 let mut selection = Box::pin(ctx.select_keyed(vec![
15868 (
15869 SelectionKey::Index(0),
15870 ParallelOperation::activity("numeric", json!([])),
15871 ),
15872 (
15873 SelectionKey::Name("named".to_string()),
15874 ParallelOperation::timer(Duration::from_secs(1)),
15875 ),
15876 ]));
15877 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15878
15879 assert!(matches!(
15880 selection.as_mut().poll(&mut task_context),
15881 Poll::Pending
15882 ));
15883 let commands = ctx.take_commands().expect("selection commands");
15884 assert_eq!(commands[0]["selection_member_key"], json!(0));
15885 assert_eq!(commands[1]["selection_member_key"], json!("named"));
15886 }
15887
15888 #[test]
15889 fn selection_replays_persisted_winner_and_loser_can_be_awaited_later() {
15890 let history = canonical_selection_history();
15891 assert_persisted_selection_replay(history.clone());
15892
15893 let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
15894 .join("tests/fixtures/durable_selection_runtime_history.json");
15895 let output =
15896 ProcessCommand::new(std::env::current_exe().expect("current Rust test binary"))
15897 .args([
15898 "--exact",
15899 "tests::selection_fresh_process_entrypoint",
15900 "--nocapture",
15901 ])
15902 .env(SELECTION_COLD_REPLAY_HISTORY, &path)
15903 .output()
15904 .expect("run fresh selection replay process");
15905
15906 assert!(
15907 output.status.success(),
15908 "fresh selection replay failed:\nstdout:\n{}\nstderr:\n{}",
15909 String::from_utf8_lossy(&output.stdout),
15910 String::from_utf8_lossy(&output.stderr),
15911 );
15912 }
15913
15914 #[test]
15915 fn selection_waits_durably_when_terminal_members_precede_the_winner_marker() {
15916 let mut history = canonical_selection_history();
15917 history.retain(|event| event.event_type != "SelectionResolved");
15918 let ctx = workflow_context(history);
15919 let mut selection = Box::pin(keyed_activity_selection(&ctx));
15920 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15921
15922 assert!(matches!(
15923 selection.as_mut().poll(&mut task_context),
15924 Poll::Pending
15925 ));
15926 assert!(ctx.take_commands().expect("commands").is_empty());
15927 assert!(
15928 ctx.matched_recorded_pending()
15929 .expect("selection pending state"),
15930 "terminal member history must keep the workflow durably pending until SelectionResolved commits"
15931 );
15932 }
15933
15934 #[test]
15935 fn selection_terminal_condition_history_waits_durably_for_its_winner_marker() {
15936 for (terminal_event, predicate_satisfied, timeout_seconds) in [
15937 ("ConditionWaitSatisfied", true, None),
15938 ("ConditionWaitTimedOut", false, Some(0)),
15939 ] {
15940 let member = SelectionMemberMetadata {
15941 key: SelectionKey::Name("condition".to_string()),
15942 index: 0,
15943 base_sequence: 1,
15944 size: 1,
15945 kind: "condition".to_string(),
15946 };
15947 let path = vec![selection_group_entry(1, 1, 0, "condition", &member)];
15948 let mut payload = json!({
15949 "sequence": 1,
15950 "condition_wait_id": "condition-1",
15951 "condition_wait_occurrence_id": "rust:condition-wait:0",
15952 "condition_key": "ready",
15953 "condition_definition_fingerprint": "sha256:ready-v1",
15954 "parallel_group_path": path,
15955 });
15956 payload
15957 .as_object_mut()
15958 .expect("condition history payload")
15959 .extend(
15960 serde_json::to_value(&path[0])
15961 .expect("condition selection metadata")
15962 .as_object()
15963 .expect("condition selection metadata object")
15964 .clone(),
15965 );
15966 if let Some(timeout_seconds) = timeout_seconds {
15967 payload["timeout_seconds"] = json!(timeout_seconds);
15968 }
15969 let history = vec![
15970 history_event("ConditionWaitOpened", payload.clone()),
15971 history_event(terminal_event, payload),
15972 ];
15973 let ctx = workflow_context(history);
15974 let mut options = ConditionWaitOptions::new("ready", "sha256:ready-v1");
15975 if timeout_seconds.is_some() {
15976 options = options.timeout(Duration::ZERO);
15977 }
15978 let mut selection = Box::pin(ctx.select_keyed(vec![(
15979 "condition",
15980 ParallelOperation::condition(options, move || Ok(predicate_satisfied)),
15981 )]));
15982 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15983
15984 assert!(matches!(
15985 selection.as_mut().poll(&mut task_context),
15986 Poll::Pending
15987 ));
15988 assert!(ctx.take_commands().expect("commands").is_empty());
15989 assert!(
15990 ctx.matched_recorded_pending()
15991 .expect("condition selection pending state"),
15992 "{terminal_event} must keep the workflow durably pending until SelectionResolved commits"
15993 );
15994 }
15995 }
15996
15997 #[test]
15998 fn selection_immediate_condition_members_open_a_durable_wait() {
15999 for predicate_satisfied in [true, false] {
16000 let ctx = workflow_context(Vec::new());
16001 let mut selection = Box::pin(ctx.select_keyed(vec![(
16002 "condition",
16003 ParallelOperation::condition(
16004 ConditionWaitOptions::new("ready", "sha256:ready-v1").timeout(Duration::ZERO),
16005 move || Ok(predicate_satisfied),
16006 ),
16007 )]));
16008 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16009
16010 assert!(matches!(
16011 selection.as_mut().poll(&mut task_context),
16012 Poll::Pending
16013 ));
16014 let commands = ctx.take_commands().expect("condition selection command");
16015 assert_eq!(commands.len(), 1);
16016 assert_eq!(commands[0]["type"], json!("open_condition_wait"));
16017 assert_eq!(commands[0]["timeout_seconds"], json!(0));
16018 assert_eq!(
16019 commands[0]["parallel_group_path"][0]["parallel_group_mode"],
16020 json!("select")
16021 );
16022 }
16023 }
16024
16025 #[test]
16026 fn selection_loser_cancellation_is_explicit_and_idempotent() {
16027 let history = vec![
16028 selection_activity_event("ActivityScheduled", 0, "slow", None),
16029 selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
16030 selection_winner_marker(),
16031 ];
16032 let ctx = workflow_context(history.clone());
16033 let mut call = Box::pin(keyed_activity_selection(&ctx));
16034 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16035 let Poll::Ready(Ok(selected)) = call.as_mut().poll(&mut task_context) else {
16036 panic!("winner must replay");
16037 };
16038 let slow = selected
16039 .handle(&SelectionKey::Name("slow".to_string()))
16040 .expect("slow handle")
16041 .clone();
16042 let mut cancel = Box::pin(slow.cancel());
16043 assert!(matches!(
16044 cancel.as_mut().poll(&mut task_context),
16045 Poll::Pending
16046 ));
16047 assert!(matches!(
16048 cancel.as_mut().poll(&mut task_context),
16049 Poll::Pending
16050 ));
16051 let commands = ctx.take_commands().expect("cancel command");
16052 assert_eq!(commands.len(), 1);
16053 assert_eq!(commands[0]["type"], json!("cancel_selection_operation"));
16054 assert_eq!(commands[0]["member_key"], json!("slow"));
16055
16056 let mut cancelled_history = history;
16057 cancelled_history.push(history_event(
16058 "SelectionOperationCancelled",
16059 json!({
16060 "selection_group_id": "select-calls:1:2",
16061 "member_key": "slow",
16062 "member_index": 0,
16063 "member_base_sequence": 1,
16064 "member_size": 1,
16065 "operation_kind": "activity",
16066 "operation_identity": "activity-slow",
16067 "cancelled_at": "2026-08-27T00:00:00Z",
16068 }),
16069 ));
16070 let replayed = workflow_context(cancelled_history);
16071 let mut call = Box::pin(keyed_activity_selection(&replayed));
16072 let Poll::Ready(Ok(selected)) = call.as_mut().poll(&mut task_context) else {
16073 panic!("winner must replay after cancellation");
16074 };
16075 let slow = selected
16076 .handle(&SelectionKey::Name("slow".to_string()))
16077 .expect("slow handle")
16078 .clone();
16079 let mut cancel = Box::pin(slow.cancel());
16080 assert!(matches!(
16081 cancel.as_mut().poll(&mut task_context),
16082 Poll::Ready(Ok(()))
16083 ));
16084 assert!(replayed.take_commands().expect("commands").is_empty());
16085 }
16086
16087 #[test]
16088 fn selection_cancellation_marker_is_bound_to_every_authored_handle_field() {
16089 let base_history = vec![
16090 selection_activity_event("ActivityScheduled", 0, "slow", None),
16091 selection_activity_event("ActivityScheduled", 1, "fast", None),
16092 selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
16093 selection_winner_marker(),
16094 ];
16095 for (field, corrupt) in [
16096 ("member_key", json!("fast")),
16097 ("member_index", json!(1)),
16098 ("member_base_sequence", json!(3)),
16099 ("member_size", json!(2)),
16100 ("operation_kind", json!("timer")),
16101 ("operation_identity", json!("forged")),
16102 ] {
16103 let mut cancellation = json!({
16104 "selection_group_id": "select-calls:1:2",
16105 "member_key": "slow",
16106 "member_index": 0,
16107 "member_base_sequence": 1,
16108 "member_size": 1,
16109 "operation_kind": "activity",
16110 "operation_identity": "activity-slow",
16111 });
16112 cancellation[field] = corrupt;
16113 let mut history = base_history.clone();
16114 history.push(history_event("SelectionOperationCancelled", cancellation));
16115 let ctx = workflow_context(history);
16116 let mut selection = Box::pin(keyed_activity_selection(&ctx));
16117 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16118
16119 assert!(matches!(
16120 selection.as_mut().poll(&mut task_context),
16121 Poll::Ready(Err(Error::NonDeterministicReplay(_)))
16122 ));
16123 }
16124 }
16125
16126 #[test]
16127 fn selection_child_identity_prefers_the_durable_run_id() {
16128 let ctx = workflow_context(vec![history_event(
16129 "ChildWorkflowScheduled",
16130 json!({
16131 "sequence": 1,
16132 "child_workflow_type": "child",
16133 "child_workflow_instance_id": "child-instance",
16134 "child_workflow_run_id": "child-run",
16135 }),
16136 )]);
16137 let state = ctx.state.lock().expect("workflow state");
16138
16139 assert_eq!(
16140 selection_operation_identity(&state, "child", 1, 1),
16141 "child-run"
16142 );
16143 }
16144
16145 #[test]
16146 fn selection_activity_identity_requires_canonical_execution_id() {
16147 let slow = selection_activity_event("ActivityScheduled", 0, "slow", None);
16148 let mut fast_open = selection_activity_event("ActivityScheduled", 1, "fast", None);
16149 let mut fast_completed =
16150 selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner")));
16151 for event in [&mut fast_open, &mut fast_completed] {
16152 event
16153 .payload
16154 .as_object_mut()
16155 .expect("activity payload")
16156 .remove("activity_execution_id");
16157 event.payload["activity_id"] = json!("forged-activity-id");
16158 }
16159 let mut marker = selection_winner_marker();
16160 marker.payload["operation_identity"] = json!("forged-activity-id");
16161 let ctx = workflow_context(vec![slow, fast_open, fast_completed, marker]);
16162 let mut selection = Box::pin(keyed_activity_selection(&ctx));
16163 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16164
16165 assert!(matches!(
16166 selection.as_mut().poll(&mut task_context),
16167 Poll::Ready(Err(Error::NonDeterministicReplay(_)))
16168 ));
16169 }
16170
16171 #[test]
16172 fn selection_completion_before_cancellation_remains_awaitable() {
16173 let history = vec![
16174 selection_activity_event("ActivityScheduled", 0, "slow", None),
16175 selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
16176 selection_winner_marker(),
16177 selection_activity_event(
16178 "ActivityCompleted",
16179 0,
16180 "slow",
16181 Some(json!("completed-first")),
16182 ),
16183 ];
16184 let ctx = workflow_context(history);
16185 let mut selection = Box::pin(keyed_activity_selection(&ctx));
16186 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16187 let Poll::Ready(Ok(selected)) = selection.as_mut().poll(&mut task_context) else {
16188 panic!("winner must replay");
16189 };
16190 let slow = selected
16191 .handle(&SelectionKey::Name("slow".to_string()))
16192 .expect("slow handle")
16193 .clone();
16194 let mut cancel = Box::pin(slow.cancel());
16195 assert!(matches!(
16196 cancel.as_mut().poll(&mut task_context),
16197 Poll::Ready(Ok(()))
16198 ));
16199 let mut await_slow = Box::pin(slow.await_result());
16200 assert!(matches!(
16201 await_slow.as_mut().poll(&mut task_context),
16202 Poll::Ready(Ok(ParallelResult::Activity(value))) if value == json!("completed-first")
16203 ));
16204 let commands = ctx.take_commands().expect("commands");
16205 assert!(commands.is_empty());
16206 }
16207
16208 #[test]
16209 fn selection_nested_later_failure_before_cancel_remains_the_awaited_failure() {
16210 let nested_member = SelectionMemberMetadata {
16211 key: SelectionKey::Name("nested".to_string()),
16212 index: 0,
16213 base_sequence: 1,
16214 size: 2,
16215 kind: "group".to_string(),
16216 };
16217 let deadline_member = SelectionMemberMetadata {
16218 key: SelectionKey::Name("deadline".to_string()),
16219 index: 1,
16220 base_sequence: 3,
16221 size: 1,
16222 kind: "timer".to_string(),
16223 };
16224 let nested_paths = [
16225 vec![
16226 selection_group_entry(1, 3, 0, "mixed", &nested_member),
16227 parallel_group_entry(1, 2, 0, "activity"),
16228 ],
16229 vec![
16230 selection_group_entry(1, 3, 1, "mixed", &nested_member),
16231 parallel_group_entry(1, 2, 1, "activity"),
16232 ],
16233 ];
16234 let deadline_path = vec![selection_group_entry(1, 3, 2, "mixed", &deadline_member)];
16235 let mut timer_fired = parallel_history_event(
16236 "TimerFired",
16237 3,
16238 "timer_id",
16239 "timer-3",
16240 deadline_path.clone(),
16241 None,
16242 );
16243 timer_fired.payload["delay_seconds"] = json!(0);
16244 timer_fired
16245 .raw
16246 .insert("id".to_string(), json!("timer-fired"));
16247 let mut timer_scheduled = parallel_history_event(
16248 "TimerScheduled",
16249 3,
16250 "timer_id",
16251 "timer-3",
16252 deadline_path,
16253 None,
16254 );
16255 timer_scheduled.payload["delay_seconds"] = json!(0);
16256 let history = vec![
16257 parallel_history_event(
16258 "ActivityScheduled",
16259 1,
16260 "activity_type",
16261 "nested-first",
16262 nested_paths[0].clone(),
16263 None,
16264 ),
16265 parallel_history_event(
16266 "ActivityScheduled",
16267 2,
16268 "activity_type",
16269 "nested-second",
16270 nested_paths[1].clone(),
16271 None,
16272 ),
16273 timer_scheduled,
16274 timer_fired,
16275 history_event(
16276 "SelectionResolved",
16277 json!({
16278 "selection_group_id": "select-calls:1:3",
16279 "selection_group_base_sequence": 1,
16280 "selection_group_size": 3,
16281 "member_key": "deadline",
16282 "member_index": 1,
16283 "member_base_sequence": 3,
16284 "member_size": 1,
16285 "operation_kind": "timer",
16286 "operation_identity": "timer-3",
16287 "outcome": "completed",
16288 "resolution_event_id": "timer-fired",
16289 "resolution_event_type": "TimerFired",
16290 }),
16291 ),
16292 parallel_history_event(
16293 "ActivityFailed",
16294 2,
16295 "activity_type",
16296 "nested-second",
16297 nested_paths[1].clone(),
16298 None,
16299 ),
16300 ];
16301 let ctx = workflow_context(history);
16302 let mut selection = Box::pin(ctx.select_keyed(vec![
16303 (
16304 "nested",
16305 ParallelOperation::group(vec![
16306 ParallelOperation::activity("nested-first", json!([])),
16307 ParallelOperation::activity("nested-second", json!([])),
16308 ]),
16309 ),
16310 ("deadline", ParallelOperation::timer(Duration::ZERO)),
16311 ]));
16312 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16313 let Poll::Ready(Ok(selected)) = selection.as_mut().poll(&mut task_context) else {
16314 panic!("deadline winner must replay");
16315 };
16316 let nested = selected
16317 .handle(&SelectionKey::Name("nested".to_string()))
16318 .expect("nested handle")
16319 .clone();
16320 let mut cancel = Box::pin(nested.cancel());
16321 assert!(matches!(
16322 cancel.as_mut().poll(&mut task_context),
16323 Poll::Ready(Ok(()))
16324 ));
16325 let mut await_nested = Box::pin(nested.await_result());
16326
16327 assert!(matches!(
16328 await_nested.as_mut().poll(&mut task_context),
16329 Poll::Ready(Err(Error::ActivityFailed(_)))
16330 ));
16331 assert!(ctx.take_commands().expect("commands").is_empty());
16332 }
16333
16334 #[test]
16335 fn selection_supports_child_timer_signal_condition_and_nested_groups() {
16336 let ctx = workflow_context(Vec::new());
16337 let mut call = Box::pin(ctx.select(vec![
16338 ParallelOperation::child_workflow(
16339 "child",
16340 ChildWorkflowOptions::new("children"),
16341 json!([]),
16342 ),
16343 ParallelOperation::timer(Duration::from_secs(30)),
16344 ParallelOperation::signal("approval"),
16345 ParallelOperation::condition(
16346 ConditionWaitOptions::new("ready", "sha256:ready"),
16347 || Ok(false),
16348 ),
16349 ParallelOperation::group(vec![
16350 ParallelOperation::activity("nested-one", json!([])),
16351 ParallelOperation::activity("nested-two", json!([])),
16352 ]),
16353 ]));
16354 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16355 assert!(matches!(
16356 call.as_mut().poll(&mut task_context),
16357 Poll::Pending
16358 ));
16359 let commands = ctx.take_commands().expect("selection commands");
16360 assert_eq!(
16361 commands
16362 .iter()
16363 .map(|command| command["type"].as_str().unwrap_or_default())
16364 .collect::<Vec<_>>(),
16365 [
16366 "start_child_workflow",
16367 "start_timer",
16368 "open_signal_wait",
16369 "open_condition_wait",
16370 "schedule_activity",
16371 "schedule_activity",
16372 ]
16373 );
16374 assert!(commands.iter().all(|command| {
16375 command["parallel_group_path"][0]["parallel_group_mode"] == json!("select")
16376 }));
16377 assert_eq!(
16378 commands[4]["parallel_group_path"].as_array().map(Vec::len),
16379 Some(2)
16380 );
16381 assert_eq!(
16382 commands[4]["parallel_group_path"][0]["selection_member_kind"],
16383 json!("group")
16384 );
16385 assert_eq!(
16386 commands[5]["parallel_group_path"][0]["selection_member_kind"],
16387 json!("group")
16388 );
16389
16390 let one_leaf_ctx = workflow_context(Vec::new());
16391 let mut one_leaf = Box::pin(one_leaf_ctx.select(vec![ParallelOperation::group(vec![
16392 ParallelOperation::activity("nested-only", json!([])),
16393 ])]));
16394 assert!(matches!(
16395 one_leaf.as_mut().poll(&mut task_context),
16396 Poll::Pending
16397 ));
16398 let one_leaf_commands = one_leaf_ctx.take_commands().expect("one-leaf commands");
16399 assert_eq!(one_leaf_commands.len(), 1);
16400 assert_eq!(
16401 one_leaf_commands[0]["parallel_group_path"][0]["selection_member_kind"],
16402 json!("group")
16403 );
16404 assert_eq!(
16405 one_leaf_commands[0]["parallel_group_path"][0]["selection_member_size"],
16406 json!(1)
16407 );
16408 }
16409
16410 async fn trip_saga(ctx: WorkflowContext) -> Result<Value> {
16411 let mut saga = ctx.saga();
16412 let outcome = async {
16413 let flight = ctx.activity("trip.reserve-flight", json!([])).await?;
16414 saga.add_compensation("trip.cancel-flight", json!([flight]))?;
16415 let hotel = ctx.activity("trip.reserve-hotel", json!([])).await?;
16416 saga.add_compensation("trip.cancel-hotel", json!([hotel]))?;
16417 ctx.activity("trip.charge", json!([])).await?;
16418 Ok(json!({"status": "booked"}))
16419 }
16420 .await;
16421 saga.finish(outcome).await
16422 }
16423
16424 fn saga_activity(
16425 event_type: &str,
16426 sequence: u64,
16427 activity_type: &str,
16428 result: Option<Value>,
16429 ) -> HistoryEvent {
16430 let mut payload = json!({
16431 "sequence": sequence,
16432 "activity_type": activity_type,
16433 "message": format!("{activity_type} failed"),
16434 "exception_type": "PlannedFailure",
16435 "non_retryable": true,
16436 });
16437 if let Some(result) = result {
16438 payload["result"] = fixture_envelope(result);
16439 }
16440 history_event(event_type, payload)
16441 }
16442
16443 #[test]
16444 fn saga_replays_reverse_compensation_across_restart_and_duplicate_delivery() {
16445 let completed_hotel_compensation = saga_activity(
16446 "ActivityCompleted",
16447 4,
16448 "trip.cancel-hotel",
16449 Some(Value::Null),
16450 );
16451 let history = vec![
16452 saga_activity(
16453 "ActivityCompleted",
16454 1,
16455 "trip.reserve-flight",
16456 Some(json!("flight-1")),
16457 ),
16458 saga_activity(
16459 "ActivityCompleted",
16460 2,
16461 "trip.reserve-hotel",
16462 Some(json!("hotel-1")),
16463 ),
16464 saga_activity("ActivityFailed", 3, "trip.charge", None),
16465 completed_hotel_compensation.clone(),
16466 completed_hotel_compensation,
16467 ];
16468
16469 for _restart in 0..2 {
16470 let ctx = workflow_context(history.clone());
16471 let mut future = Box::pin(trip_saga(ctx.clone()));
16472 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16473 assert!(matches!(
16474 future.as_mut().poll(&mut task_context),
16475 Poll::Pending
16476 ));
16477 let commands = ctx.take_commands().expect("compensation command");
16478 assert_eq!(commands.len(), 1);
16479 assert_eq!(commands[0]["activity_type"], "trip.cancel-flight");
16480 }
16481 }
16482
16483 #[test]
16484 fn saga_compensation_failure_preserves_both_typed_failures() {
16485 let history = vec![
16486 saga_activity(
16487 "ActivityCompleted",
16488 1,
16489 "trip.reserve-flight",
16490 Some(json!("flight-1")),
16491 ),
16492 saga_activity(
16493 "ActivityCompleted",
16494 2,
16495 "trip.reserve-hotel",
16496 Some(json!("hotel-1")),
16497 ),
16498 saga_activity("ActivityFailed", 3, "trip.charge", None),
16499 saga_activity("ActivityFailed", 4, "trip.cancel-hotel", None),
16500 ];
16501 let ctx = workflow_context(history);
16502 let mut future = Box::pin(trip_saga(ctx));
16503 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16504 let Poll::Ready(Err(Error::SagaCompensationFailed(failure))) =
16505 future.as_mut().poll(&mut task_context)
16506 else {
16507 panic!("compensation failure must remain structured");
16508 };
16509 assert!(matches!(
16510 *failure.initiating_failure,
16511 Error::ActivityFailed(_)
16512 ));
16513 assert!(matches!(
16514 *failure.compensation_failure,
16515 Error::ActivityFailed(_)
16516 ));
16517 assert_eq!(failure.compensation_activity_type, "trip.cancel-hotel");
16518 assert_eq!(failure.compensation_registration_order, 2);
16519 }
16520
16521 #[test]
16522 fn saga_compensates_cooperative_cancellation() {
16523 let ctx = workflow_context(vec![saga_activity(
16524 "ActivityCompleted",
16525 1,
16526 "trip.reserve-flight",
16527 Some(json!("flight-1")),
16528 )]);
16529 ctx.state.lock().expect("state").cancel_requested = true;
16530 let run = {
16531 let ctx = ctx.clone();
16532 async move {
16533 let mut saga = ctx.saga();
16534 let outcome = async {
16535 let flight = ctx.activity("trip.reserve-flight", json!([])).await?;
16536 saga.add_compensation("trip.cancel-flight", json!([flight]))?;
16537 ctx.throw_if_cancellation_requested()?;
16538 Ok(json!("unexpected"))
16539 }
16540 .await;
16541 saga.finish(outcome).await
16542 }
16543 };
16544 let mut future = Box::pin(run);
16545 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16546 assert!(matches!(
16547 future.as_mut().poll(&mut task_context),
16548 Poll::Pending
16549 ));
16550 let commands = ctx.take_commands().expect("cancellation compensation");
16551 assert_eq!(commands[0]["activity_type"], "trip.cancel-flight");
16552 }
16553
16554 fn workflow_task(
16555 workflow_type: &str,
16556 history_events: Vec<HistoryEvent>,
16557 payload_codec: &str,
16558 ) -> WorkflowTask {
16559 WorkflowTask {
16560 task_id: format!("wft-{workflow_type}"),
16561 workflow_command_id: None,
16562 workflow_id: Some(format!("wf-{workflow_type}")),
16563 run_id: Some(format!("run-{workflow_type}")),
16564 workflow_type: workflow_type.to_string(),
16565 cancel_requested: false,
16566 payload_codec: payload_codec.to_string(),
16567 arguments: Some(
16568 encode_value_envelope(&json!([]), payload_codec).expect("workflow arguments"),
16569 ),
16570 total_history_events: Some(history_events.len() as u64),
16571 history_size_bytes: None,
16572 continue_as_new_recommended: None,
16573 history_budget_pressure: None,
16574 history_events,
16575 next_history_page_token: None,
16576 workflow_task_attempt: 1,
16577 workflow_signal_id: None,
16578 signal_name: None,
16579 signal_arguments: None,
16580 workflow_update_id: None,
16581 update_name: None,
16582 lease_owner: Some("rust-worker".to_string()),
16583 }
16584 }
16585
16586 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
16587 struct SideEffectProbe {
16588 request_id: String,
16589 attempt: u32,
16590 }
16591
16592 #[test]
16593 fn typed_side_effect_runs_callback_once_and_replay_skips_it() {
16594 let calls = AtomicUsize::new(0);
16595 let ctx = workflow_context(Vec::new());
16596 let value = ctx
16597 .side_effect(|| {
16598 calls.fetch_add(1, Ordering::SeqCst);
16599 SideEffectProbe {
16600 request_id: "request-42".to_string(),
16601 attempt: 3,
16602 }
16603 })
16604 .expect("first side effect");
16605 assert_eq!(value.attempt, 3);
16606 assert_eq!(calls.load(Ordering::SeqCst), 1);
16607 let commands = ctx.take_commands().expect("commands");
16608 assert_eq!(commands.len(), 1);
16609 assert_eq!(commands[0]["type"], "record_side_effect");
16610 assert_eq!(
16611 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
16612 serde_json::to_value(&value).expect("value")
16613 );
16614
16615 let replay = workflow_context(vec![history_event(
16616 "SideEffectRecorded",
16617 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16618 )]);
16619 let replayed: SideEffectProbe = replay
16620 .side_effect(|| {
16621 calls.fetch_add(1, Ordering::SeqCst);
16622 panic!("committed side-effect callbacks must not run during replay")
16623 })
16624 .expect("replayed side effect");
16625 assert_eq!(replayed, value);
16626 assert_eq!(calls.load(Ordering::SeqCst), 1);
16627 assert!(replay.take_commands().expect("commands").is_empty());
16628 replay.ensure_history_consumed().expect("history consumed");
16629 }
16630
16631 #[test]
16632 fn side_effect_uses_avro_envelope_and_uuid_is_replay_stable() {
16633 let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
16634 let value = ctx
16635 .side_effect(|| SideEffectProbe {
16636 request_id: "avro-request".to_string(),
16637 attempt: 1,
16638 })
16639 .expect("Avro side effect");
16640 let uuid = ctx.uuid_v4().expect("deterministic UUID");
16641 let commands = ctx.take_commands().expect("commands");
16642 assert_eq!(commands.len(), 2);
16643 assert_eq!(commands[0]["result"]["codec"], DEFAULT_CODEC);
16644 assert_eq!(commands[1]["result"]["codec"], DEFAULT_CODEC);
16645 assert_eq!(
16646 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
16647 serde_json::to_value(&value).expect("value")
16648 );
16649
16650 let replay = workflow_context_with_codec(
16651 vec![
16652 history_event(
16653 "SideEffectRecorded",
16654 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16655 ),
16656 history_event(
16657 "SideEffectRecorded",
16658 json!({"sequence": 2, "result": commands[1]["result"].clone()}),
16659 ),
16660 ],
16661 DEFAULT_CODEC,
16662 );
16663 let replayed: SideEffectProbe = replay
16664 .side_effect(|| panic!("Avro callback must not run"))
16665 .expect("replayed Avro value");
16666 let replayed_uuid = replay.uuid_v4().expect("replayed UUID");
16667 assert_eq!(replayed, value);
16668 assert_eq!(replayed_uuid, uuid);
16669 assert!(replay.take_commands().expect("commands").is_empty());
16670 }
16671
16672 #[test]
16673 fn typed_side_effect_replay_preserves_bytes_and_maps() {
16674 let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
16675 let value = ctx
16676 .side_effect_avro_value(typed_fidelity_probe)
16677 .expect("typed side effect");
16678 let commands = ctx.take_commands().expect("side-effect command");
16679 assert_eq!(
16680 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
16681 .expect("recorded side effect"),
16682 value
16683 );
16684
16685 let replay = workflow_context_with_codec(
16686 vec![history_event(
16687 "SideEffectRecorded",
16688 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16689 )],
16690 DEFAULT_CODEC,
16691 );
16692 assert_eq!(
16693 replay
16694 .side_effect_avro_value(|| panic!("replay must not invoke callback"))
16695 .expect("replayed typed side effect"),
16696 value
16697 );
16698 }
16699
16700 #[test]
16701 fn ordered_side_effects_share_the_durable_command_stream() {
16702 let first = encode_value_envelope(&json!("first"), DEFAULT_CODEC).expect("first");
16703 let second = encode_value_envelope(&json!(29), DEFAULT_CODEC).expect("second");
16704 let ctx = workflow_context(vec![
16705 history_event(
16706 "SideEffectRecorded",
16707 json!({"sequence": 1, "result": first}),
16708 ),
16709 history_event(
16710 "SideEffectRecorded",
16711 json!({"sequence": 2, "result": second}),
16712 ),
16713 ]);
16714 let first: String = ctx
16715 .side_effect(|| panic!("first callback must not run"))
16716 .expect("first replay");
16717 let second: i32 = ctx
16718 .side_effect(|| panic!("second callback must not run"))
16719 .expect("second replay");
16720 assert_eq!(first, "first");
16721 assert_eq!(second, 29);
16722 ctx.ensure_history_consumed().expect("ordered history");
16723
16724 let reordered = workflow_context(vec![history_event(
16725 "VersionMarkerRecorded",
16726 json!({
16727 "sequence": 1,
16728 "change_id": "before-side-effect",
16729 "version": 1,
16730 "min_supported": 1,
16731 "max_supported": 1,
16732 }),
16733 )]);
16734 let error = reordered
16735 .side_effect(|| "new".to_string())
16736 .expect_err("command reordering must fail");
16737 assert!(matches!(
16738 error,
16739 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16740 if reason == "recorded_command_mismatch"
16741 ));
16742 }
16743
16744 #[test]
16745 fn version_markers_replay_across_upgrades_and_do_not_duplicate() {
16746 let ctx = workflow_context(Vec::new());
16747 assert_eq!(ctx.get_version("checkout-v2", 1, 2).expect("version"), 2);
16748 assert_eq!(ctx.get_version("checkout-v2", 1, 3).expect("cached"), 2);
16749 assert!(ctx.patched("new-search").expect("patch"));
16750 ctx.deprecate_patch("new-search").expect("deprecate patch");
16751 let commands = ctx.take_commands().expect("commands");
16752 assert_eq!(commands.len(), 2);
16753 assert_eq!(commands[0]["type"], "record_version_marker");
16754 assert_eq!(commands[0]["version"], 2);
16755 assert_eq!(commands[1]["change_id"], "new-search");
16756
16757 let replay = workflow_context(vec![history_event(
16758 "VersionMarkerRecorded",
16759 json!({
16760 "sequence": 1,
16761 "change_id": "checkout-v2",
16762 "version": 2,
16763 "min_supported": 1,
16764 "max_supported": 2,
16765 }),
16766 )]);
16767 assert_eq!(replay.get_version("checkout-v2", 1, 4).expect("upgrade"), 2);
16768 assert_eq!(replay.get_version("checkout-v2", 2, 5).expect("repeat"), 2);
16769 assert!(replay.take_commands().expect("commands").is_empty());
16770 replay.ensure_history_consumed().expect("history consumed");
16771 }
16772
16773 #[test]
16774 fn version_markers_reject_incompatible_or_malformed_history() {
16775 let incompatible = workflow_context(vec![history_event(
16776 "VersionMarkerRecorded",
16777 json!({
16778 "sequence": 1,
16779 "change_id": "checkout-v2",
16780 "version": 1,
16781 "min_supported": 1,
16782 "max_supported": 2,
16783 }),
16784 )]);
16785 let error = incompatible
16786 .get_version("checkout-v2", 2, 3)
16787 .expect_err("old version is unsupported");
16788 assert!(matches!(
16789 error,
16790 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16791 if reason == "version_marker_incompatible_range"
16792 ));
16793
16794 for (history, reason) in [
16795 (
16796 vec![history_event("SideEffectRecorded", json!({"sequence": 1}))],
16797 "side_effect_result_missing",
16798 ),
16799 (
16800 vec![history_event(
16801 "SideEffectRecorded",
16802 json!({
16803 "sequence": 1,
16804 "result": {"codec": "avro", "blob": "not-base64"},
16805 }),
16806 )],
16807 "side_effect_payload_incompatible",
16808 ),
16809 (
16810 vec![history_event(
16811 "SideEffectRecorded",
16812 json!({"sequence": 1, "result": {"unwrapped": true}}),
16813 )],
16814 "side_effect_payload_malformed",
16815 ),
16816 (
16817 vec![history_event(
16818 "VersionMarkerRecorded",
16819 json!({
16820 "sequence": 1,
16821 "change_id": "change",
16822 "version": 1,
16823 "min_supported": 2,
16824 "max_supported": 1,
16825 }),
16826 )],
16827 "version_marker_history_range_invalid",
16828 ),
16829 ] {
16830 let error = WorkflowState::new(
16831 history,
16832 "rust-workers".to_string(),
16833 DEFAULT_CODEC.to_string(),
16834 None,
16835 )
16836 .expect_err("malformed history must fail");
16837 assert!(matches!(
16838 error,
16839 Error::NonDeterministicReplay(ReplayFailure { reason: actual, .. })
16840 if actual == reason
16841 ));
16842 }
16843 }
16844
16845 #[test]
16846 fn typed_search_attributes_replay_value_and_type_identity_after_restart() {
16847 let history = vec![history_event(
16848 "SearchAttributesUpserted",
16849 json!({
16850 "sequence": 1,
16851 "attributes": {"customer_tier": "gold"},
16852 "attribute_types": {"customer_tier": "keyword"},
16853 "merged": {"customer_tier": "gold"}
16854 }),
16855 )];
16856
16857 let matching = workflow_context(history.clone());
16858 matching
16859 .upsert_search_attributes(
16860 SearchAttributeUpdate::new()
16861 .keyword("customer_tier", "gold")
16862 .expect("keyword update"),
16863 )
16864 .expect("matching typed update must replay");
16865 matching
16866 .ensure_history_consumed()
16867 .expect("history consumed");
16868
16869 let changed_type = workflow_context(history.clone());
16870 let error = changed_type
16871 .upsert_search_attributes(
16872 SearchAttributeUpdate::new()
16873 .string("customer_tier", "gold")
16874 .expect("string update"),
16875 )
16876 .expect_err("same JSON value with a different declaration must be nondeterministic");
16877 let Error::NonDeterministicReplay(failure) = error else {
16878 panic!("typed identity drift must be a replay failure");
16879 };
16880 assert_eq!(failure.reason, "search_attribute_type_mismatch");
16881 assert_eq!(failure.sequence, Some(1));
16882
16883 let changed_value = workflow_context(history);
16884 let error = changed_value
16885 .upsert_search_attributes(
16886 SearchAttributeUpdate::new()
16887 .keyword("customer_tier", "platinum")
16888 .expect("keyword update"),
16889 )
16890 .expect_err("changed values must be nondeterministic");
16891 let Error::NonDeterministicReplay(failure) = error else {
16892 panic!("value drift must be a replay failure");
16893 };
16894 assert_eq!(failure.reason, "search_attribute_value_mismatch");
16895 }
16896
16897 #[test]
16898 fn legacy_search_attribute_history_keeps_type_identity_unknown() {
16899 let history = vec![history_event(
16900 "SearchAttributesUpserted",
16901 json!({
16902 "sequence": 1,
16903 "attributes": {"customer_tier": "gold"},
16904 "merged": {"customer_tier": "gold"}
16905 }),
16906 )];
16907
16908 for update in [
16909 SearchAttributeUpdate::new()
16910 .keyword("customer_tier", "gold")
16911 .expect("keyword update"),
16912 SearchAttributeUpdate::new()
16913 .string("customer_tier", "gold")
16914 .expect("string update"),
16915 ] {
16916 let restarted = workflow_context(history.clone());
16917 restarted
16918 .upsert_search_attributes(update)
16919 .expect("legacy history constrains values but has unknown type identity");
16920 restarted
16921 .ensure_history_consumed()
16922 .expect("history consumed");
16923 }
16924 }
16925
16926 #[test]
16927 fn search_attribute_command_emits_canonical_types() {
16928 let ctx = workflow_context(Vec::new());
16929 ctx.upsert_search_attributes(
16930 SearchAttributeUpdate::new()
16931 .keyword("customer_tier", "gold")
16932 .expect("keyword update")
16933 .int("attempts", 3)
16934 .expect("int update")
16935 .delete("obsolete")
16936 .expect("delete update"),
16937 )
16938 .expect("valid search attributes");
16939
16940 assert_eq!(
16941 ctx.take_commands().expect("commands"),
16942 vec![json!({
16943 "type": "upsert_search_attributes",
16944 "attributes": {
16945 "attempts": 3,
16946 "customer_tier": "gold",
16947 "obsolete": null
16948 },
16949 "attribute_types": {
16950 "attempts": "int",
16951 "customer_tier": "keyword"
16952 }
16953 })]
16954 );
16955 }
16956
16957 #[test]
16958 fn duplicate_side_effects_and_version_markers_are_rejected() {
16959 let duplicate_side_effect = WorkflowState::new(
16960 vec![
16961 history_event(
16962 "SideEffectRecorded",
16963 json!({"sequence": 1, "result": fixture_envelope(json!(1))}),
16964 ),
16965 history_event(
16966 "SideEffectRecorded",
16967 json!({"sequence": 1, "result": fixture_envelope(json!(2))}),
16968 ),
16969 ],
16970 "rust-workers".to_string(),
16971 DEFAULT_CODEC.to_string(),
16972 None,
16973 )
16974 .expect_err("duplicate side effect");
16975 assert!(matches!(
16976 duplicate_side_effect,
16977 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16978 if reason == "duplicate_side_effect_record"
16979 ));
16980
16981 let marker = |sequence| {
16982 history_event(
16983 "VersionMarkerRecorded",
16984 json!({
16985 "sequence": sequence,
16986 "change_id": "same-change",
16987 "version": 1,
16988 "min_supported": 1,
16989 "max_supported": 1,
16990 }),
16991 )
16992 };
16993 let duplicate_marker = WorkflowState::new(
16994 vec![marker(1), marker(3)],
16995 "rust-workers".to_string(),
16996 DEFAULT_CODEC.to_string(),
16997 None,
16998 )
16999 .expect_err("duplicate marker");
17000 assert!(matches!(
17001 duplicate_marker,
17002 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
17003 if reason == "duplicate_version_marker"
17004 ));
17005 }
17006
17007 #[test]
17008 fn workflow_stream_authoring_derives_identity_and_replay_skips_duplicate_append() {
17009 let mut state = WorkflowState::new(
17010 Vec::new(),
17011 "rust-workers".to_string(),
17012 DEFAULT_CODEC.to_string(),
17013 None,
17014 )
17015 .expect("workflow state");
17016 state.workflow_command_identity = "command-7".to_string();
17017 let context = WorkflowContext {
17018 state: Arc::new(Mutex::new(state)),
17019 };
17020 let item =
17021 WorkflowStreamAppendItem::from_reference("s3://bucket/item.avro").item_type("receipt");
17022
17023 context
17024 .append_workflow_stream("output", &[item], Some(10))
17025 .expect("append command");
17026 context
17027 .error_workflow_stream("output", "producer failed", None)
17028 .expect("error command");
17029 let commands = context.take_commands().expect("commands");
17030
17031 assert_eq!(commands[0]["type"], "record_side_effect");
17032 assert_eq!(
17033 commands[0]["workflow_stream"]["command_identity"],
17034 "command-7"
17035 );
17036 assert_eq!(commands[0]["workflow_stream"]["command_ordinal"], 0);
17037 assert_eq!(
17038 commands[0]["workflow_stream"]["items"][0]["idempotency_key"],
17039 "dw-stream:command-7:0:0"
17040 );
17041 assert_eq!(commands[1]["workflow_stream"]["operation"], "error");
17042
17043 let recorded = history_event(
17044 "SideEffectRecorded",
17045 json!({"sequence": 1, "result": fixture_envelope(Value::Null)}),
17046 );
17047 let mut replay_state = WorkflowState::new(
17048 vec![recorded],
17049 "rust-workers".to_string(),
17050 DEFAULT_CODEC.to_string(),
17051 None,
17052 )
17053 .expect("replay state");
17054 replay_state.workflow_command_identity = "command-7".to_string();
17055 let replay_context = WorkflowContext {
17056 state: Arc::new(Mutex::new(replay_state)),
17057 };
17058 replay_context
17059 .append_workflow_stream(
17060 "output",
17061 &[WorkflowStreamAppendItem::from_reference(
17062 "s3://bucket/item.avro",
17063 )],
17064 Some(10),
17065 )
17066 .expect("replayed append");
17067 assert!(replay_context
17068 .take_commands()
17069 .expect("replayed commands")
17070 .is_empty());
17071 }
17072
17073 #[test]
17074 fn workflow_stream_authoring_requires_server_durable_command_identity() {
17075 let context = workflow_context(Vec::new());
17076 let error = context
17077 .append_workflow_stream(
17078 "output",
17079 &[WorkflowStreamAppendItem::from_reference(
17080 "s3://bucket/item.avro",
17081 )],
17082 None,
17083 )
17084 .expect_err("stream append without durable command identity must fail closed");
17085
17086 assert!(matches!(error, Error::MissingWorkflowCommandIdentity));
17087 assert!(context.take_commands().expect("commands").is_empty());
17088 }
17089
17090 #[test]
17091 fn cold_worker_replay_does_not_repeat_committed_side_effects_or_markers() {
17092 fn worker(calls: Arc<AtomicUsize>) -> Worker {
17093 let client = Client::new("http://127.0.0.1:8080").expect("client");
17094 let mut worker = Worker::new(client, "rust-workers");
17095 worker.register_workflow("rust.side-effect-version", move |ctx, _input| {
17096 let calls = Arc::clone(&calls);
17097 async move {
17098 let captured = ctx.side_effect(|| {
17099 calls.fetch_add(1, Ordering::SeqCst);
17100 "captured-once".to_string()
17101 })?;
17102 let version = ctx.get_version("cold-restart", 1, 2)?;
17103 Ok(json!({"captured": captured, "version": version}))
17104 }
17105 });
17106 worker
17107 }
17108
17109 fn task(history_events: Vec<HistoryEvent>) -> WorkflowTask {
17110 WorkflowTask {
17111 task_id: "wft-side-effect-version".to_string(),
17112 workflow_command_id: None,
17113 workflow_id: Some("wf-side-effect-version".to_string()),
17114 run_id: Some("run-side-effect-version".to_string()),
17115 workflow_type: "rust.side-effect-version".to_string(),
17116 cancel_requested: false,
17117 payload_codec: DEFAULT_CODEC.to_string(),
17118 arguments: Some(
17119 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("arguments"),
17120 ),
17121 history_events,
17122 total_history_events: None,
17123 history_size_bytes: None,
17124 continue_as_new_recommended: None,
17125 history_budget_pressure: None,
17126 next_history_page_token: None,
17127 workflow_task_attempt: 1,
17128 workflow_signal_id: None,
17129 signal_name: None,
17130 signal_arguments: None,
17131 workflow_update_id: None,
17132 update_name: None,
17133 lease_owner: Some("rust-worker".to_string()),
17134 }
17135 }
17136
17137 let calls = Arc::new(AtomicUsize::new(0));
17138 let initial = worker(Arc::clone(&calls))
17139 .execute_workflow_task(task(Vec::new()))
17140 .expect("initial execution");
17141 assert_eq!(
17142 initial
17143 .iter()
17144 .map(|command| &command["type"])
17145 .collect::<Vec<_>>(),
17146 vec![
17147 "record_side_effect",
17148 "record_version_marker",
17149 "complete_workflow"
17150 ]
17151 );
17152 assert_eq!(calls.load(Ordering::SeqCst), 1);
17153
17154 let restarted = worker(Arc::clone(&calls));
17155 let replayed = restarted
17156 .execute_workflow_task(task(vec![
17157 history_event(
17158 "SideEffectRecorded",
17159 json!({"sequence": 1, "result": initial[0]["result"].clone()}),
17160 ),
17161 history_event(
17162 "VersionMarkerRecorded",
17163 json!({
17164 "sequence": 2,
17165 "change_id": "cold-restart",
17166 "version": 2,
17167 "min_supported": 1,
17168 "max_supported": 2,
17169 }),
17170 ),
17171 ]))
17172 .expect("cold replay");
17173 assert_eq!(replayed.len(), 1);
17174 assert_eq!(replayed[0]["type"], "complete_workflow");
17175 assert_eq!(calls.load(Ordering::SeqCst), 1);
17176 }
17177
17178 #[test]
17179 fn side_effect_replay_rejects_changed_rust_value_type() {
17180 let result = encode_value_envelope(&json!({"value": 42}), DEFAULT_CODEC).expect("result");
17181 let ctx = workflow_context(vec![history_event(
17182 "SideEffectRecorded",
17183 json!({"sequence": 1, "result": result}),
17184 )]);
17185 let error = ctx
17186 .side_effect::<Vec<String>, _>(|| panic!("callback must not run"))
17187 .expect_err("changed type must fail replay");
17188 assert!(matches!(
17189 error,
17190 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
17191 if reason == "side_effect_type_mismatch"
17192 ));
17193 }
17194
17195 fn completed_retry_activity_history() -> Vec<HistoryEvent> {
17196 vec![
17197 history_event(
17198 "ActivityScheduled",
17199 json!({
17200 "sequence": 1,
17201 "activity_type": "flaky",
17202 "activity_execution_id": "act-1",
17203 "activity": {
17204 "id": "act-1",
17205 "sequence": 1,
17206 "type": "flaky",
17207 "queue": "critical-activities",
17208 "execution_mode": null,
17209 "retry_policy": {
17210 "snapshot_version": 1,
17211 "max_attempts": 3,
17212 "backoff_seconds": [2, 4],
17213 "start_to_close_timeout": 30,
17214 "schedule_to_start_timeout": 5,
17215 "schedule_to_close_timeout": 90,
17216 "heartbeat_timeout": 10,
17217 "non_retryable_error_types": ["PermanentError"]
17218 }
17219 }
17220 }),
17221 ),
17222 history_event(
17223 "ActivityStarted",
17224 json!({
17225 "sequence": 1,
17226 "activity_type": "flaky",
17227 "activity_execution_id": "act-1",
17228 "activity_attempt_id": "attempt-1",
17229 "attempt_number": 1
17230 }),
17231 ),
17232 history_event(
17233 "ActivityRetryScheduled",
17234 json!({
17235 "sequence": 1,
17236 "activity_type": "flaky",
17237 "activity_execution_id": "act-1",
17238 "activity_attempt_id": "attempt-1",
17239 "attempt_number": 1,
17240 "retry_after_attempt": 1,
17241 "retry_backoff_seconds": 2,
17242 "failure_category": "activity",
17243 "exception_type": "TransientError"
17244 }),
17245 ),
17246 history_event(
17247 "ActivityStarted",
17248 json!({
17249 "sequence": 1,
17250 "activity_type": "flaky",
17251 "activity_execution_id": "act-1",
17252 "activity_attempt_id": "attempt-2",
17253 "attempt_number": 2
17254 }),
17255 ),
17256 history_event(
17257 "ActivityCompleted",
17258 json!({
17259 "sequence": 1,
17260 "activity_type": "flaky",
17261 "activity_execution_id": "act-1",
17262 "activity_attempt_id": "attempt-2",
17263 "attempt_number": 2,
17264 "payload_codec": DEFAULT_CODEC,
17265 "result": fixture_envelope(json!({"status":"recovered"}))
17266 }),
17267 ),
17268 ]
17269 }
17270
17271 fn retry_activity_options() -> ActivityOptions {
17272 ActivityOptions::new()
17273 .task_queue("critical-activities")
17274 .retry_policy(
17275 ActivityRetryPolicy::new(3)
17276 .backoff_intervals([Duration::from_secs(2), Duration::from_secs(4)])
17277 .non_retryable_error_type("PermanentError"),
17278 )
17279 .start_to_close_timeout(Duration::from_secs(30))
17280 .schedule_to_start_timeout(Duration::from_secs(5))
17281 .schedule_to_close_timeout(Duration::from_secs(90))
17282 .heartbeat_timeout(Duration::from_secs(10))
17283 }
17284
17285 #[test]
17286 fn fixed_avro_value_round_trips_json_values() {
17287 let value = json!({"greeting": "hello", "count": 3, "ok": true});
17288 let envelope = PayloadEnvelope::avro(&value).expect("encode");
17289 assert_eq!(envelope.codec, DEFAULT_CODEC);
17290 assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
17291 }
17292
17293 #[tokio::test]
17294 async fn typed_handler_adapters_round_trip_serde_contracts_on_the_fixed_wire() {
17295 let client = Client::new("http://127.0.0.1:8080").expect("client");
17296 let mut worker = Worker::new(client, "rust-workers");
17297 worker.register_typed_workflow(
17298 "typed.contract.workflow",
17299 |_ctx, input: TypedContract| async move { Ok(input) },
17300 );
17301 worker.register_typed_activity(
17302 "typed.contract.activity",
17303 |_ctx, input: TypedContract| async move { Ok(input) },
17304 );
17305
17306 let expected = typed_contract();
17307 let arguments = AvroValue::Array(vec![
17308 AvroValue::from_serialize(&expected).expect("typed request")
17309 ]);
17310 let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("arguments");
17311 let mut workflow = workflow_task("typed.contract.workflow", Vec::new(), DEFAULT_CODEC);
17312 workflow.arguments = Some(envelope.clone());
17313 let commands = worker
17314 .execute_workflow_task(workflow)
17315 .expect("typed workflow task");
17316 let workflow_result: TypedContract =
17317 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17318 .expect("workflow result envelope")
17319 .deserialize()
17320 .expect("workflow result type");
17321 assert_eq!(workflow_result, expected);
17322
17323 let activity = ActivityTask {
17324 task_id: "typed-contract-activity".to_string(),
17325 activity_attempt_id: Some("typed-contract-attempt".to_string()),
17326 attempt_id: None,
17327 activity_type: "typed.contract.activity".to_string(),
17328 payload_codec: DEFAULT_CODEC.to_string(),
17329 arguments: Some(envelope),
17330 attempt_number: 1,
17331 lease_owner: Some("rust-worker".to_string()),
17332 };
17333 let activity_result: TypedContract = worker
17334 .execute_activity_task(activity)
17335 .await
17336 .expect("typed activity task")
17337 .deserialize()
17338 .expect("activity result type");
17339 assert_eq!(activity_result, expected);
17340 }
17341
17342 #[tokio::test]
17343 async fn typed_handler_errors_include_handler_name_direction_and_rust_type() {
17344 let client = Client::new("http://127.0.0.1:8080").expect("client");
17345 let mut worker = Worker::new(client, "rust-workers");
17346 worker.register_typed_workflow(
17347 "typed.shape.workflow",
17348 |_ctx, input: TypedContract| async move { Ok(input) },
17349 );
17350 worker.register_typed_activity("typed.unsupported.activity", |_ctx, (): ()| async move {
17351 Ok(f64::NAN)
17352 });
17353
17354 let mut workflow = workflow_task("typed.shape.workflow", Vec::new(), DEFAULT_CODEC);
17355 workflow.arguments = Some(
17356 encode_typed_envelope(
17357 &AvroValue::Array(vec![
17358 AvroValue::String("first".to_string()),
17359 AvroValue::String("second".to_string()),
17360 ]),
17361 DEFAULT_CODEC,
17362 )
17363 .expect("malformed typed arguments"),
17364 );
17365 let commands = worker
17366 .execute_workflow_task(workflow)
17367 .expect("shape mismatch becomes a workflow failure");
17368 let message = commands[0]["message"].as_str().expect("failure message");
17369 assert!(message.contains("workflow handler \"typed.shape.workflow\" input type"));
17370 assert!(message.contains(type_name::<TypedContract>()));
17371 assert!(message.contains("task carried 2 arguments"));
17372
17373 let activity = ActivityTask {
17374 task_id: "typed-unsupported-activity".to_string(),
17375 activity_attempt_id: Some("typed-unsupported-attempt".to_string()),
17376 attempt_id: None,
17377 activity_type: "typed.unsupported.activity".to_string(),
17378 payload_codec: DEFAULT_CODEC.to_string(),
17379 arguments: Some(
17380 encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
17381 .expect("unit arguments"),
17382 ),
17383 attempt_number: 1,
17384 lease_owner: Some("rust-worker".to_string()),
17385 };
17386 let Error::HandlerType {
17387 handler_kind,
17388 handler_name,
17389 value_kind,
17390 rust_type,
17391 message,
17392 } = worker
17393 .execute_activity_task(activity)
17394 .await
17395 .expect_err("non-finite handler output must fail")
17396 else {
17397 panic!("expected contextual handler type failure");
17398 };
17399 assert_eq!(handler_kind, HandlerKind::Activity);
17400 assert_eq!(handler_name, "typed.unsupported.activity");
17401 assert_eq!(value_kind, HandlerValueKind::Result);
17402 assert_eq!(rust_type, type_name::<f64>());
17403 assert!(message.contains("non_finite_float"));
17404 }
17405
17406 #[tokio::test]
17407 async fn typed_replayed_workflow_decodes_input_and_activity_result_losslessly() {
17408 #[derive(Clone, Default)]
17409 struct State {
17410 observed: Option<TypedContract>,
17411 }
17412
17413 let client = Client::new("http://127.0.0.1:8080").expect("client");
17414 let mut worker = Worker::new(client, "rust-workers");
17415 worker.register_typed_replayed_workflow(
17416 "typed.contract.replayed",
17417 State::default,
17418 |ctx, input: TypedContract, state| async move {
17419 let result: TypedContract =
17420 ctx.activity_typed("typed.contract.activity", input).await?;
17421 state.update(|current| current.observed = Some(result.clone()))?;
17422 Ok(result)
17423 },
17424 );
17425 worker.register_replayed_query::<State, _, _>(
17426 "typed.contract.replayed",
17427 "observed",
17428 |_ctx, state, _args| async move {
17429 Ok(json!(state.observed.as_ref().map(|value| value.signed)))
17430 },
17431 );
17432
17433 let expected = typed_contract();
17434 let typed_value = AvroValue::from_serialize(&expected).expect("typed value");
17435 let workflow_arguments =
17436 encode_typed_envelope(&AvroValue::Array(vec![typed_value.clone()]), DEFAULT_CODEC)
17437 .expect("workflow arguments");
17438 let result = encode_typed_envelope(&typed_value, DEFAULT_CODEC).expect("activity result");
17439 let task = QueryTask {
17440 query_task_id: "typed-replay-query".to_string(),
17441 query_task_attempt: 1,
17442 lease_owner: Some("rust-worker".to_string()),
17443 workflow_id: Some("typed-replay".to_string()),
17444 run_id: Some("typed-replay-run".to_string()),
17445 workflow_type: "typed.contract.replayed".to_string(),
17446 query_name: "observed".to_string(),
17447 payload_codec: DEFAULT_CODEC.to_string(),
17448 workflow_arguments: Some(workflow_arguments),
17449 query_arguments: Some(
17450 encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
17451 .expect("query arguments"),
17452 ),
17453 history_events: vec![
17454 history_event(
17455 "ActivityScheduled",
17456 json!({
17457 "sequence": 1,
17458 "activity_type": "typed.contract.activity"
17459 }),
17460 ),
17461 history_event(
17462 "ActivityCompleted",
17463 json!({
17464 "sequence": 1,
17465 "activity_type": "typed.contract.activity",
17466 "payload_codec": DEFAULT_CODEC,
17467 "result": result
17468 }),
17469 ),
17470 ],
17471 history_export: None,
17472 run_status: Some("completed".to_string()),
17473 };
17474
17475 assert_eq!(
17476 worker
17477 .execute_query_task(task)
17478 .await
17479 .expect("typed replay query")
17480 .deserialize::<i64>()
17481 .expect("query result"),
17482 expected.signed
17483 );
17484 }
17485
17486 #[tokio::test]
17487 async fn typed_worker_surfaces_preserve_bytes_and_map_list_identity() {
17488 let client = Client::new("http://127.0.0.1:8080").expect("client");
17489 let mut worker = Worker::new(client, "rust-workers");
17490 worker.register_workflow_avro_value("typed.echo", |_ctx, input| async move { Ok(input) });
17491 worker
17492 .register_activity_avro_value("typed.activity", |_ctx, input| async move { Ok(input) });
17493 worker.register_query_avro_value("typed.echo", "inspect", |_ctx, input| async move {
17494 Ok(input)
17495 });
17496 worker.register_update_avro_value("typed.echo", "replace", |_ctx, input| async move {
17497 Ok(input)
17498 });
17499 worker.register_workflow_avro_value("typed.signal", |ctx, _input| async move {
17500 Ok(AvroValue::Array(
17501 ctx.wait_signal_avro_value("changed").await?,
17502 ))
17503 });
17504
17505 let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
17506 let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed envelope");
17507
17508 let mut workflow = workflow_task("typed.echo", Vec::new(), DEFAULT_CODEC);
17509 workflow.arguments = Some(envelope.clone());
17510 let commands = worker
17511 .execute_workflow_task(workflow)
17512 .expect("typed workflow task");
17513 assert_eq!(commands[0]["type"], "complete_workflow");
17514 assert_eq!(
17515 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17516 .expect("typed workflow result"),
17517 arguments
17518 );
17519
17520 let activity = ActivityTask {
17521 task_id: "activity-typed".to_string(),
17522 activity_attempt_id: Some("attempt-typed".to_string()),
17523 attempt_id: None,
17524 activity_type: "typed.activity".to_string(),
17525 payload_codec: DEFAULT_CODEC.to_string(),
17526 arguments: Some(envelope.clone()),
17527 attempt_number: 1,
17528 lease_owner: Some("rust-worker".to_string()),
17529 };
17530 assert_eq!(
17531 worker
17532 .execute_activity_task(activity)
17533 .await
17534 .expect("typed activity result"),
17535 arguments
17536 );
17537
17538 let query = QueryTask {
17539 query_task_id: "query-typed".to_string(),
17540 query_task_attempt: 1,
17541 lease_owner: Some("rust-worker".to_string()),
17542 workflow_id: Some("typed-1".to_string()),
17543 run_id: Some("run-typed".to_string()),
17544 workflow_type: "typed.echo".to_string(),
17545 query_name: "inspect".to_string(),
17546 payload_codec: DEFAULT_CODEC.to_string(),
17547 workflow_arguments: Some(
17548 encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
17549 .expect("workflow input"),
17550 ),
17551 query_arguments: Some(envelope.clone()),
17552 history_events: Vec::new(),
17553 history_export: None,
17554 run_status: Some("running".to_string()),
17555 };
17556 assert_eq!(
17557 worker
17558 .execute_query_task(query)
17559 .await
17560 .expect("typed query result"),
17561 arguments
17562 );
17563
17564 let mut update = workflow_task(
17565 "typed.echo",
17566 vec![history_event(
17567 "UpdateAccepted",
17568 json!({
17569 "update_id": "update-typed",
17570 "update_name": "replace",
17571 "arguments": envelope.clone(),
17572 }),
17573 )],
17574 DEFAULT_CODEC,
17575 );
17576 update.workflow_update_id = Some("update-typed".to_string());
17577 update.update_name = Some("replace".to_string());
17578 let commands = worker
17579 .execute_workflow_task(update)
17580 .expect("typed update task");
17581 assert_eq!(commands[0]["type"], "complete_update");
17582 assert_eq!(
17583 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17584 .expect("typed update result"),
17585 arguments
17586 );
17587
17588 let mut signal = workflow_task(
17589 "typed.signal",
17590 vec![history_event(
17591 "SignalReceived",
17592 json!({
17593 "signal_id": "signal-typed",
17594 "signal_name": "changed",
17595 "arguments": envelope.clone(),
17596 }),
17597 )],
17598 DEFAULT_CODEC,
17599 );
17600 signal.workflow_signal_id = Some("signal-typed".to_string());
17601 signal.signal_name = Some("changed".to_string());
17602 signal.signal_arguments = Some(envelope);
17603 let commands = worker
17604 .execute_workflow_task(signal)
17605 .expect("typed signal resume");
17606 assert_eq!(
17607 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17608 .expect("typed signal result"),
17609 arguments
17610 );
17611 }
17612
17613 #[tokio::test]
17614 async fn typed_helpers_never_parse_json_inspection_projection() {
17615 let collision_values = projection_collision_probe();
17616 let expected = AvroValue::Array(collision_values.clone());
17617 let envelope = encode_typed_envelope(&expected, DEFAULT_CODEC).expect("collision envelope");
17618
17619 let activity_context = workflow_context_with_codec(
17620 vec![history_event(
17621 "ActivityCompleted",
17622 json!({
17623 "sequence": 1,
17624 "activity_type": "collision.activity",
17625 "payload_codec": DEFAULT_CODEC,
17626 "result": envelope.clone(),
17627 }),
17628 )],
17629 DEFAULT_CODEC,
17630 );
17631 assert_eq!(
17632 activity_context
17633 .activity_avro_value("collision.activity", AvroValue::Array(Vec::new()))
17634 .await
17635 .expect("typed activity collision result"),
17636 expected
17637 );
17638
17639 let signal_context = workflow_context_with_codec(
17640 vec![
17641 history_event(
17642 "SignalWaitOpened",
17643 json!({"sequence": 1, "signal_name": "collision"}),
17644 ),
17645 history_event(
17646 "SignalApplied",
17647 json!({
17648 "sequence": 1,
17649 "signal_name": "collision",
17650 "payload_codec": DEFAULT_CODEC,
17651 "value": envelope.clone(),
17652 }),
17653 ),
17654 ],
17655 DEFAULT_CODEC,
17656 );
17657 assert_eq!(
17658 signal_context
17659 .wait_signal_avro_value("collision")
17660 .await
17661 .expect("typed signal collision arguments"),
17662 collision_values
17663 );
17664
17665 let child_context = workflow_context_with_codec(
17666 vec![
17667 history_event(
17668 "ChildWorkflowScheduled",
17669 json!({
17670 "sequence": 1,
17671 "child_workflow_instance_id": "collision-child",
17672 "child_workflow_run_id": "collision-run",
17673 "child_workflow_type": "collision.child",
17674 }),
17675 ),
17676 history_event(
17677 "ChildRunCompleted",
17678 json!({
17679 "sequence": 1,
17680 "child_workflow_instance_id": "collision-child",
17681 "child_workflow_run_id": "collision-run",
17682 "child_workflow_type": "collision.child",
17683 "payload_codec": DEFAULT_CODEC,
17684 "result": envelope,
17685 }),
17686 ),
17687 ],
17688 DEFAULT_CODEC,
17689 );
17690 let child = child_context
17691 .start_child_workflow_avro_value(
17692 "collision.child",
17693 ChildWorkflowOptions::new("collision-workers"),
17694 AvroValue::Array(Vec::new()),
17695 )
17696 .await
17697 .expect("typed child collision result");
17698 assert_eq!(child.result, expected);
17699 }
17700
17701 #[tokio::test]
17702 async fn replayed_typed_query_keeps_lossless_workflow_and_query_inputs() {
17703 let client = Client::new("http://127.0.0.1:8080").expect("client");
17704 let mut worker = Worker::new(client, "rust-workers");
17705 worker.register_replayed_workflow_avro_value(
17706 "typed.replayed",
17707 || (),
17708 |_ctx, input, _state| async move { Ok(input) },
17709 );
17710 worker.register_replayed_query_avro_value::<(), _, _>(
17711 "typed.replayed",
17712 "inspect",
17713 |ctx, _state, args| async move {
17714 let mut signals = ctx.signals_avro_value("collision");
17715 let signal = signals
17716 .pop()
17717 .map(AvroValue::Array)
17718 .unwrap_or_else(|| AvroValue::Array(Vec::new()));
17719 Ok(AvroValue::Array(vec![
17720 ctx.workflow_input_avro_value().clone(),
17721 signal,
17722 args,
17723 ]))
17724 },
17725 );
17726 let arguments = AvroValue::Array(projection_collision_probe());
17727 let signal_arguments =
17728 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed query signal arguments");
17729 let task = QueryTask {
17730 query_task_id: "query-typed-replay".to_string(),
17731 query_task_attempt: 1,
17732 lease_owner: Some("rust-worker".to_string()),
17733 workflow_id: Some("typed-replay".to_string()),
17734 run_id: Some("run-typed-replay".to_string()),
17735 workflow_type: "typed.replayed".to_string(),
17736 query_name: "inspect".to_string(),
17737 payload_codec: DEFAULT_CODEC.to_string(),
17738 workflow_arguments: Some(
17739 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("workflow arguments"),
17740 ),
17741 query_arguments: Some(
17742 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("query arguments"),
17743 ),
17744 history_events: vec![history_event(
17745 "SignalReceived",
17746 json!({
17747 "signal_id": "collision-signal",
17748 "signal_name": "collision",
17749 "workflow_sequence": 1,
17750 "payload_codec": DEFAULT_CODEC,
17751 "arguments": signal_arguments,
17752 }),
17753 )],
17754 history_export: None,
17755 run_status: Some("completed".to_string()),
17756 };
17757
17758 assert_eq!(
17759 worker
17760 .execute_query_task(task)
17761 .await
17762 .expect("typed replay query"),
17763 AvroValue::Array(vec![arguments.clone(), arguments.clone(), arguments])
17764 );
17765 }
17766
17767 #[test]
17768 fn public_avro_adapter_rejects_non_string_map_keys_before_json_conversion() {
17769 let value = BTreeMap::from([(1_i32, "integer key")]);
17770 let error = PayloadEnvelope::avro(&value)
17771 .expect_err("integer map keys must fail")
17772 .to_string();
17773
17774 assert!(error.contains("invalid_map_key"));
17775 }
17776
17777 #[test]
17778 fn json_tagged_payload_fails_closed_with_actionable_diagnostic() {
17779 let envelope = PayloadEnvelope {
17780 codec: "json".to_string(),
17781 blob: r#"{"greeting":"hello"}"#.to_string(),
17782 };
17783
17784 let error = decode_payload::<Value>(&envelope).expect_err("JSON payload must fail");
17785 let diagnostic = error.to_string();
17786 assert!(diagnostic.contains("unsupported_payload_codec"));
17787 assert!(diagnostic.contains("codec=\"avro\""));
17788 assert!(diagnostic.contains("HTTP document transport"));
17789 }
17790
17791 #[test]
17792 fn untagged_json_payload_value_fails_closed() {
17793 let error = decode_wire_value(&json!({"stale": true}), DEFAULT_CODEC)
17794 .expect_err("untagged JSON payload values must fail");
17795 let diagnostic = error.to_string();
17796 assert!(diagnostic.contains("unsupported_payload_codec"));
17797 assert!(diagnostic.contains("untagged durable payload"));
17798 assert!(diagnostic.contains("HTTP document transport"));
17799 }
17800
17801 #[test]
17802 fn prerelease_avro_payload_without_single_object_frame_is_rejected() {
17803 let envelope = PayloadEnvelope {
17804 codec: DEFAULT_CODEC.to_string(),
17805 blob: BASE64.encode([0x01]),
17806 };
17807
17808 let error = decode_payload::<Value>(&envelope).expect_err("prerelease payload must fail");
17809 assert!(error.to_string().contains("invalid_payload_framing"));
17810 }
17811
17812 #[tokio::test]
17813 async fn workflow_completion_rejects_invalid_payload_slots_without_transport() {
17814 let server = MockWorkerServer::start();
17815 let client = Client::builder(server.base_url())
17816 .timeout(Duration::from_secs(2))
17817 .build()
17818 .expect("client");
17819 let invalid_commands = [
17820 json!({
17821 "type": "complete_workflow",
17822 "result": {"codec": "json", "blob": null}
17823 }),
17824 json!({
17825 "type": "schedule_activity",
17826 "arguments": {"codec": "yaml", "blob": "ignored"}
17827 }),
17828 json!({
17829 "type": "start_child_workflow",
17830 "arguments": {"codec": DEFAULT_CODEC, "blob": null}
17831 }),
17832 json!({"type": "continue_as_new", "arguments": []}),
17833 json!({"type": "complete_update"}),
17834 json!({"type": "record_side_effect", "result": null}),
17835 json!({
17836 "type": "start_service_operation",
17837 "payload_codec": DEFAULT_CODEC,
17838 "request_payload": "raw-avro-bytes"
17839 }),
17840 ];
17841
17842 for command in invalid_commands {
17843 let error = client
17844 .complete_workflow_task("invalid-codec", "rust-worker", 1, vec![command])
17845 .await
17846 .expect_err("invalid durable payload must fail locally");
17847 let diagnostic = error.to_string();
17848 assert!(
17849 diagnostic.contains("unsupported_payload_codec")
17850 || diagnostic.contains("invalid_payload_envelope")
17851 || diagnostic.contains("untagged durable payload"),
17852 "unexpected validation diagnostic: {diagnostic}"
17853 );
17854 }
17855
17856 assert_eq!(
17857 server.request_count("/api/worker/workflow-tasks/invalid-codec/complete"),
17858 0,
17859 "invalid command payloads must not reach HTTP transport"
17860 );
17861 }
17862
17863 #[test]
17864 fn workflow_completion_validates_only_protocol_owned_payload_slots() {
17865 let envelope = fixture_envelope(json!({"codec": "customer-value"}));
17866 let commands = [
17867 json!({"type": "complete_workflow", "result": envelope.clone()}),
17868 json!({"type": "schedule_activity", "arguments": envelope.clone()}),
17869 json!({"type": "start_child_workflow", "arguments": envelope.clone()}),
17870 json!({"type": "continue_as_new", "arguments": envelope.clone()}),
17871 json!({"type": "complete_update", "result": envelope.clone()}),
17872 json!({"type": "record_side_effect", "result": envelope.clone()}),
17873 json!({
17874 "type": "start_service_operation",
17875 "payload_codec": DEFAULT_CODEC,
17876 "request_payload": envelope.clone()
17877 }),
17878 json!({
17879 "type": "complete_workflow",
17880 "result": envelope,
17881 "metadata": {
17882 "codec": "json",
17883 "payload_codec": "customer-codec",
17884 "result": {"codec": "yaml", "blob": null}
17885 }
17886 }),
17887 ];
17888
17889 validate_workflow_task_commands(&commands)
17890 .expect("customer metadata must not become a protocol codec declaration");
17891 }
17892
17893 #[test]
17894 fn valid_avro_tasks_normalize_absent_and_null_arguments_to_empty_lists() {
17895 assert_eq!(
17896 decode_task_avro_arguments(None, DEFAULT_CODEC).expect("absent arguments"),
17897 AvroValue::Array(Vec::new())
17898 );
17899 assert_eq!(
17900 decode_task_avro_arguments(Some(&Value::Null), DEFAULT_CODEC).expect("null arguments"),
17901 AvroValue::Array(Vec::new())
17902 );
17903
17904 let mut signal = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
17905 signal.signal_name = Some("empty-signal".to_string());
17906 signal.signal_arguments = None;
17907 let decoded = decode_resume_signal(&signal)
17908 .expect("valid Avro signal")
17909 .expect("named signal resumes the workflow");
17910 assert!(decoded.arguments.is_empty());
17911 }
17912
17913 #[tokio::test]
17914 async fn malformed_task_level_codecs_become_pre_handler_failures() {
17915 let client = Client::new("http://127.0.0.1:8080").expect("client");
17916 let mut worker = Worker::new(client, "rust-workers");
17917 let handler_calls = Arc::new(AtomicUsize::new(0));
17918
17919 let calls = Arc::clone(&handler_calls);
17920 worker.register_workflow("codec.workflow", move |_ctx, _args| {
17921 calls.fetch_add(1, Ordering::SeqCst);
17922 async move { Ok(Value::Null) }
17923 });
17924 let calls = Arc::clone(&handler_calls);
17925 worker.register_activity("codec.activity", move |_ctx, _args| {
17926 calls.fetch_add(1, Ordering::SeqCst);
17927 async move { Ok(Value::Null) }
17928 });
17929 let calls = Arc::clone(&handler_calls);
17930 worker.register_query("codec.workflow", "known", move |_ctx, _args| {
17931 calls.fetch_add(1, Ordering::SeqCst);
17932 async move { Ok(Value::Null) }
17933 });
17934
17935 let mut failures = Vec::new();
17936 for codec_case in [
17937 InvalidTaskPayloadCodec::Missing,
17938 InvalidTaskPayloadCodec::Null,
17939 InvalidTaskPayloadCodec::NonString,
17940 ] {
17941 let mut workflow = json!({
17942 "task_id": format!("workflow-{}", codec_case.label()),
17943 "workflow_type": "codec.workflow"
17944 });
17945 codec_case.apply(&mut workflow);
17946 match serde_json::from_value::<WorkflowTask>(workflow) {
17947 Ok(task) => match worker.execute_workflow_task(task) {
17948 Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
17949 outcome => failures.push(format!(
17950 "workflow {} codec returned {outcome:?}",
17951 codec_case.label()
17952 )),
17953 },
17954 Err(error) => failures.push(format!(
17955 "workflow {} codec failed transport deserialization: {error}",
17956 codec_case.label()
17957 )),
17958 }
17959
17960 let mut activity = json!({
17961 "task_id": format!("activity-{}", codec_case.label()),
17962 "activity_attempt_id": format!("attempt-{}", codec_case.label()),
17963 "activity_type": "codec.activity",
17964 "attempt_number": 1
17965 });
17966 codec_case.apply(&mut activity);
17967 match serde_json::from_value::<ActivityTask>(activity) {
17968 Ok(task) => match worker.execute_activity_task(task).await {
17969 Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
17970 outcome => failures.push(format!(
17971 "activity {} codec returned {outcome:?}",
17972 codec_case.label()
17973 )),
17974 },
17975 Err(error) => failures.push(format!(
17976 "activity {} codec failed transport deserialization: {error}",
17977 codec_case.label()
17978 )),
17979 }
17980
17981 let mut query = json!({
17982 "query_task_id": format!("query-{}", codec_case.label()),
17983 "workflow_type": "codec.workflow",
17984 "query_name": "known"
17985 });
17986 codec_case.apply(&mut query);
17987 match serde_json::from_value::<QueryTask>(query) {
17988 Ok(task) => match worker.execute_query_task(task).await {
17989 Err(failure) if failure.message.contains("unsupported_payload_codec") => {}
17990 outcome => failures.push(format!(
17991 "query {} codec returned {outcome:?}",
17992 codec_case.label()
17993 )),
17994 },
17995 Err(error) => failures.push(format!(
17996 "query {} codec failed transport deserialization: {error}",
17997 codec_case.label()
17998 )),
17999 }
18000 }
18001
18002 assert!(failures.is_empty(), "{}", failures.join("\n"));
18003 assert_eq!(
18004 handler_calls.load(Ordering::SeqCst),
18005 0,
18006 "invalid task codecs must not invoke a handler"
18007 );
18008 }
18009
18010 #[tokio::test]
18011 async fn polled_malformed_task_codecs_are_settled_without_handler_execution() {
18012 for codec_case in [
18013 InvalidTaskPayloadCodec::Missing,
18014 InvalidTaskPayloadCodec::Null,
18015 InvalidTaskPayloadCodec::NonString,
18016 ] {
18017 let server = MockWorkerServer::invalid_task_payload_codec(codec_case);
18018 let client = Client::builder(server.base_url())
18019 .timeout(Duration::from_secs(2))
18020 .build()
18021 .expect("client");
18022 let mut worker = Worker::new(client, "rust-workers")
18023 .worker_id("codec-worker")
18024 .poll_timeout(Duration::from_millis(10));
18025 let handler_calls = Arc::new(AtomicUsize::new(0));
18026
18027 let calls = Arc::clone(&handler_calls);
18028 worker.register_workflow("codec.workflow", move |_ctx, _args| {
18029 calls.fetch_add(1, Ordering::SeqCst);
18030 async move { Ok(Value::Null) }
18031 });
18032 let calls = Arc::clone(&handler_calls);
18033 worker.register_activity("codec.activity", move |_ctx, _args| {
18034 calls.fetch_add(1, Ordering::SeqCst);
18035 async move { Ok(Value::Null) }
18036 });
18037 let calls = Arc::clone(&handler_calls);
18038 worker.register_query("codec.workflow", "known", move |_ctx, _args| {
18039 calls.fetch_add(1, Ordering::SeqCst);
18040 async move { Ok(Value::Null) }
18041 });
18042
18043 assert_eq!(
18044 worker.run_once().await.expect("invalid tasks are settled"),
18045 3,
18046 "all {} codec tasks must be handled",
18047 codec_case.label()
18048 );
18049 assert_eq!(
18050 handler_calls.load(Ordering::SeqCst),
18051 0,
18052 "{} task codecs must fail before every handler",
18053 codec_case.label()
18054 );
18055
18056 for path in [
18057 "/api/worker/workflow-tasks/codec-workflow/fail",
18058 "/api/worker/activity-tasks/codec-activity/fail",
18059 "/api/worker/query-tasks/codec-query/fail",
18060 ] {
18061 let body = server.request_body(path);
18062 assert!(
18063 body["failure"]["message"]
18064 .as_str()
18065 .is_some_and(|message| message.contains("unsupported_payload_codec")),
18066 "{path} must receive the stable codec diagnostic for the {} case: {body}",
18067 codec_case.label()
18068 );
18069 }
18070 assert_eq!(
18071 server.request_body("/api/worker/query-tasks/codec-query/fail")["failure"]
18072 ["reason"],
18073 "query_payload_decode_failed"
18074 );
18075 for path in [
18076 "/api/worker/workflow-tasks/codec-workflow/complete",
18077 "/api/worker/activity-tasks/codec-activity/complete",
18078 "/api/worker/query-tasks/codec-query/complete",
18079 ] {
18080 assert_eq!(
18081 server.request_count(path),
18082 0,
18083 "invalid {} codec task reached {path}",
18084 codec_case.label()
18085 );
18086 }
18087 }
18088 }
18089
18090 #[tokio::test]
18091 async fn invalid_inbound_codecs_precede_handlers_and_unrelated_outcomes() {
18092 let client = Client::new("http://127.0.0.1:8080").expect("client");
18093 let mut worker = Worker::new(client, "rust-workers");
18094 let handler_calls = Arc::new(AtomicUsize::new(0));
18095
18096 let calls = Arc::clone(&handler_calls);
18097 worker.register_workflow("codec.workflow", move |_ctx, _args| {
18098 calls.fetch_add(1, Ordering::SeqCst);
18099 async move { Ok(Value::Null) }
18100 });
18101 let calls = Arc::clone(&handler_calls);
18102 worker.register_activity("codec.activity", move |_ctx, _args| {
18103 calls.fetch_add(1, Ordering::SeqCst);
18104 async move { Ok(Value::Null) }
18105 });
18106 let calls = Arc::clone(&handler_calls);
18107 worker.register_update("codec.workflow", "known", move |_ctx, _args| {
18108 calls.fetch_add(1, Ordering::SeqCst);
18109 async move { Ok(Value::Null) }
18110 });
18111 let calls = Arc::clone(&handler_calls);
18112 worker.register_query("codec.workflow", "known", move |_ctx, _args| {
18113 calls.fetch_add(1, Ordering::SeqCst);
18114 async move { Ok(Value::Null) }
18115 });
18116
18117 let mut workflow = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
18118 workflow.payload_codec = "json".to_string();
18119 workflow.arguments = None;
18120 let error = worker
18121 .execute_workflow_task(workflow)
18122 .expect_err("task codec must be checked before workflow invocation");
18123 assert!(error.to_string().contains("unsupported_payload_codec"));
18124
18125 let activity = ActivityTask {
18126 task_id: "activity-invalid-codec".to_string(),
18127 activity_attempt_id: None,
18128 attempt_id: None,
18129 activity_type: "codec.activity".to_string(),
18130 payload_codec: "unknown".to_string(),
18131 arguments: None,
18132 attempt_number: 1,
18133 lease_owner: None,
18134 };
18135 let error = worker
18136 .execute_activity_task(activity)
18137 .await
18138 .expect_err("task codec must be checked before activity invocation");
18139 assert!(error.to_string().contains("unsupported_payload_codec"));
18140
18141 let mut update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
18142 update.workflow_update_id = Some("update-invalid-codec".to_string());
18143 update.update_name = Some("known".to_string());
18144 update.history_events.push(history_event(
18145 "UpdateAccepted",
18146 json!({
18147 "update_id": "update-invalid-codec",
18148 "update_name": "known",
18149 "arguments": {"codec": "json", "blob": null}
18150 }),
18151 ));
18152 let error = worker
18153 .execute_workflow_task(update)
18154 .expect_err("nested update codec must be checked before handler lookup");
18155 assert!(error.to_string().contains("unsupported_payload_codec"));
18156
18157 let query: QueryTask = serde_json::from_value(json!({
18158 "query_task_id": "query-invalid-codec",
18159 "workflow_type": "codec.workflow",
18160 "query_name": "known",
18161 "payload_codec": DEFAULT_CODEC,
18162 "workflow_arguments": null,
18163 "query_arguments": null,
18164 "history_export": {
18165 "payloads": {"codec": DEFAULT_CODEC},
18166 "signals": [{
18167 "name": "empty",
18168 "payload_codec": "json",
18169 "arguments": null
18170 }]
18171 }
18172 }))
18173 .expect("query task");
18174 let failure = worker
18175 .execute_query_task(query)
18176 .await
18177 .expect_err("exported signal codec must be checked before query invocation");
18178 assert_eq!(failure.reason, "query_payload_decode_failed");
18179 assert!(failure.message.contains("unsupported_payload_codec"));
18180
18181 let exported_history: QueryTask = serde_json::from_value(json!({
18182 "query_task_id": "query-invalid-history-codec",
18183 "workflow_type": "codec.workflow",
18184 "query_name": "known",
18185 "payload_codec": DEFAULT_CODEC,
18186 "history_export": {
18187 "payloads": {"codec": DEFAULT_CODEC},
18188 "history_events": [{
18189 "type": "ActivityCompleted",
18190 "payload": {"payload_codec": "unknown", "result": null}
18191 }]
18192 }
18193 }))
18194 .expect("query task");
18195 let failure = worker
18196 .execute_query_task(exported_history)
18197 .await
18198 .expect_err("exported history codec must be checked before query invocation");
18199 assert_eq!(failure.reason, "query_payload_decode_failed");
18200 assert!(failure.message.contains("unsupported_payload_codec"));
18201 assert_eq!(handler_calls.load(Ordering::SeqCst), 0);
18202
18203 let mut unknown_workflow = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
18204 unknown_workflow.arguments = None;
18205 unknown_workflow.history_events.push(history_event(
18206 "SignalReceived",
18207 json!({
18208 "signal_name": "empty",
18209 "payload_codec": "json",
18210 "arguments": null
18211 }),
18212 ));
18213 let error = worker
18214 .execute_workflow_task(unknown_workflow)
18215 .expect_err("history codec must precede unknown workflow outcome");
18216 assert!(error.to_string().contains("unsupported_payload_codec"));
18217
18218 let unknown_activity = ActivityTask {
18219 task_id: "activity-unknown".to_string(),
18220 activity_attempt_id: None,
18221 attempt_id: None,
18222 activity_type: "missing".to_string(),
18223 payload_codec: "json".to_string(),
18224 arguments: None,
18225 attempt_number: 1,
18226 lease_owner: None,
18227 };
18228 let error = worker
18229 .execute_activity_task(unknown_activity)
18230 .await
18231 .expect_err("codec must precede unknown activity outcome");
18232 assert!(error.to_string().contains("unsupported_payload_codec"));
18233
18234 let mut unknown_update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
18235 unknown_update.payload_codec = "json".to_string();
18236 unknown_update.arguments = None;
18237 unknown_update.workflow_update_id = Some("update-unknown".to_string());
18238 unknown_update.update_name = Some("missing".to_string());
18239 let error = worker
18240 .execute_workflow_task(unknown_update)
18241 .expect_err("codec must precede fail_update shortcut");
18242 assert!(error.to_string().contains("unsupported_payload_codec"));
18243
18244 let unknown_query: QueryTask = serde_json::from_value(json!({
18245 "query_task_id": "query-unknown",
18246 "workflow_type": "missing",
18247 "query_name": "missing",
18248 "payload_codec": "json",
18249 "workflow_arguments": null,
18250 "query_arguments": null
18251 }))
18252 .expect("query task");
18253 let failure = worker
18254 .execute_query_task(unknown_query)
18255 .await
18256 .expect_err("codec must precede unknown query outcome");
18257 assert_eq!(failure.reason, "query_payload_decode_failed");
18258 assert!(failure.message.contains("unsupported_payload_codec"));
18259 }
18260
18261 #[tokio::test]
18262 async fn invalid_signal_history_payload_aliases_precede_shortcuts() {
18263 let client = Client::new("http://127.0.0.1:8080").expect("client");
18264 let worker = Worker::new(client, "rust-workers");
18265
18266 for event_type in ["SignalReceived", "SignalApplied"] {
18267 for (payload_field, codec) in [
18268 ("value", "json"),
18269 ("input", "unknown"),
18270 ("arguments", "json"),
18271 ] {
18272 let payload = json!({
18273 "signal_name": "empty",
18274 payload_field: {"codec": codec, "blob": null}
18275 });
18276 let workflow = workflow_task(
18277 "missing",
18278 vec![history_event(event_type, payload.clone())],
18279 DEFAULT_CODEC,
18280 );
18281 let error = worker
18282 .execute_workflow_task(workflow)
18283 .expect_err("signal payload codec must precede unknown workflow outcome");
18284 assert!(
18285 error.to_string().contains("unsupported_payload_codec"),
18286 "{event_type}.{payload_field} returned an unrelated workflow error: {error}"
18287 );
18288
18289 let query: QueryTask = serde_json::from_value(json!({
18290 "query_task_id": format!("query-{event_type}-{payload_field}"),
18291 "workflow_type": "missing",
18292 "query_name": "missing",
18293 "payload_codec": DEFAULT_CODEC,
18294 "workflow_arguments": null,
18295 "query_arguments": null,
18296 "history_events": [{
18297 "event_type": event_type,
18298 "payload": payload
18299 }]
18300 }))
18301 .expect("query task");
18302 let failure = worker
18303 .execute_query_task(query)
18304 .await
18305 .expect_err("signal payload codec must precede unknown query outcome");
18306 assert_eq!(
18307 failure.reason, "query_payload_decode_failed",
18308 "{event_type}.{payload_field} returned an unrelated query outcome"
18309 );
18310 assert!(
18311 failure.message.contains("unsupported_payload_codec"),
18312 "{event_type}.{payload_field} returned an unrelated query error: {}",
18313 failure.message
18314 );
18315 }
18316 }
18317 }
18318
18319 #[test]
18320 fn workflow_context_schedules_activity_until_completion_is_in_history() {
18321 let ctx = WorkflowContext {
18322 state: Arc::new(Mutex::new(
18323 WorkflowState::new_with_identity(
18324 Vec::new(),
18325 Some("wf-parent".to_string()),
18326 Some("run-parent".to_string()),
18327 "rust-workers".to_string(),
18328 DEFAULT_CODEC.to_string(),
18329 None,
18330 )
18331 .expect("workflow state"),
18332 )),
18333 };
18334
18335 let mut call = Box::pin(ctx.activity("hello.activity", json!(["Ada"])));
18336 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18337 assert!(matches!(
18338 call.as_mut().poll(&mut task_context),
18339 Poll::Pending
18340 ));
18341
18342 let commands = ctx.take_commands().expect("commands");
18343 assert_eq!(commands[0]["type"], "schedule_activity");
18344 assert_eq!(commands[0]["activity_type"], "hello.activity");
18345 }
18346
18347 #[test]
18348 fn activity_options_encode_retry_policy_queue_and_every_timeout() {
18349 let ctx = workflow_context(Vec::new());
18350 let options = ActivityOptions::new()
18351 .task_queue("payments")
18352 .retry_policy(
18353 ActivityRetryPolicy::new(4)
18354 .exponential_backoff(Duration::from_secs(1), 3, Some(Duration::from_secs(10)))
18355 .non_retryable_error_type("ValidationError"),
18356 )
18357 .start_to_close_timeout(Duration::from_secs(120))
18358 .schedule_to_start_timeout(Duration::from_secs(10))
18359 .schedule_to_close_timeout(Duration::from_secs(300))
18360 .heartbeat_timeout(Duration::from_secs(15));
18361 let mut call = Box::pin(ctx.activity_with_options(
18362 "charge-card",
18363 options,
18364 json!([{"order_id": "o-1"}]),
18365 ));
18366 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18367
18368 assert!(matches!(
18369 call.as_mut().poll(&mut task_context),
18370 Poll::Pending
18371 ));
18372 assert!(matches!(
18373 call.as_mut().poll(&mut task_context),
18374 Poll::Pending
18375 ));
18376
18377 let commands = ctx.take_commands().expect("activity command");
18378 assert_eq!(commands.len(), 1, "one future emits one logical schedule");
18379 assert_eq!(commands[0]["queue"], "payments");
18380 assert_eq!(
18381 commands[0]["retry_policy"],
18382 json!({
18383 "max_attempts": 4,
18384 "backoff_seconds": [1, 3, 9],
18385 "non_retryable_error_types": ["ValidationError"],
18386 })
18387 );
18388 assert_eq!(commands[0]["start_to_close_timeout"], 120);
18389 assert_eq!(commands[0]["schedule_to_start_timeout"], 10);
18390 assert_eq!(commands[0]["schedule_to_close_timeout"], 300);
18391 assert_eq!(commands[0]["heartbeat_timeout"], 15);
18392 }
18393
18394 #[test]
18395 fn activity_options_encode_explicit_and_rounded_backoff_intervals() {
18396 let ctx = workflow_context(Vec::new());
18397 let options = ActivityOptions::new().retry_policy(
18398 ActivityRetryPolicy::new(3)
18399 .backoff_intervals([Duration::from_millis(1), Duration::from_millis(1_001)]),
18400 );
18401 let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
18402 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18403
18404 assert!(matches!(
18405 call.as_mut().poll(&mut task_context),
18406 Poll::Pending
18407 ));
18408 assert_eq!(
18409 ctx.take_commands().expect("command")[0]["retry_policy"]["backoff_seconds"],
18410 json!([1, 2])
18411 );
18412 }
18413
18414 #[test]
18415 fn invalid_activity_options_return_typed_errors_before_emitting_commands() {
18416 let cases = [
18417 (
18418 ActivityOptions::new().task_queue(" "),
18419 ActivityOptionsErrorKind::EmptyTaskQueue,
18420 ),
18421 (
18422 ActivityOptions::new().retry_policy(ActivityRetryPolicy::default()),
18423 ActivityOptionsErrorKind::EmptyRetryPolicy,
18424 ),
18425 (
18426 ActivityOptions::new().retry_policy(ActivityRetryPolicy::new(0)),
18427 ActivityOptionsErrorKind::InvalidMaxAttempts,
18428 ),
18429 (
18430 ActivityOptions::new().retry_policy(ActivityRetryPolicy {
18431 max_attempts: None,
18432 backoff: Some(ActivityBackoff::Explicit(vec![Duration::from_secs(1)])),
18433 non_retryable_error_types: Vec::new(),
18434 }),
18435 ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
18436 ),
18437 (
18438 ActivityOptions::new().retry_policy(
18439 ActivityRetryPolicy::new(2)
18440 .backoff_intervals([Duration::from_secs(1), Duration::from_secs(2)]),
18441 ),
18442 ActivityOptionsErrorKind::TooManyBackoffIntervals,
18443 ),
18444 (
18445 ActivityOptions::new().retry_policy(
18446 ActivityRetryPolicy::new(2).exponential_backoff(
18447 Duration::from_secs(1),
18448 0,
18449 None,
18450 ),
18451 ),
18452 ActivityOptionsErrorKind::InvalidBackoffCoefficient,
18453 ),
18454 (
18455 ActivityOptions::new()
18456 .retry_policy(ActivityRetryPolicy::new(2).non_retryable_error_type(" ")),
18457 ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
18458 ),
18459 (
18460 ActivityOptions::new().retry_policy(
18461 ActivityRetryPolicy::new(10_002).exponential_backoff(
18462 Duration::from_secs(1),
18463 1,
18464 None,
18465 ),
18466 ),
18467 ActivityOptionsErrorKind::BackoffGenerationTooLarge,
18468 ),
18469 (
18470 ActivityOptions::new().retry_policy(
18471 ActivityRetryPolicy::new(2)
18472 .backoff_intervals([Duration::from_secs(i64::MAX as u64 + 1)]),
18473 ),
18474 ActivityOptionsErrorKind::BackoffOverflow,
18475 ),
18476 ];
18477
18478 for (options, expected_kind) in cases {
18479 let ctx = workflow_context(Vec::new());
18480 let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
18481 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18482 let Poll::Ready(Err(Error::InvalidActivityOptions(error))) =
18483 call.as_mut().poll(&mut task_context)
18484 else {
18485 panic!("expected typed activity validation error");
18486 };
18487 assert_eq!(error.kind, expected_kind);
18488 assert!(ctx.take_commands().expect("commands").is_empty());
18489 }
18490 }
18491
18492 #[test]
18493 fn activity_options_validate_positive_and_ordered_timeouts() {
18494 let zero_timeout_cases = [
18495 ActivityOptions::new().start_to_close_timeout(Duration::ZERO),
18496 ActivityOptions::new().schedule_to_start_timeout(Duration::ZERO),
18497 ActivityOptions::new().schedule_to_close_timeout(Duration::ZERO),
18498 ActivityOptions::new().heartbeat_timeout(Duration::ZERO),
18499 ];
18500 for options in zero_timeout_cases {
18501 assert_eq!(
18502 options.validate().expect_err("zero timeout").kind,
18503 ActivityOptionsErrorKind::TimeoutNotPositive
18504 );
18505 }
18506
18507 let ordering_cases = [
18508 ActivityOptions::new()
18509 .heartbeat_timeout(Duration::from_secs(11))
18510 .start_to_close_timeout(Duration::from_secs(10)),
18511 ActivityOptions::new()
18512 .start_to_close_timeout(Duration::from_secs(31))
18513 .schedule_to_close_timeout(Duration::from_secs(30)),
18514 ActivityOptions::new()
18515 .schedule_to_start_timeout(Duration::from_secs(31))
18516 .schedule_to_close_timeout(Duration::from_secs(30)),
18517 ];
18518 for options in ordering_cases {
18519 assert_eq!(
18520 options.validate().expect_err("timeout order").kind,
18521 ActivityOptionsErrorKind::TimeoutOrder
18522 );
18523 }
18524
18525 assert_eq!(
18526 ActivityOptions::new()
18527 .start_to_close_timeout(Duration::from_secs(i64::MAX as u64 + 1))
18528 .validate()
18529 .expect_err("protocol integer overflow")
18530 .kind,
18531 ActivityOptionsErrorKind::TimeoutOverflow
18532 );
18533 }
18534
18535 #[test]
18536 fn replayed_activity_retry_history_completes_without_duplicate_schedule() {
18537 let ctx = workflow_context(completed_retry_activity_history());
18538 let mut call =
18539 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18540 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18541
18542 assert!(matches!(
18543 call.as_mut().poll(&mut task_context),
18544 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18545 ));
18546 assert!(ctx.take_commands().expect("commands").is_empty());
18547 ctx.ensure_history_consumed().expect("history consumed");
18548 }
18549
18550 #[test]
18551 fn duplicate_non_retryable_types_use_one_command_and_replay_representation() {
18552 let mut options = retry_activity_options();
18553 options
18554 .retry_policy
18555 .as_mut()
18556 .expect("retry policy")
18557 .non_retryable_error_types
18558 .extend([" PermanentError ".to_string(), "PermanentError".to_string()]);
18559
18560 let new_ctx = workflow_context(Vec::new());
18561 let mut new_call =
18562 Box::pin(new_ctx.activity_with_options("flaky", options.clone(), json!([])));
18563 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18564 assert!(matches!(
18565 new_call.as_mut().poll(&mut task_context),
18566 Poll::Pending
18567 ));
18568 let commands = new_ctx.take_commands().expect("commands");
18569 assert_eq!(commands.len(), 1);
18570 assert_eq!(
18571 commands[0]["retry_policy"]["non_retryable_error_types"],
18572 json!(["PermanentError"])
18573 );
18574
18575 let replay_ctx = workflow_context(completed_retry_activity_history());
18576 let mut replay_call =
18577 Box::pin(replay_ctx.activity_with_options("flaky", options, json!([])));
18578 assert!(matches!(
18579 replay_call.as_mut().poll(&mut task_context),
18580 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18581 ));
18582 assert!(replay_ctx.take_commands().expect("commands").is_empty());
18583 replay_ctx
18584 .ensure_history_consumed()
18585 .expect("history consumed");
18586 }
18587
18588 #[test]
18589 fn replayed_intermediate_retry_remains_pending_across_restarts() {
18590 let history = completed_retry_activity_history()
18591 .into_iter()
18592 .take(3)
18593 .collect::<Vec<_>>();
18594
18595 for _restart in 0..2 {
18596 let ctx = workflow_context(history.clone());
18597 let mut call =
18598 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18599 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18600 assert!(matches!(
18601 call.as_mut().poll(&mut task_context),
18602 Poll::Pending
18603 ));
18604 assert!(ctx.take_commands().expect("commands").is_empty());
18605 }
18606 }
18607
18608 #[test]
18609 fn replayed_activity_rejects_changed_queue_retry_and_every_timeout_field() {
18610 let mut changed_queue = retry_activity_options();
18611 changed_queue.task_queue = Some("different-queue".to_string());
18612
18613 let mut changed_max_attempts = retry_activity_options();
18614 let retry_policy = changed_max_attempts
18615 .retry_policy
18616 .as_mut()
18617 .expect("retry policy");
18618 retry_policy.max_attempts = Some(4);
18619
18620 let mut changed_backoff = retry_activity_options();
18621 let retry_policy = changed_backoff.retry_policy.as_mut().expect("retry policy");
18622 retry_policy.backoff = Some(ActivityBackoff::Explicit(vec![
18623 Duration::from_secs(3),
18624 Duration::from_secs(4),
18625 ]));
18626
18627 let mut changed_non_retryable_types = retry_activity_options();
18628 let retry_policy = changed_non_retryable_types
18629 .retry_policy
18630 .as_mut()
18631 .expect("retry policy");
18632 retry_policy.non_retryable_error_types = vec!["AnotherPermanentError".to_string()];
18633
18634 let mut changed_start_to_close = retry_activity_options();
18635 changed_start_to_close.start_to_close_timeout = Some(Duration::from_secs(31));
18636 let mut changed_schedule_to_start = retry_activity_options();
18637 changed_schedule_to_start.schedule_to_start_timeout = Some(Duration::from_secs(6));
18638 let mut changed_schedule_to_close = retry_activity_options();
18639 changed_schedule_to_close.schedule_to_close_timeout = Some(Duration::from_secs(91));
18640 let mut changed_heartbeat = retry_activity_options();
18641 changed_heartbeat.heartbeat_timeout = Some(Duration::from_secs(11));
18642
18643 let cases = [
18644 (changed_queue, "activity_task_queue_mismatch"),
18645 (changed_max_attempts, "activity_retry_policy_mismatch"),
18646 (changed_backoff, "activity_retry_policy_mismatch"),
18647 (
18648 changed_non_retryable_types,
18649 "activity_retry_policy_mismatch",
18650 ),
18651 (changed_start_to_close, "activity_retry_policy_mismatch"),
18652 (changed_schedule_to_start, "activity_retry_policy_mismatch"),
18653 (changed_schedule_to_close, "activity_retry_policy_mismatch"),
18654 (changed_heartbeat, "activity_retry_policy_mismatch"),
18655 ];
18656
18657 for (options, expected_reason) in cases {
18658 let ctx = workflow_context(completed_retry_activity_history());
18659 let mut call = Box::pin(ctx.activity_with_options("flaky", options, json!([])));
18660 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18661 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18662 call.as_mut().poll(&mut task_context)
18663 else {
18664 panic!("changed activity options must fail replay");
18665 };
18666 assert_eq!(failure.reason, expected_reason);
18667 assert_eq!(failure.sequence, Some(1));
18668 assert!(ctx.take_commands().expect("commands").is_empty());
18669 }
18670 }
18671
18672 #[test]
18673 fn replayed_activity_rejects_changed_execution_mode_and_snapshot_version() {
18674 let cases = [
18675 (
18676 "execution_mode",
18677 json!("local"),
18678 "activity_execution_mode_mismatch",
18679 ),
18680 (
18681 "snapshot_version",
18682 json!(2),
18683 "activity_retry_policy_mismatch",
18684 ),
18685 ];
18686
18687 for (field, value, expected_reason) in cases {
18688 let mut history = completed_retry_activity_history();
18689 let activity = history[0].payload["activity"]
18690 .as_object_mut()
18691 .expect("activity snapshot");
18692 if field == "execution_mode" {
18693 activity.insert(field.to_string(), value);
18694 } else {
18695 activity["retry_policy"]
18696 .as_object_mut()
18697 .expect("retry snapshot")
18698 .insert(field.to_string(), value);
18699 }
18700
18701 let ctx = workflow_context(history);
18702 let mut call =
18703 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18704 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18705 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18706 call.as_mut().poll(&mut task_context)
18707 else {
18708 panic!("changed {field} must fail replay");
18709 };
18710 assert_eq!(failure.reason, expected_reason);
18711 assert_eq!(failure.sequence, Some(1));
18712 assert!(ctx.take_commands().expect("commands").is_empty());
18713 }
18714 }
18715
18716 #[test]
18717 fn replayed_legacy_activity_treats_missing_option_snapshot_as_unknown() {
18718 let mut history = completed_retry_activity_history();
18719 let activity = history[0].payload["activity"]
18720 .as_object_mut()
18721 .expect("activity snapshot");
18722 activity.remove("execution_mode");
18723 activity.remove("retry_policy");
18724
18725 let mut current = retry_activity_options();
18726 current.start_to_close_timeout = Some(Duration::from_secs(45));
18727 current.schedule_to_start_timeout = Some(Duration::from_secs(8));
18728 current.schedule_to_close_timeout = Some(Duration::from_secs(120));
18729 current.heartbeat_timeout = Some(Duration::from_secs(12));
18730
18731 let ctx = workflow_context(history);
18732 let mut call = Box::pin(ctx.activity_with_options("flaky", current, json!([])));
18733 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18734 assert!(matches!(
18735 call.as_mut().poll(&mut task_context),
18736 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18737 ));
18738 assert!(ctx.take_commands().expect("commands").is_empty());
18739 ctx.ensure_history_consumed().expect("history consumed");
18740 }
18741
18742 #[test]
18743 fn terminal_activity_failed_after_start_returns_typed_failure() {
18744 let history = vec![
18745 history_event(
18746 "ActivityScheduled",
18747 json!({
18748 "sequence": 1,
18749 "activity_type": "flaky",
18750 "activity_execution_id": "act-terminal",
18751 "activity": {
18752 "id": "act-terminal",
18753 "sequence": 1,
18754 "type": "flaky",
18755 "queue": "critical-activities",
18756 "retry_policy": {
18757 "snapshot_version": 1,
18758 "max_attempts": 3,
18759 "backoff_seconds": [2, 4],
18760 "non_retryable_error_types": ["PermanentError"]
18761 }
18762 }
18763 }),
18764 ),
18765 history_event(
18766 "ActivityStarted",
18767 json!({
18768 "sequence": 1,
18769 "activity_type": "flaky",
18770 "activity_execution_id": "act-terminal",
18771 "activity_attempt_id": "attempt-1",
18772 "attempt_number": 1
18773 }),
18774 ),
18775 history_event(
18776 "ActivityFailed",
18777 json!({
18778 "sequence": 1,
18779 "activity_type": "flaky",
18780 "activity_execution_id": "act-terminal",
18781 "activity_attempt_id": "attempt-1",
18782 "attempt_number": 1,
18783 "failure_id": "failure-terminal",
18784 "failure_category": "activity",
18785 "exception_type": "PermanentError",
18786 "message": "cannot retry",
18787 "non_retryable": true
18788 }),
18789 ),
18790 ];
18791 let ctx = workflow_context(history);
18792 let mut call =
18793 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18794 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18795
18796 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18797 call.as_mut().poll(&mut task_context)
18798 else {
18799 panic!("terminal ActivityFailed must settle the activity future");
18800 };
18801 assert_eq!(failure.kind, ActivityFailureKind::Failed);
18802 assert_eq!(
18803 failure.activity_execution_id.as_deref(),
18804 Some("act-terminal")
18805 );
18806 assert_eq!(failure.exception_type.as_deref(), Some("PermanentError"));
18807 assert!(failure.non_retryable);
18808 assert!(ctx.take_commands().expect("commands").is_empty());
18809 ctx.ensure_history_consumed().expect("history consumed");
18810 }
18811
18812 #[test]
18813 fn activity_terminal_events_return_machine_readable_failures() {
18814 let cases = [
18815 (
18816 "ActivityFailed",
18817 json!({
18818 "sequence": 1,
18819 "activity_type": "charge-card",
18820 "activity_execution_id": "act-1",
18821 "activity_attempt_id": "attempt-2",
18822 "attempt_number": 2,
18823 "failure_id": "failure-1",
18824 "failure_category": "activity",
18825 "exception_type": "PaymentDeclined",
18826 "exception_class": "payments.PaymentDeclined",
18827 "message": "card declined",
18828 "non_retryable": true
18829 }),
18830 ActivityFailureKind::Failed,
18831 "activity",
18832 ),
18833 (
18834 "ActivityCancelled",
18835 json!({
18836 "sequence": 1,
18837 "activity_type": "charge-card",
18838 "activity_execution_id": "act-1",
18839 "activity_attempt_id": "attempt-1"
18840 }),
18841 ActivityFailureKind::Cancelled,
18842 "cancelled",
18843 ),
18844 ];
18845
18846 for (event_type, payload, expected_kind, expected_reason) in cases {
18847 let ctx = workflow_context(vec![history_event(event_type, payload)]);
18848 let mut call = Box::pin(ctx.activity("charge-card", json!([])));
18849 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18850 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18851 call.as_mut().poll(&mut task_context)
18852 else {
18853 panic!("expected terminal activity failure");
18854 };
18855 assert_eq!(failure.kind, expected_kind);
18856 assert_eq!(failure.reason, expected_reason);
18857 assert_eq!(failure.activity_execution_id.as_deref(), Some("act-1"));
18858 assert_eq!(failure.activity_type.as_deref(), Some("charge-card"));
18859 }
18860 }
18861
18862 #[test]
18863 fn every_activity_timeout_class_is_typed() {
18864 for timeout_kind in [
18865 "start_to_close",
18866 "schedule_to_start",
18867 "schedule_to_close",
18868 "heartbeat",
18869 ] {
18870 let ctx = workflow_context(vec![history_event(
18871 "ActivityTimedOut",
18872 json!({
18873 "sequence": 1,
18874 "activity_type": "slow",
18875 "activity_execution_id": "act-timeout",
18876 "activity_attempt_id": "attempt-timeout",
18877 "failure_category": "timeout",
18878 "timeout_kind": timeout_kind,
18879 "message": "deadline expired"
18880 }),
18881 )]);
18882 let mut call = Box::pin(ctx.activity("slow", json!([])));
18883 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18884 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18885 call.as_mut().poll(&mut task_context)
18886 else {
18887 panic!("expected timeout failure");
18888 };
18889 assert_eq!(failure.kind, ActivityFailureKind::TimedOut);
18890 assert_eq!(failure.reason, timeout_kind);
18891 assert_eq!(failure.timeout_kind.as_deref(), Some(timeout_kind));
18892 assert_eq!(failure.failure_category.as_deref(), Some("timeout"));
18893 }
18894 }
18895
18896 #[test]
18897 fn workflow_sleep_emits_one_durable_timer_and_rounds_up() {
18898 let ctx = workflow_context(Vec::new());
18899 let mut sleep = Box::pin(ctx.sleep(Duration::from_millis(1_001)));
18900 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18901
18902 assert!(matches!(
18903 sleep.as_mut().poll(&mut task_context),
18904 Poll::Pending
18905 ));
18906 assert!(matches!(
18907 sleep.as_mut().poll(&mut task_context),
18908 Poll::Pending
18909 ));
18910
18911 let commands = ctx.take_commands().expect("timer command");
18912 assert_eq!(
18913 commands,
18914 vec![json!({
18915 "type": "start_timer",
18916 "delay_seconds": 2,
18917 })]
18918 );
18919 }
18920
18921 #[test]
18922 fn workflow_sleep_replays_matching_schedule_and_fire_without_a_command() {
18923 let history = vec![
18924 history_event(
18925 "TimerScheduled",
18926 json!({
18927 "sequence": 1,
18928 "timer_id": "timer-1",
18929 "delay_seconds": 5,
18930 "fire_at": "2026-07-11T12:00:05Z",
18931 }),
18932 ),
18933 history_event(
18934 "TimerFired",
18935 json!({
18936 "sequence": 1,
18937 "timer_id": "timer-1",
18938 "delay_seconds": 5,
18939 "fire_at": "2026-07-11T12:00:05Z",
18940 "fired_at": "2026-07-11T12:00:05Z",
18941 }),
18942 ),
18943 ];
18944
18945 for _restart in 0..2 {
18946 let ctx = workflow_context(history.clone());
18947 let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(5)));
18948 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18949 assert!(matches!(
18950 sleep.as_mut().poll(&mut task_context),
18951 Poll::Ready(Ok(()))
18952 ));
18953 assert!(ctx.take_commands().expect("commands").is_empty());
18954 ctx.ensure_history_consumed().expect("history consumed");
18955 }
18956 }
18957
18958 #[test]
18959 fn workflow_sleep_rejects_changed_delay_during_replay() {
18960 let ctx = workflow_context(vec![
18961 history_event(
18962 "TimerScheduled",
18963 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
18964 ),
18965 history_event(
18966 "TimerFired",
18967 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
18968 ),
18969 ]);
18970 let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(500)));
18971 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18972
18973 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18974 sleep.as_mut().poll(&mut task_context)
18975 else {
18976 panic!("changed timer delay must be rejected");
18977 };
18978 assert_eq!(failure.reason, "timer_delay_mismatch");
18979 assert_eq!(failure.sequence, Some(1));
18980 }
18981
18982 #[test]
18983 fn workflow_condition_wait_emits_published_identity_and_timeout_contract() {
18984 let ctx = workflow_context(Vec::new());
18985 let mut wait = Box::pin(
18986 ctx.wait_condition(
18987 ConditionWaitOptions::new("approval.ready", "sha256:approval-v1")
18988 .timeout(Duration::from_millis(60_001)),
18989 || Ok(false),
18990 ),
18991 );
18992 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18993
18994 assert!(matches!(
18995 wait.as_mut().poll(&mut task_context),
18996 Poll::Pending
18997 ));
18998 assert!(matches!(
18999 wait.as_mut().poll(&mut task_context),
19000 Poll::Pending
19001 ));
19002 assert_eq!(
19003 ctx.take_commands().expect("condition command"),
19004 vec![json!({
19005 "type": "open_condition_wait",
19006 "condition_wait_occurrence_id": "rust:condition-wait:0",
19007 "condition_key": "approval.ready",
19008 "condition_definition_fingerprint": "sha256:approval-v1",
19009 "timeout_seconds": 61,
19010 })]
19011 );
19012 }
19013
19014 #[test]
19015 fn workflow_condition_wait_returns_explicit_immediate_results_without_commands() {
19016 let ctx = workflow_context(Vec::new());
19017 let mut satisfied = Box::pin(wait_condition!(ctx, "already-ready", || Ok(true)));
19018 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19019 assert!(matches!(
19020 satisfied.as_mut().poll(&mut task_context),
19021 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19022 ));
19023
19024 let mut timed_out = Box::pin(wait_condition!(
19025 ctx,
19026 "no-wait",
19027 timeout: Duration::ZERO,
19028 || Ok(false),
19029 ));
19030 assert!(matches!(
19031 timed_out.as_mut().poll(&mut task_context),
19032 Poll::Ready(Ok(ConditionWaitResult::TimedOut))
19033 ));
19034 assert!(ctx.take_commands().expect("commands").is_empty());
19035 }
19036
19037 #[test]
19038 fn signal_and_update_history_reevaluate_open_conditions_after_restart() {
19039 let signal_history = vec![
19040 history_event(
19041 "ConditionWaitOpened",
19042 json!({
19043 "sequence": 4,
19044 "condition_wait_id": "condition:4",
19045 "condition_wait_occurrence_id": "rust:condition-wait:0",
19046 "condition_key": "approval",
19047 "condition_definition_fingerprint": "sha256:approval-v1",
19048 "timeout_seconds": 30,
19049 }),
19050 ),
19051 history_event(
19052 "SignalReceived",
19053 json!({
19054 "workflow_sequence": 4,
19055 "signal_name": "approve",
19056 "arguments": fixture_envelope(json!(["Ada"])),
19057 }),
19058 ),
19059 ];
19060 for _worker_before_or_after_restart in 0..2 {
19061 let ctx = workflow_context(signal_history.clone());
19062 let predicate_ctx = ctx.clone();
19063 let mut wait = Box::pin(
19064 ctx.wait_condition(
19065 ConditionWaitOptions::new("approval", "sha256:approval-v1")
19066 .timeout(Duration::from_secs(30)),
19067 move || Ok(!predicate_ctx.signals("approve")?.is_empty()),
19068 ),
19069 );
19070 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19071 assert!(matches!(
19072 wait.as_mut().poll(&mut task_context),
19073 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19074 ));
19075 assert!(ctx.take_commands().expect("commands").is_empty());
19076 ctx.ensure_history_consumed().expect("condition consumed");
19077 }
19078
19079 let update_history = vec![
19080 history_event(
19081 "ConditionWaitOpened",
19082 json!({
19083 "sequence": 7,
19084 "condition_wait_id": "condition:7",
19085 "condition_wait_occurrence_id": "rust:condition-wait:0",
19086 "condition_key": "update-approval",
19087 "condition_definition_fingerprint": "sha256:update-approval-v1",
19088 }),
19089 ),
19090 history_event(
19091 "UpdateApplied",
19092 json!({
19093 "sequence": 7,
19094 "update_id": "update-1",
19095 "update_name": "approve",
19096 "arguments": fixture_envelope(json!([true])),
19097 }),
19098 ),
19099 ];
19100 let ctx = workflow_context(update_history);
19101 let predicate_ctx = ctx.clone();
19102 let mut wait = Box::pin(ctx.wait_condition(
19103 ConditionWaitOptions::new("update-approval", "sha256:update-approval-v1"),
19104 move || {
19105 Ok(predicate_ctx
19106 .updates("approve")?
19107 .first()
19108 .and_then(|arguments| arguments.first())
19109 .and_then(Value::as_bool)
19110 == Some(true))
19111 },
19112 ));
19113 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19114 assert!(matches!(
19115 wait.as_mut().poll(&mut task_context),
19116 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19117 ));
19118 assert!(ctx.take_commands().expect("commands").is_empty());
19119 ctx.ensure_history_consumed().expect("condition consumed");
19120 }
19121
19122 #[test]
19123 fn condition_wait_preserves_open_satisfied_and_timed_out_replay_states() {
19124 let open_history = vec![
19125 history_event(
19126 "ConditionWaitOpened",
19127 json!({
19128 "sequence": 3,
19129 "condition_wait_id": "condition:3",
19130 "condition_wait_occurrence_id": "rust:condition-wait:0",
19131 "condition_key": "two-votes",
19132 "condition_definition_fingerprint": "sha256:two-votes-v1",
19133 "timeout_seconds": 120,
19134 }),
19135 ),
19136 history_event(
19137 "SignalReceived",
19138 json!({
19139 "workflow_sequence": 3,
19140 "signal_name": "vote",
19141 "arguments": fixture_envelope(json!(["first"])),
19142 }),
19143 ),
19144 ];
19145 for _worker_before_or_after_restart in 0..2 {
19146 let ctx = workflow_context(open_history.clone());
19147 let predicate_ctx = ctx.clone();
19148 let mut wait = Box::pin(
19149 ctx.wait_condition(
19150 ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1")
19151 .timeout(Duration::from_secs(120)),
19152 move || Ok(predicate_ctx.signals("vote")?.len() >= 2),
19153 ),
19154 );
19155 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19156 assert!(matches!(
19157 wait.as_mut().poll(&mut task_context),
19158 Poll::Pending
19159 ));
19160 assert_eq!(
19161 ctx.take_commands().expect("reopened condition"),
19162 vec![json!({
19163 "type": "open_condition_wait",
19164 "condition_wait_occurrence_id": "rust:condition-wait:0",
19165 "condition_key": "two-votes",
19166 "condition_definition_fingerprint": "sha256:two-votes-v1",
19167 "timeout_seconds": 120,
19168 })]
19169 );
19170 }
19171
19172 let satisfied_ctx = workflow_context(vec![
19173 history_event(
19174 "ConditionWaitOpened",
19175 json!({
19176 "sequence": 5,
19177 "condition_wait_id": "condition:5",
19178 "condition_wait_occurrence_id": "rust:condition-wait:0",
19179 "condition_key": "approval",
19180 "condition_definition_fingerprint": "sha256:approval-v1",
19181 }),
19182 ),
19183 history_event(
19184 "ConditionWaitSatisfied",
19185 json!({
19186 "sequence": 5,
19187 "condition_wait_id": "condition:5",
19188 "condition_wait_occurrence_id": "rust:condition-wait:0",
19189 "condition_key": "approval",
19190 "condition_definition_fingerprint": "sha256:approval-v1",
19191 }),
19192 ),
19193 ]);
19194 let mut satisfied = Box::pin(satisfied_ctx.wait_condition(
19195 ConditionWaitOptions::new("approval", "sha256:approval-v1"),
19196 || Ok(false),
19197 ));
19198 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19199 assert!(matches!(
19200 satisfied.as_mut().poll(&mut task_context),
19201 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19202 ));
19203
19204 let timed_out_ctx = workflow_context(vec![
19205 history_event(
19206 "ConditionWaitOpened",
19207 json!({
19208 "sequence": 8,
19209 "condition_wait_id": "condition:8",
19210 "condition_wait_occurrence_id": "rust:condition-wait:0",
19211 "condition_key": "approval-timeout",
19212 "condition_definition_fingerprint": "sha256:approval-timeout-v1",
19213 "timeout_seconds": 5,
19214 }),
19215 ),
19216 history_event(
19217 "TimerScheduled",
19218 json!({
19219 "sequence": 9,
19220 "timer_id": "condition-timer:9",
19221 "timer_kind": "condition_timeout",
19222 "condition_wait_id": "condition:8",
19223 "delay_seconds": 5,
19224 }),
19225 ),
19226 history_event(
19227 "TimerFired",
19228 json!({
19229 "sequence": 9,
19230 "timer_id": "condition-timer:9",
19231 "timer_kind": "condition_timeout",
19232 "condition_wait_id": "condition:8",
19233 "delay_seconds": 5,
19234 }),
19235 ),
19236 ]);
19237 let mut timed_out = Box::pin(
19238 timed_out_ctx.wait_condition(
19239 ConditionWaitOptions::new("approval-timeout", "sha256:approval-timeout-v1")
19240 .timeout(Duration::from_secs(5)),
19241 || Ok(true),
19242 ),
19243 );
19244 assert!(matches!(
19245 timed_out.as_mut().poll(&mut task_context),
19246 Poll::Ready(Ok(ConditionWaitResult::TimedOut))
19247 ));
19248 }
19249
19250 #[test]
19251 fn condition_wait_replays_repeated_physical_opens_as_one_logical_wait() {
19252 let history = vec![
19253 history_event(
19254 "ConditionWaitOpened",
19255 json!({
19256 "sequence": 3,
19257 "condition_wait_id": "condition:3",
19258 "condition_wait_occurrence_id": "rust:condition-wait:0",
19259 "condition_key": "two-votes",
19260 "condition_definition_fingerprint": "sha256:two-votes-v1",
19261 }),
19262 ),
19263 history_event(
19264 "SignalReceived",
19265 json!({
19266 "workflow_sequence": 3,
19267 "signal_name": "vote",
19268 "arguments": fixture_envelope(json!(["first"])),
19269 }),
19270 ),
19271 history_event(
19272 "ConditionWaitSatisfied",
19273 json!({
19274 "sequence": 3,
19275 "condition_wait_id": "condition:3",
19276 "condition_wait_occurrence_id": "rust:condition-wait:0",
19277 "condition_key": "two-votes",
19278 "condition_definition_fingerprint": "sha256:two-votes-v1",
19279 }),
19280 ),
19281 history_event(
19282 "ConditionWaitOpened",
19283 json!({
19284 "sequence": 5,
19285 "condition_wait_id": "condition:5",
19286 "condition_wait_occurrence_id": "rust:condition-wait:0",
19287 "condition_key": "two-votes",
19288 "condition_definition_fingerprint": "sha256:two-votes-v1",
19289 }),
19290 ),
19291 history_event(
19292 "SignalReceived",
19293 json!({
19294 "workflow_sequence": 5,
19295 "signal_name": "vote",
19296 "arguments": fixture_envelope(json!(["second"])),
19297 }),
19298 ),
19299 history_event(
19300 "ConditionWaitSatisfied",
19301 json!({
19302 "sequence": 5,
19303 "condition_wait_id": "condition:5",
19304 "condition_wait_occurrence_id": "rust:condition-wait:0",
19305 "condition_key": "two-votes",
19306 "condition_definition_fingerprint": "sha256:two-votes-v1",
19307 }),
19308 ),
19309 ];
19310 for _cold_worker_or_restart in 0..2 {
19311 let ctx = workflow_context(history.clone());
19312 let predicate_ctx = ctx.clone();
19313 let mut wait = Box::pin(ctx.wait_condition(
19314 ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1"),
19315 move || Ok(predicate_ctx.signals("vote")?.len() >= 2),
19316 ));
19317 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19318
19319 assert!(matches!(
19320 wait.as_mut().poll(&mut task_context),
19321 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19322 ));
19323 assert!(ctx.take_commands().expect("commands").is_empty());
19324 ctx.ensure_history_consumed()
19325 .expect("every physical wait-open is consumed");
19326 }
19327 }
19328
19329 #[test]
19330 fn condition_wait_replays_update_driven_physical_opens_as_one_occurrence() {
19331 let history = vec![
19332 history_event(
19333 "ConditionWaitOpened",
19334 json!({
19335 "sequence": 3,
19336 "condition_wait_id": "condition:3",
19337 "condition_wait_occurrence_id": "rust:condition-wait:0",
19338 "condition_key": "approved",
19339 "condition_definition_fingerprint": "sha256:approved-v1",
19340 }),
19341 ),
19342 history_event(
19343 "UpdateApplied",
19344 json!({
19345 "sequence": 3,
19346 "update_id": "update-1",
19347 "update_name": "approve",
19348 "arguments": fixture_envelope(json!([false])),
19349 }),
19350 ),
19351 history_event(
19352 "ConditionWaitOpened",
19353 json!({
19354 "sequence": 5,
19355 "condition_wait_id": "condition:5",
19356 "condition_wait_occurrence_id": "rust:condition-wait:0",
19357 "condition_key": "approved",
19358 "condition_definition_fingerprint": "sha256:approved-v1",
19359 }),
19360 ),
19361 history_event(
19362 "UpdateApplied",
19363 json!({
19364 "sequence": 5,
19365 "update_id": "update-2",
19366 "update_name": "approve",
19367 "arguments": fixture_envelope(json!([true])),
19368 }),
19369 ),
19370 ];
19371
19372 for _cold_worker_or_restart in 0..2 {
19373 let ctx = workflow_context(history.clone());
19374 let predicate_ctx = ctx.clone();
19375 let mut wait = Box::pin(ctx.wait_condition(
19376 ConditionWaitOptions::new("approved", "sha256:approved-v1"),
19377 move || {
19378 Ok(predicate_ctx
19379 .updates("approve")?
19380 .last()
19381 .and_then(|arguments| arguments.first())
19382 .and_then(Value::as_bool)
19383 == Some(true))
19384 },
19385 ));
19386 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19387
19388 assert!(matches!(
19389 wait.as_mut().poll(&mut task_context),
19390 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19391 ));
19392 assert!(ctx.take_commands().expect("commands").is_empty());
19393 ctx.ensure_history_consumed()
19394 .expect("every update-driven reopen is consumed");
19395 }
19396 }
19397
19398 #[test]
19399 fn condition_wait_replay_keeps_every_adjacent_authored_occurrence_distinct() {
19400 for (first_key, first_fingerprint, second_key, second_fingerprint) in [
19401 ("shared", "sha256:first", "shared", "sha256:second"),
19402 ("first", "sha256:shared", "second", "sha256:shared"),
19403 ("shared", "sha256:shared", "shared", "sha256:shared"),
19404 ("first", "sha256:first", "second", "sha256:second"),
19405 ] {
19406 let history = vec![
19407 history_event(
19408 "ConditionWaitOpened",
19409 json!({
19410 "sequence": 3,
19411 "condition_wait_id": "condition:3",
19412 "condition_wait_occurrence_id": "rust:condition-wait:0",
19413 "condition_key": first_key,
19414 "condition_definition_fingerprint": first_fingerprint,
19415 }),
19416 ),
19417 history_event(
19418 "ConditionWaitSatisfied",
19419 json!({
19420 "sequence": 3,
19421 "condition_wait_id": "condition:3",
19422 "condition_wait_occurrence_id": "rust:condition-wait:0",
19423 "condition_key": first_key,
19424 "condition_definition_fingerprint": first_fingerprint,
19425 }),
19426 ),
19427 history_event(
19428 "ConditionWaitOpened",
19429 json!({
19430 "sequence": 4,
19431 "condition_wait_id": "condition:4",
19432 "condition_wait_occurrence_id": "rust:condition-wait:1",
19433 "condition_key": second_key,
19434 "condition_definition_fingerprint": second_fingerprint,
19435 }),
19436 ),
19437 history_event(
19438 "ConditionWaitSatisfied",
19439 json!({
19440 "sequence": 4,
19441 "condition_wait_id": "condition:4",
19442 "condition_wait_occurrence_id": "rust:condition-wait:1",
19443 "condition_key": second_key,
19444 "condition_definition_fingerprint": second_fingerprint,
19445 }),
19446 ),
19447 ];
19448 for _cold_worker_or_restart in 0..2 {
19449 let ctx = workflow_context(history.clone());
19450 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19451 let mut first = Box::pin(ctx.wait_condition(
19452 ConditionWaitOptions::new(first_key, first_fingerprint),
19453 || Ok(false),
19454 ));
19455 assert!(matches!(
19456 first.as_mut().poll(&mut task_context),
19457 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19458 ));
19459
19460 let mut second = Box::pin(ctx.wait_condition(
19461 ConditionWaitOptions::new(second_key, second_fingerprint),
19462 || Ok(false),
19463 ));
19464 assert!(matches!(
19465 second.as_mut().poll(&mut task_context),
19466 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19467 ));
19468 assert!(ctx.take_commands().expect("commands").is_empty());
19469 ctx.ensure_history_consumed()
19470 .expect("each authored wait consumes one occurrence");
19471 }
19472 }
19473 }
19474
19475 #[test]
19476 fn cold_workers_replay_adjacent_condition_waits_from_one_loop_call_site() {
19477 fn worker() -> Worker {
19478 let client = Client::new("http://127.0.0.1:8080").expect("client");
19479 let mut worker = Worker::new(client, "rust-workers");
19480 worker.register_workflow("rust.condition-loop", |ctx, _input| async move {
19481 let mut outcomes = Vec::new();
19482 for _ in 0..2 {
19483 outcomes.push(
19484 ctx.wait_condition(
19485 ConditionWaitOptions::new("shared", "sha256:shared"),
19486 || Ok(false),
19487 )
19488 .await?,
19489 );
19490 }
19491 Ok(json!(outcomes))
19492 });
19493 worker
19494 }
19495
19496 let task = workflow_task(
19497 "rust.condition-loop",
19498 vec![
19499 history_event(
19500 "ConditionWaitOpened",
19501 json!({
19502 "sequence": 1,
19503 "condition_wait_id": "condition:1",
19504 "condition_wait_occurrence_id": "rust:condition-wait:0",
19505 "condition_key": "shared",
19506 "condition_definition_fingerprint": "sha256:shared",
19507 }),
19508 ),
19509 history_event(
19510 "ConditionWaitSatisfied",
19511 json!({
19512 "sequence": 1,
19513 "condition_wait_id": "condition:1",
19514 "condition_wait_occurrence_id": "rust:condition-wait:0",
19515 "condition_key": "shared",
19516 "condition_definition_fingerprint": "sha256:shared",
19517 }),
19518 ),
19519 history_event(
19520 "ConditionWaitOpened",
19521 json!({
19522 "sequence": 2,
19523 "condition_wait_id": "condition:2",
19524 "condition_wait_occurrence_id": "rust:condition-wait:1",
19525 "condition_key": "shared",
19526 "condition_definition_fingerprint": "sha256:shared",
19527 }),
19528 ),
19529 history_event(
19530 "ConditionWaitSatisfied",
19531 json!({
19532 "sequence": 2,
19533 "condition_wait_id": "condition:2",
19534 "condition_wait_occurrence_id": "rust:condition-wait:1",
19535 "condition_key": "shared",
19536 "condition_definition_fingerprint": "sha256:shared",
19537 }),
19538 ),
19539 ],
19540 DEFAULT_CODEC,
19541 );
19542
19543 for _cold_worker_or_restart in 0..2 {
19544 let commands = worker()
19545 .execute_workflow_task(task.clone())
19546 .expect("adjacent loop waits replay deterministically");
19547 assert_eq!(commands.len(), 1);
19548 assert_eq!(commands[0]["type"], "complete_workflow");
19549 assert_eq!(
19550 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
19551 json!(["satisfied", "satisfied"])
19552 );
19553 }
19554 }
19555
19556 #[test]
19557 fn condition_wait_replay_rejects_identity_predicate_and_timeout_changes() {
19558 let history = vec![history_event(
19559 "ConditionWaitOpened",
19560 json!({
19561 "sequence": 12,
19562 "condition_wait_id": "condition:12",
19563 "condition_wait_occurrence_id": "rust:condition-wait:0",
19564 "condition_key": "approval",
19565 "condition_definition_fingerprint": "sha256:approval-v1",
19566 "timeout_seconds": 30,
19567 }),
19568 )];
19569 for (options, expected_reason) in [
19570 (
19571 ConditionWaitOptions::new("changed", "sha256:approval-v1")
19572 .timeout(Duration::from_secs(30)),
19573 "condition_wait_key_mismatch",
19574 ),
19575 (
19576 ConditionWaitOptions::new("approval", "sha256:approval-v2")
19577 .timeout(Duration::from_secs(30)),
19578 "condition_wait_predicate_mismatch",
19579 ),
19580 (
19581 ConditionWaitOptions::new("approval", "sha256:approval-v1")
19582 .timeout(Duration::from_secs(29)),
19583 "condition_wait_timeout_mismatch",
19584 ),
19585 ] {
19586 let ctx = workflow_context(history.clone());
19587 let mut wait = Box::pin(ctx.wait_condition(options, || Ok(false)));
19588 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19589 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
19590 wait.as_mut().poll(&mut task_context)
19591 else {
19592 panic!("changed condition definition must fail replay");
19593 };
19594 assert_eq!(failure.reason, expected_reason);
19595 assert_eq!(failure.sequence, Some(12));
19596 }
19597 }
19598
19599 #[test]
19600 fn condition_wait_history_requires_the_canonical_predicate_fingerprint() {
19601 let error = WorkflowState::new(
19602 vec![history_event(
19603 "ConditionWaitOpened",
19604 json!({
19605 "sequence": 12,
19606 "condition_wait_id": "condition:12",
19607 "condition_wait_occurrence_id": "rust:condition-wait:0",
19608 "condition_key": "approval",
19609 }),
19610 )],
19611 "rust-workers".to_string(),
19612 DEFAULT_CODEC.to_string(),
19613 None,
19614 )
19615 .expect_err("condition history without a predicate fingerprint must fail");
19616
19617 assert!(matches!(
19618 error,
19619 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19620 if reason == "condition_wait_predicate_fingerprint_missing"
19621 ));
19622 }
19623
19624 #[test]
19625 fn condition_wait_history_requires_authored_occurrence_identity() {
19626 let error = WorkflowState::new(
19627 vec![history_event(
19628 "ConditionWaitOpened",
19629 json!({
19630 "sequence": 12,
19631 "condition_wait_id": "condition:12",
19632 "condition_key": "approval",
19633 "condition_definition_fingerprint": "sha256:approval-v1",
19634 }),
19635 )],
19636 "rust-workers".to_string(),
19637 DEFAULT_CODEC.to_string(),
19638 None,
19639 )
19640 .expect_err("condition history without occurrence identity must fail");
19641
19642 assert!(matches!(
19643 error,
19644 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19645 if reason == "condition_wait_occurrence_id_missing"
19646 ));
19647 }
19648
19649 #[test]
19650 fn typed_search_attribute_updates_validate_emit_and_replay() {
19651 let update = SearchAttributeUpdate::new()
19652 .keyword("OrderStatus", " waiting ")
19653 .expect("keyword")
19654 .int("Attempt", 3)
19655 .expect("int")
19656 .bool("Escalated", false)
19657 .expect("bool")
19658 .keyword_list("Regions", ["us-east", "eu-west"])
19659 .expect("list")
19660 .datetime("UpdatedAt", "2026-08-22T04:00:00Z")
19661 .expect("datetime")
19662 .delete("LegacyStatus")
19663 .expect("delete");
19664 let ctx = workflow_context(Vec::new());
19665 ctx.upsert_search_attributes(update.clone())
19666 .expect("typed update");
19667 assert_eq!(
19668 ctx.take_commands().expect("search-attribute command"),
19669 vec![json!({
19670 "type": "upsert_search_attributes",
19671 "attributes": {
19672 "Attempt": 3,
19673 "Escalated": false,
19674 "LegacyStatus": null,
19675 "OrderStatus": "waiting",
19676 "Regions": ["us-east", "eu-west"],
19677 "UpdatedAt": "2026-08-22T04:00:00Z",
19678 },
19679 "attribute_types": {
19680 "Attempt": "int",
19681 "Escalated": "bool",
19682 "OrderStatus": "keyword",
19683 "Regions": "keyword_list",
19684 "UpdatedAt": "datetime",
19685 },
19686 })]
19687 );
19688
19689 let replay = workflow_context(vec![history_event(
19690 "SearchAttributesUpserted",
19691 json!({
19692 "sequence": 6,
19693 "attributes": {
19694 "Attempt": 3,
19695 "Escalated": false,
19696 "LegacyStatus": null,
19697 "OrderStatus": "waiting",
19698 "Regions": ["us-east", "eu-west"],
19699 "UpdatedAt": "2026-08-22T04:00:00Z",
19700 },
19701 "attribute_types": {
19702 "Attempt": "int",
19703 "Escalated": "bool",
19704 "OrderStatus": "keyword",
19705 "Regions": "keyword_list",
19706 "UpdatedAt": "datetime",
19707 },
19708 "merged": {},
19709 }),
19710 )]);
19711 replay
19712 .upsert_search_attributes(update)
19713 .expect("matching update replays");
19714 assert!(replay.take_commands().expect("commands").is_empty());
19715 replay.ensure_history_consumed().expect("history consumed");
19716
19717 let type_drift = workflow_context(vec![history_event(
19718 "SearchAttributesUpserted",
19719 json!({
19720 "sequence": 7,
19721 "attributes": {"OrderStatus": "waiting"},
19722 "attribute_types": {"OrderStatus": "keyword"},
19723 "merged": {"OrderStatus": "waiting"},
19724 }),
19725 )]);
19726 let error = type_drift
19727 .upsert_search_attributes(
19728 SearchAttributeUpdate::new()
19729 .string("OrderStatus", "waiting")
19730 .expect("string update"),
19731 )
19732 .expect_err("same JSON value with a changed type must fail replay");
19733 assert!(matches!(
19734 error,
19735 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19736 if reason == "search_attribute_type_mismatch"
19737 ));
19738
19739 let malformed_types = WorkflowState::new(
19740 vec![history_event(
19741 "SearchAttributesUpserted",
19742 json!({
19743 "sequence": 8,
19744 "attributes": {"OrderStatus": "waiting"},
19745 "attribute_types": {"OrderStatus": "unsupported"},
19746 "merged": {"OrderStatus": "waiting"},
19747 }),
19748 )],
19749 "rust-workers".to_string(),
19750 DEFAULT_CODEC.to_string(),
19751 None,
19752 )
19753 .expect_err("unsupported search-attribute type metadata must fail");
19754 assert!(matches!(
19755 malformed_types,
19756 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19757 if reason == "search_attribute_types_malformed"
19758 ));
19759
19760 assert!(matches!(
19761 SearchAttributeUpdate::new().keyword("bad key", "value"),
19762 Err(SearchAttributeUpdateError::InvalidKey(_))
19763 ));
19764 assert!(matches!(
19765 SearchAttributeUpdate::new().float("Ratio", f64::NAN),
19766 Err(SearchAttributeUpdateError::NonFiniteFloat(_))
19767 ));
19768 assert!(matches!(
19769 SearchAttributeUpdate::new().keyword("UnicodeKeyword", "é".repeat(128)),
19770 Err(SearchAttributeUpdateError::ValueTooLong { .. })
19771 ));
19772 assert!(matches!(
19773 SearchAttributeUpdate::new().datetime("UpdatedAt", "2026-02-30T04:00:00Z"),
19774 Err(SearchAttributeUpdateError::InvalidDateTime(_))
19775 ));
19776 assert!(matches!(
19777 workflow_context(Vec::new()).upsert_search_attributes(SearchAttributeUpdate::new()),
19778 Err(Error::InvalidSearchAttributeUpdate(
19779 SearchAttributeUpdateError::Empty
19780 ))
19781 ));
19782 }
19783
19784 #[test]
19785 fn typed_search_attribute_text_uses_the_runtime_byte_limit() {
19786 let ascii = "a".repeat(MAX_SEARCH_ATTRIBUTE_STRING_LENGTH);
19787 let utf8 = "é".repeat(MAX_SEARCH_ATTRIBUTE_STRING_LENGTH / 2);
19788
19789 assert!(SearchAttributeUpdate::new()
19790 .string("AsciiDescription", ascii)
19791 .is_ok());
19792 assert!(SearchAttributeUpdate::new()
19793 .string("Utf8Description", utf8)
19794 .is_ok());
19795 assert!(matches!(
19796 SearchAttributeUpdate::new().string(
19797 "TooLongDescription",
19798 "é".repeat((MAX_SEARCH_ATTRIBUTE_STRING_LENGTH / 2) + 1),
19799 ),
19800 Err(SearchAttributeUpdateError::ValueTooLong {
19801 kind: "string",
19802 limit: MAX_SEARCH_ATTRIBUTE_STRING_LENGTH,
19803 ..
19804 })
19805 ));
19806 }
19807
19808 #[test]
19809 fn workflow_history_rejects_unpaired_or_mismatched_timer_events() {
19810 let lone_fire = WorkflowState::new(
19811 vec![history_event(
19812 "TimerFired",
19813 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19814 )],
19815 "rust-workers".to_string(),
19816 DEFAULT_CODEC.to_string(),
19817 None,
19818 )
19819 .expect_err("TimerFired requires TimerScheduled");
19820 assert!(matches!(
19821 lone_fire,
19822 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19823 if reason == "timer_schedule_missing_or_duplicate"
19824 ));
19825
19826 let wrong_identity = WorkflowState::new(
19827 vec![
19828 history_event(
19829 "TimerScheduled",
19830 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19831 ),
19832 history_event(
19833 "TimerFired",
19834 json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
19835 ),
19836 ],
19837 "rust-workers".to_string(),
19838 DEFAULT_CODEC.to_string(),
19839 None,
19840 )
19841 .expect_err("fire must match scheduled timer identity");
19842 assert!(matches!(
19843 wrong_identity,
19844 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19845 if reason == "timer_identity_mismatch"
19846 ));
19847
19848 let duplicate_fire = WorkflowState::new(
19849 vec![
19850 history_event(
19851 "TimerScheduled",
19852 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19853 ),
19854 history_event(
19855 "TimerFired",
19856 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19857 ),
19858 history_event(
19859 "TimerFired",
19860 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19861 ),
19862 ],
19863 "rust-workers".to_string(),
19864 DEFAULT_CODEC.to_string(),
19865 None,
19866 )
19867 .expect_err("a durable timer cannot fire twice");
19868 assert!(matches!(
19869 duplicate_fire,
19870 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19871 if reason == "duplicate_timer_fire"
19872 ));
19873
19874 let wrong_fired_delay = WorkflowState::new(
19875 vec![
19876 history_event(
19877 "TimerScheduled",
19878 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19879 ),
19880 history_event(
19881 "TimerFired",
19882 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 6}),
19883 ),
19884 ],
19885 "rust-workers".to_string(),
19886 DEFAULT_CODEC.to_string(),
19887 None,
19888 )
19889 .expect_err("timer schedule and fire delays must agree");
19890 assert!(matches!(
19891 wrong_fired_delay,
19892 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19893 if reason == "timer_history_delay_mismatch"
19894 ));
19895 }
19896
19897 #[test]
19898 fn replay_rejects_activity_moved_before_recorded_timer() {
19899 let ctx = workflow_context(vec![
19900 history_event(
19901 "TimerScheduled",
19902 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19903 ),
19904 history_event(
19905 "TimerFired",
19906 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19907 ),
19908 history_event(
19909 "ActivityCompleted",
19910 json!({
19911 "sequence": 2,
19912 "activity_type": "after-timer",
19913 "payload_codec": DEFAULT_CODEC,
19914 "result": fixture_envelope(json!("done")),
19915 }),
19916 ),
19917 ]);
19918 let mut activity = Box::pin(ctx.activity("after-timer", json!([])));
19919 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19920
19921 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
19922 activity.as_mut().poll(&mut task_context)
19923 else {
19924 panic!("reordered durable command must be rejected");
19925 };
19926 assert_eq!(failure.reason, "recorded_command_mismatch");
19927 assert_eq!(failure.sequence, Some(1));
19928 assert_eq!(failure.expected.as_deref(), Some("timer"));
19929 assert_eq!(failure.actual.as_deref(), Some("activity:after-timer"));
19930 }
19931
19932 #[test]
19933 fn workflow_context_emits_a_typed_named_signal_wait() {
19934 let ctx = workflow_context(Vec::new());
19935 let mut signal = Box::pin(ctx.wait_signal("finish"));
19936 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19937
19938 assert!(matches!(
19939 signal.as_mut().poll(&mut task_context),
19940 Poll::Pending
19941 ));
19942 assert_eq!(
19943 ctx.take_commands().expect("signal-wait command"),
19944 vec![json!({
19945 "type": "open_signal_wait",
19946 "signal_name": "finish",
19947 })]
19948 );
19949 }
19950
19951 #[test]
19952 fn runtime_message_stream_transport_cannot_be_opened_as_a_user_signal() {
19953 let ctx = workflow_context(Vec::new());
19954 let mut signal = Box::pin(ctx.wait_signal(MESSAGE_STREAM_SIGNAL));
19955 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19956
19957 let Poll::Ready(Err(Error::Codec(message))) = signal.as_mut().poll(&mut task_context)
19958 else {
19959 panic!("runtime-reserved signal should be rejected");
19960 };
19961 assert!(message.contains("reserved by the workflow runtime"));
19962 assert!(ctx.take_commands().expect("commands").is_empty());
19963 }
19964
19965 #[tokio::test]
19966 async fn runtime_message_stream_transport_cannot_be_sent_as_a_user_signal() {
19967 let client = Client::builder("http://127.0.0.1:9")
19968 .build()
19969 .expect("client");
19970 let error = client
19971 .signal_workflow("workflow-1", MESSAGE_STREAM_SIGNAL, json!(["forged"]))
19972 .await
19973 .expect_err("runtime-reserved signal should be rejected before transport");
19974
19975 assert!(
19976 matches!(error, Error::Codec(ref message) if message.contains("reserved by the workflow runtime"))
19977 );
19978 }
19979
19980 #[test]
19981 fn message_stream_worker_task_consumes_current_contiguous_bounded_batch() {
19982 fn delivery(message_id: &str, position: u64, value: &str) -> Value {
19983 let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
19984 value.to_string(),
19985 )]))
19986 .expect("message payload");
19987 json!({
19988 "schema": MESSAGE_STREAM_SCHEMA,
19989 "stream_name": "orders",
19990 "message_id": message_id,
19991 "position": position,
19992 "payload_envelope": payload,
19993 })
19994 }
19995
19996 fn opened(sequence: u64) -> HistoryEvent {
19997 history_event(
19998 "SignalWaitOpened",
19999 json!({
20000 "sequence": sequence,
20001 "signal_name": MESSAGE_STREAM_SIGNAL,
20002 }),
20003 )
20004 }
20005
20006 fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
20007 history_event(
20008 "SignalApplied",
20009 json!({
20010 "sequence": sequence,
20011 "signal_name": MESSAGE_STREAM_SIGNAL,
20012 "value": fixture_envelope(json!([delivery])),
20013 }),
20014 )
20015 }
20016
20017 fn received(delivery: Value) -> HistoryEvent {
20018 history_event(
20019 "SignalReceived",
20020 json!({
20021 "signal_name": MESSAGE_STREAM_SIGNAL,
20022 "arguments": fixture_envelope(json!([delivery])),
20023 "payload_codec": DEFAULT_CODEC,
20024 }),
20025 )
20026 }
20027
20028 let client = Client::new("http://127.0.0.1:8080").expect("client");
20029 let mut worker = Worker::new(client, "rust-workers");
20030 worker.register_workflow("rust.message-stream-batch", |ctx, _input| async move {
20031 let messages = ctx.message_stream("orders")?.receive(2).await?;
20032 Ok(json!(messages
20033 .into_iter()
20034 .map(|message| message.message_id)
20035 .collect::<Vec<_>>()))
20036 });
20037
20038 let first = delivery("message-1", 1, "one");
20039 let second = delivery("message-2", 2, "two");
20040 let batch = worker
20041 .execute_workflow_task_decision(workflow_task(
20042 "rust.message-stream-batch",
20043 vec![
20044 opened(1),
20045 received(first.clone()),
20046 applied(1, first.clone()),
20047 received(first.clone()),
20048 received(second),
20049 ],
20050 DEFAULT_CODEC,
20051 ))
20052 .expect("worker task consumes the available batch");
20053
20054 assert_eq!(batch.commands.len(), 1);
20055 assert_eq!(batch.commands[0]["type"], "complete_workflow");
20056 assert_eq!(
20057 decode_wire_value(&batch.commands[0]["result"], DEFAULT_CODEC)
20058 .expect("workflow result"),
20059 json!(["message-1", "message-2"])
20060 );
20061 assert_eq!(
20062 batch.message_stream_cursors,
20063 vec![json!({"stream_name": "orders", "through_position": 2})]
20064 );
20065 assert!(batch.message_stream_waits.is_empty());
20066
20067 let partial = worker
20068 .execute_workflow_task_decision(workflow_task(
20069 "rust.message-stream-batch",
20070 vec![opened(1), received(first.clone()), applied(1, first)],
20071 DEFAULT_CODEC,
20072 ))
20073 .expect("worker task returns without waiting for a missing second item");
20074 assert_eq!(partial.commands.len(), 1);
20075 assert_eq!(partial.commands[0]["type"], "complete_workflow");
20076 assert_eq!(
20077 decode_wire_value(&partial.commands[0]["result"], DEFAULT_CODEC)
20078 .expect("workflow result"),
20079 json!(["message-1"])
20080 );
20081 assert_eq!(
20082 partial.message_stream_cursors,
20083 vec![json!({"stream_name": "orders", "through_position": 1})]
20084 );
20085 assert!(partial.message_stream_waits.is_empty());
20086 }
20087
20088 #[test]
20089 fn message_stream_replay_preserves_partial_batch_boundary_before_later_wait() {
20090 fn delivery(message_id: &str, position: u64, value: &str) -> Value {
20091 let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
20092 value.to_string(),
20093 )]))
20094 .expect("message payload");
20095 json!({
20096 "schema": MESSAGE_STREAM_SCHEMA,
20097 "stream_name": "orders",
20098 "message_id": message_id,
20099 "position": position,
20100 "payload_envelope": payload,
20101 })
20102 }
20103
20104 fn opened(sequence: u64) -> HistoryEvent {
20105 history_event(
20106 "SignalWaitOpened",
20107 json!({
20108 "sequence": sequence,
20109 "signal_name": MESSAGE_STREAM_SIGNAL,
20110 }),
20111 )
20112 }
20113
20114 fn received(delivery: Value) -> HistoryEvent {
20115 history_event(
20116 "SignalReceived",
20117 json!({
20118 "signal_name": MESSAGE_STREAM_SIGNAL,
20119 "arguments": fixture_envelope(json!([delivery])),
20120 "payload_codec": DEFAULT_CODEC,
20121 }),
20122 )
20123 }
20124
20125 fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
20126 history_event(
20127 "SignalApplied",
20128 json!({
20129 "sequence": sequence,
20130 "signal_name": MESSAGE_STREAM_SIGNAL,
20131 "value": fixture_envelope(json!([delivery])),
20132 }),
20133 )
20134 }
20135
20136 let client = Client::new("http://127.0.0.1:8080").expect("client");
20137 let mut worker = Worker::new(client, "rust-workers");
20138 worker.register_workflow(
20139 "rust.message-stream-partial-batches",
20140 |ctx, _input| async move {
20141 let stream = ctx.message_stream("orders")?;
20142 let first = stream.receive(10).await?;
20143 let second = stream.receive(10).await?;
20144 Ok(json!([
20145 first
20146 .into_iter()
20147 .map(|message| message.message_id)
20148 .collect::<Vec<_>>(),
20149 second
20150 .into_iter()
20151 .map(|message| message.message_id)
20152 .collect::<Vec<_>>(),
20153 ]))
20154 },
20155 );
20156
20157 let first = delivery("message-1", 1, "one");
20158 let second = delivery("message-2", 2, "two");
20159 let decision = worker
20160 .execute_workflow_task_decision(workflow_task(
20161 "rust.message-stream-partial-batches",
20162 vec![
20163 opened(1),
20164 received(first.clone()),
20165 applied(1, first),
20166 opened(2),
20167 received(second.clone()),
20168 applied(2, second),
20169 ],
20170 DEFAULT_CODEC,
20171 ))
20172 .expect("cold replay preserves both authored receive boundaries");
20173
20174 assert_eq!(decision.commands.len(), 1);
20175 assert_eq!(decision.commands[0]["type"], "complete_workflow");
20176 assert_eq!(
20177 decode_wire_value(&decision.commands[0]["result"], DEFAULT_CODEC)
20178 .expect("workflow result"),
20179 json!([["message-1"], ["message-2"]])
20180 );
20181 assert_eq!(
20182 decision.message_stream_cursors,
20183 vec![json!({"stream_name": "orders", "through_position": 2})]
20184 );
20185 assert!(decision.message_stream_waits.is_empty());
20186 }
20187
20188 #[test]
20189 fn empty_message_stream_opens_internal_signal_wait_and_reports_position() {
20190 let ctx = workflow_context(Vec::new());
20191 let stream = ctx.message_stream("orders").expect("message stream");
20192 let mut receive = Box::pin(stream.receive(10));
20193 let mut task_context = TaskContext::from_waker(noop_waker_ref());
20194
20195 assert!(matches!(
20196 receive.as_mut().poll(&mut task_context),
20197 Poll::Pending
20198 ));
20199 assert_eq!(
20200 ctx.take_commands().expect("message-stream wait command"),
20201 vec![json!({
20202 "type": "open_signal_wait",
20203 "signal_name": MESSAGE_STREAM_SIGNAL,
20204 })]
20205 );
20206 let (cursors, waits) = ctx.message_stream_metadata().expect("stream metadata");
20207 assert!(cursors.is_empty());
20208 assert_eq!(
20209 waits,
20210 vec![json!({"stream_name": "orders", "after_position": 0})]
20211 );
20212 }
20213
20214 #[test]
20215 fn continue_as_new_cursor_checkpoint_preserves_global_pending_position() {
20216 let ctx = workflow_context(vec![history_event(
20217 "SignalReceived",
20218 json!({
20219 "signal_name": MESSAGE_STREAM_SIGNAL,
20220 "arguments": fixture_envelope(json!([{
20221 "schema": MESSAGE_STREAM_CURSOR_SCHEMA,
20222 "stream_name": "orders",
20223 "through_position": 2,
20224 }])),
20225 "payload_codec": DEFAULT_CODEC,
20226 }),
20227 )]);
20228 let stream = ctx.message_stream("orders").expect("message stream");
20229 let mut receive = Box::pin(stream.receive(10));
20230 let mut task_context = TaskContext::from_waker(noop_waker_ref());
20231
20232 assert!(matches!(
20233 receive.as_mut().poll(&mut task_context),
20234 Poll::Pending
20235 ));
20236 let (cursors, waits) = ctx.message_stream_metadata().expect("stream metadata");
20237 assert_eq!(
20238 cursors,
20239 vec![json!({"stream_name": "orders", "through_position": 2})]
20240 );
20241 assert_eq!(
20242 waits,
20243 vec![json!({"stream_name": "orders", "after_position": 2})]
20244 );
20245 }
20246
20247 #[test]
20248 fn message_stream_delivery_preserves_typed_avro_arguments_across_replay() {
20249 let mut empty_map = BTreeMap::new();
20250 let mut nested = BTreeMap::new();
20251 nested.insert(
20252 "value".to_string(),
20253 AvroValue::Array(vec![AvroValue::Bytes(b"nested".to_vec())]),
20254 );
20255 let values = vec![
20256 AvroValue::Bytes(vec![0, 255]),
20257 AvroValue::Long(1),
20258 AvroValue::Double(1.0),
20259 AvroValue::Array(Vec::new()),
20260 AvroValue::Map(std::mem::take(&mut empty_map)),
20261 AvroValue::Map(nested),
20262 ];
20263 let payload = encode_avro_value(&AvroValue::Array(values.clone())).expect("payload");
20264 let transport = vec![json!({
20265 "schema": MESSAGE_STREAM_SCHEMA,
20266 "stream_name": "orders",
20267 "message_id": "message-1",
20268 "position": 1,
20269 "payload_envelope": payload,
20270 })];
20271
20272 for _ in 0..2 {
20273 let Some(MessageStreamDelivery::Message(message)) =
20274 decode_message_stream_delivery(transport.clone()).expect("delivery")
20275 else {
20276 panic!("message delivery expected");
20277 };
20278 assert_eq!(message.arguments, values);
20279 assert!(matches!(message.arguments[1], AvroValue::Long(1)));
20280 assert!(matches!(message.arguments[2], AvroValue::Double(1.0)));
20281 }
20282 }
20283
20284 #[test]
20285 fn cold_worker_replacement_consumes_message_stream_wait_arrivals_once_in_order() {
20286 fn delivery(message_id: &str, position: u64, value: &str) -> Value {
20287 let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
20288 value.to_string(),
20289 )]))
20290 .expect("message payload");
20291 json!({
20292 "schema": MESSAGE_STREAM_SCHEMA,
20293 "stream_name": "orders",
20294 "message_id": message_id,
20295 "position": position,
20296 "payload_envelope": payload,
20297 })
20298 }
20299
20300 fn opened(sequence: u64) -> HistoryEvent {
20301 history_event(
20302 "SignalWaitOpened",
20303 json!({
20304 "sequence": sequence,
20305 "signal_name": MESSAGE_STREAM_SIGNAL,
20306 }),
20307 )
20308 }
20309
20310 fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
20311 history_event(
20312 "SignalApplied",
20313 json!({
20314 "sequence": sequence,
20315 "signal_name": MESSAGE_STREAM_SIGNAL,
20316 "value": fixture_envelope(json!([delivery])),
20317 }),
20318 )
20319 }
20320
20321 fn worker() -> Worker {
20322 let client = Client::new("http://127.0.0.1:8080").expect("client");
20323 let mut worker = Worker::new(client, "rust-workers");
20324 worker.register_workflow("rust.message-stream", |ctx, _input| async move {
20325 let stream = ctx.message_stream("orders")?;
20326 let first = stream.receive_one().await?;
20327 let second = stream.receive_one().await?;
20328 Ok(json!([first.message_id, second.message_id]))
20329 });
20330 worker
20331 }
20332
20333 fn task_with_resume(history: Vec<HistoryEvent>, delivery: Value) -> WorkflowTask {
20334 let mut task = workflow_task("rust.message-stream", history, DEFAULT_CODEC);
20335 task.signal_name = Some(MESSAGE_STREAM_SIGNAL.to_string());
20336 task.signal_arguments = Some(fixture_envelope(json!([delivery])));
20337 task
20338 }
20339
20340 let waiting = worker()
20341 .execute_workflow_task_decision(workflow_task(
20342 "rust.message-stream",
20343 Vec::new(),
20344 DEFAULT_CODEC,
20345 ))
20346 .expect("first worker opens the stream wait");
20347 assert_eq!(
20348 waiting.commands,
20349 vec![json!({
20350 "type": "open_signal_wait",
20351 "signal_name": MESSAGE_STREAM_SIGNAL,
20352 })]
20353 );
20354 assert!(waiting.message_stream_cursors.is_empty());
20355 assert_eq!(
20356 waiting.message_stream_waits,
20357 vec![json!({"stream_name": "orders", "after_position": 0})]
20358 );
20359
20360 let first_delivery = delivery("message-1", 1, "one");
20361 let first_arrival = worker()
20362 .execute_workflow_task_decision(task_with_resume(
20363 vec![opened(1)],
20364 first_delivery.clone(),
20365 ))
20366 .expect("replacement worker consumes the first arrival");
20367 assert_eq!(
20368 first_arrival.commands,
20369 vec![json!({
20370 "type": "open_signal_wait",
20371 "signal_name": MESSAGE_STREAM_SIGNAL,
20372 })]
20373 );
20374 assert_eq!(
20375 first_arrival.message_stream_cursors,
20376 vec![json!({"stream_name": "orders", "through_position": 1})]
20377 );
20378 assert_eq!(
20379 first_arrival.message_stream_waits,
20380 vec![json!({"stream_name": "orders", "after_position": 1})]
20381 );
20382
20383 let second_delivery = delivery("message-2", 2, "two");
20384 let first_applied = applied(1, first_delivery);
20385 let completed = worker()
20386 .execute_workflow_task_decision(task_with_resume(
20387 vec![opened(1), first_applied.clone(), opened(2)],
20388 second_delivery.clone(),
20389 ))
20390 .expect("next replacement worker consumes the second arrival");
20391 assert_eq!(completed.commands.len(), 1);
20392 assert_eq!(completed.commands[0]["type"], "complete_workflow");
20393 assert_eq!(
20394 decode_wire_value(&completed.commands[0]["result"], DEFAULT_CODEC)
20395 .expect("workflow result"),
20396 json!(["message-1", "message-2"])
20397 );
20398 assert_eq!(
20399 completed.message_stream_cursors,
20400 vec![json!({"stream_name": "orders", "through_position": 2})]
20401 );
20402 assert!(completed.message_stream_waits.is_empty());
20403
20404 let replay_history = vec![
20405 opened(1),
20406 first_applied,
20407 opened(2),
20408 applied(2, second_delivery),
20409 ];
20410 for _cold_worker_or_restart in 0..2 {
20411 let replayed = worker()
20412 .execute_workflow_task_decision(workflow_task(
20413 "rust.message-stream",
20414 replay_history.clone(),
20415 DEFAULT_CODEC,
20416 ))
20417 .expect("cold worker replays each logical message exactly once");
20418 assert_eq!(replayed.commands.len(), 1);
20419 assert_eq!(
20420 decode_wire_value(&replayed.commands[0]["result"], DEFAULT_CODEC)
20421 .expect("replayed workflow result"),
20422 json!(["message-1", "message-2"])
20423 );
20424 assert_eq!(
20425 replayed.message_stream_cursors,
20426 vec![json!({"stream_name": "orders", "through_position": 2})]
20427 );
20428 assert!(replayed.message_stream_waits.is_empty());
20429 }
20430 }
20431
20432 #[test]
20433 fn message_stream_capability_and_completion_require_protocol_one_fifteen() {
20434 assert!(!worker_protocol_supports_message_streams("1.14"));
20435 assert!(worker_protocol_supports_message_streams("1.15"));
20436 assert!(worker_protocol_supports_message_streams("1.16"));
20437 assert!(worker_protocol_supports_message_streams(
20438 WORKER_PROTOCOL_VERSION
20439 ));
20440 assert_eq!(MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION, "1.15");
20441 }
20442
20443 #[test]
20444 fn condition_wait_history_cannot_be_consumed_as_a_typed_signal_wait() {
20445 let ctx = workflow_context(vec![
20446 history_event(
20447 "ConditionWaitOpened",
20448 json!({
20449 "sequence": 1,
20450 "condition_wait_id": "condition:1",
20451 "condition_wait_occurrence_id": "rust:condition-wait:0",
20452 "condition_key": "signal:finish",
20453 "condition_definition_fingerprint": "sha256:signal-finish-v1",
20454 }),
20455 ),
20456 history_event(
20457 "ConditionWaitSatisfied",
20458 json!({
20459 "sequence": 1,
20460 "condition_wait_id": "condition:1",
20461 "condition_wait_occurrence_id": "rust:condition-wait:0",
20462 "condition_key": "signal:finish",
20463 "condition_definition_fingerprint": "sha256:signal-finish-v1",
20464 }),
20465 ),
20466 history_event(
20467 "SignalReceived",
20468 json!({"signal_name": "finish", "arguments": []}),
20469 ),
20470 ]);
20471 let mut signal = Box::pin(ctx.wait_signal("finish"));
20472 let mut task_context = TaskContext::from_waker(noop_waker_ref());
20473
20474 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
20475 signal.as_mut().poll(&mut task_context)
20476 else {
20477 panic!("condition history must not resolve as a typed signal wait");
20478 };
20479 assert_eq!(failure.reason, "recorded_command_mismatch");
20480 assert_eq!(failure.expected.as_deref(), Some("condition wait"));
20481 }
20482
20483 #[test]
20484 fn replay_orders_signal_waits_and_timers_in_one_command_stream() {
20485 let signal_then_timer = vec![
20486 history_event(
20487 "SignalWaitOpened",
20488 json!({"sequence": 1, "signal_name": "go"}),
20489 ),
20490 history_event(
20491 "SignalApplied",
20492 json!({
20493 "sequence": 1,
20494 "signal_name": "go",
20495 "value": fixture_envelope(json!(["now"])),
20496 }),
20497 ),
20498 history_event(
20499 "TimerScheduled",
20500 json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
20501 ),
20502 history_event(
20503 "TimerFired",
20504 json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
20505 ),
20506 ];
20507
20508 let ctx = workflow_context(signal_then_timer.clone());
20509 let mut signal = Box::pin(ctx.wait_signal("go"));
20510 let mut task_context = TaskContext::from_waker(noop_waker_ref());
20511 assert!(matches!(
20512 signal.as_mut().poll(&mut task_context),
20513 Poll::Ready(Ok(arguments)) if arguments == vec![json!("now")]
20514 ));
20515 let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5)));
20516 assert!(matches!(
20517 timer.as_mut().poll(&mut task_context),
20518 Poll::Ready(Ok(()))
20519 ));
20520 ctx.ensure_history_consumed()
20521 .expect("signal and timer history consumed in order");
20522
20523 let reordered = workflow_context(signal_then_timer);
20524 let mut timer_first = Box::pin(reordered.sleep(Duration::from_secs(5)));
20525 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
20526 timer_first.as_mut().poll(&mut task_context)
20527 else {
20528 panic!("timer cannot consume signal-wait-first history");
20529 };
20530 assert_eq!(failure.reason, "recorded_command_mismatch");
20531 assert_eq!(failure.sequence, Some(1));
20532 assert_eq!(failure.expected.as_deref(), Some("signal wait"));
20533
20534 let timer_then_signal = vec![
20535 history_event(
20536 "TimerScheduled",
20537 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20538 ),
20539 history_event(
20540 "TimerFired",
20541 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20542 ),
20543 history_event(
20544 "SignalWaitOpened",
20545 json!({"sequence": 2, "signal_name": "go"}),
20546 ),
20547 history_event(
20548 "SignalApplied",
20549 json!({
20550 "sequence": 2,
20551 "signal_name": "go",
20552 "value": fixture_envelope(json!([])),
20553 }),
20554 ),
20555 ];
20556 let reordered = workflow_context(timer_then_signal);
20557 let mut signal_first = Box::pin(reordered.wait_signal("go"));
20558 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
20559 signal_first.as_mut().poll(&mut task_context)
20560 else {
20561 panic!("signal wait cannot consume timer-first history");
20562 };
20563 assert_eq!(failure.reason, "recorded_command_mismatch");
20564 assert_eq!(failure.sequence, Some(1));
20565 assert_eq!(failure.expected.as_deref(), Some("timer"));
20566 }
20567
20568 #[test]
20569 fn workflow_history_rejects_duplicate_or_colliding_command_sequences() {
20570 let duplicate_timer = WorkflowState::new(
20571 vec![
20572 history_event(
20573 "TimerScheduled",
20574 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20575 ),
20576 history_event(
20577 "TimerScheduled",
20578 json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
20579 ),
20580 ],
20581 "rust-workers".to_string(),
20582 DEFAULT_CODEC.to_string(),
20583 None,
20584 )
20585 .expect_err("one workflow sequence cannot schedule two timers");
20586 assert!(matches!(
20587 duplicate_timer,
20588 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20589 if reason == "timer_schedule_missing_or_duplicate"
20590 ));
20591
20592 let colliding_kinds = WorkflowState::new(
20593 vec![
20594 history_event(
20595 "TimerScheduled",
20596 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20597 ),
20598 history_event(
20599 "ActivityCompleted",
20600 json!({"sequence": 1, "activity_type": "same-sequence"}),
20601 ),
20602 ],
20603 "rust-workers".to_string(),
20604 DEFAULT_CODEC.to_string(),
20605 None,
20606 )
20607 .expect_err("one workflow sequence cannot identify two command kinds");
20608 assert!(matches!(
20609 colliding_kinds,
20610 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20611 if reason == "durable_command_sequence_collision"
20612 ));
20613
20614 let duplicate_signal_wait = WorkflowState::new(
20615 vec![
20616 history_event(
20617 "SignalWaitOpened",
20618 json!({"sequence": 1, "signal_name": "go"}),
20619 ),
20620 history_event(
20621 "SignalWaitOpened",
20622 json!({"sequence": 1, "signal_name": "go"}),
20623 ),
20624 ],
20625 "rust-workers".to_string(),
20626 DEFAULT_CODEC.to_string(),
20627 None,
20628 )
20629 .expect_err("one workflow sequence cannot open two signal waits");
20630 assert!(matches!(
20631 duplicate_signal_wait,
20632 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20633 if reason == "signal_wait_open_missing_or_duplicate"
20634 ));
20635 }
20636
20637 #[test]
20638 fn workflow_history_accepts_a_first_command_after_global_sequence_gaps() {
20639 let result = encode_value_envelope(&json!({"captured": true}), DEFAULT_CODEC)
20640 .expect("side-effect result");
20641 let ctx = workflow_context(vec![history_event(
20642 "SideEffectRecorded",
20643 json!({"sequence": 99, "result": result}),
20644 )]);
20645
20646 let replayed: Value = ctx
20647 .side_effect(|| panic!("recorded side effect must not run"))
20648 .expect("positive global workflow sequence is valid");
20649 assert_eq!(replayed, json!({"captured": true}));
20650 ctx.ensure_history_consumed().expect("history consumed");
20651 }
20652
20653 #[test]
20654 fn workflow_history_rejects_zero_and_descending_command_sequences() {
20655 let result =
20656 encode_value_envelope(&json!("captured"), DEFAULT_CODEC).expect("side-effect result");
20657 let zero = WorkflowState::new(
20658 vec![history_event(
20659 "SideEffectRecorded",
20660 json!({"sequence": 0, "result": result.clone()}),
20661 )],
20662 "rust-workers".to_string(),
20663 DEFAULT_CODEC.to_string(),
20664 None,
20665 )
20666 .expect_err("durable command sequences must be positive");
20667 assert!(matches!(
20668 zero,
20669 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20670 if reason == "durable_command_sequence_invalid"
20671 ));
20672
20673 let descending = WorkflowState::new(
20674 vec![
20675 history_event(
20676 "SideEffectRecorded",
20677 json!({"sequence": 3, "result": result}),
20678 ),
20679 history_event(
20680 "VersionMarkerRecorded",
20681 json!({
20682 "sequence": 2,
20683 "change_id": "descending-marker",
20684 "version": 1,
20685 "min_supported": 1,
20686 "max_supported": 1,
20687 }),
20688 ),
20689 ],
20690 "rust-workers".to_string(),
20691 DEFAULT_CODEC.to_string(),
20692 None,
20693 )
20694 .expect_err("new durable commands must remain strictly ordered");
20695 let Error::NonDeterministicReplay(failure) = descending else {
20696 panic!("expected typed replay failure");
20697 };
20698 assert_eq!(failure.reason, "durable_command_sequence_mismatch");
20699 assert_eq!(failure.sequence, Some(2));
20700 assert_eq!(
20701 failure.expected.as_deref(),
20702 Some("workflow sequence greater than 3")
20703 );
20704 assert_eq!(failure.actual.as_deref(), Some("2"));
20705 }
20706
20707 #[test]
20708 fn workflow_task_replay_completes_after_signals_create_sequence_gaps() {
20709 fn worker() -> Worker {
20710 let client = Client::new("http://127.0.0.1:8080").expect("client");
20711 let mut worker = Worker::new(client, "rust-workers");
20712 worker.register_workflow("rust.finish-after-gaps", |ctx, _input| async move {
20713 ctx.wait_signal("finish").await?;
20714 let marker: String =
20715 ctx.side_effect(|| panic!("recorded side effect must not run"))?;
20716 assert_eq!(marker, "after-finish");
20717 Ok(json!("finished"))
20718 });
20719 worker
20720 }
20721
20722 let marker = encode_value_envelope(&json!("after-finish"), DEFAULT_CODEC)
20723 .expect("side-effect result");
20724 let task = workflow_task(
20725 "rust.finish-after-gaps",
20726 vec![
20727 history_event(
20728 "SignalWaitOpened",
20729 json!({"sequence": 1, "signal_name": "finish"}),
20730 ),
20731 history_event(
20732 "SignalReceived",
20733 json!({
20734 "signal_id": "increment-3",
20735 "signal_name": "increment",
20736 "workflow_sequence": 2,
20737 "payload_codec": DEFAULT_CODEC,
20738 "arguments": fixture_envelope(json!([3])),
20739 }),
20740 ),
20741 history_event(
20742 "SignalReceived",
20743 json!({
20744 "signal_id": "increment-5",
20745 "signal_name": "increment",
20746 "workflow_sequence": 3,
20747 "payload_codec": DEFAULT_CODEC,
20748 "arguments": fixture_envelope(json!([5])),
20749 }),
20750 ),
20751 history_event(
20752 "SignalReceived",
20753 json!({
20754 "signal_id": "finish",
20755 "signal_name": "finish",
20756 "workflow_sequence": 4,
20757 "payload_codec": DEFAULT_CODEC,
20758 "arguments": fixture_envelope(json!([])),
20759 }),
20760 ),
20761 history_event(
20762 "SignalApplied",
20763 json!({
20764 "sequence": 1,
20765 "signal_id": "finish",
20766 "signal_name": "finish",
20767 "payload_codec": DEFAULT_CODEC,
20768 "value": fixture_envelope(json!([])),
20769 }),
20770 ),
20771 history_event(
20772 "SideEffectRecorded",
20773 json!({"sequence": 5, "result": marker}),
20774 ),
20775 ],
20776 DEFAULT_CODEC,
20777 );
20778
20779 for _original_or_cold_worker in 0..2 {
20780 let commands = worker()
20781 .execute_workflow_task(task.clone())
20782 .expect("signal gaps preserve deterministic replay");
20783 assert_eq!(commands.len(), 1, "replay emits only terminal completion");
20784 assert_eq!(commands[0]["type"], "complete_workflow");
20785 assert_eq!(
20786 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
20787 json!("finished")
20788 );
20789 }
20790 }
20791
20792 #[test]
20793 fn workflow_sleep_rejects_unrepresentable_rounded_duration() {
20794 let ctx = workflow_context(Vec::new());
20795 let mut sleep = Box::pin(ctx.start_timer(Duration::new(u64::MAX, 1)));
20796 let mut task_context = TaskContext::from_waker(noop_waker_ref());
20797 assert!(matches!(
20798 sleep.as_mut().poll(&mut task_context),
20799 Poll::Ready(Err(Error::TimerDurationOverflow))
20800 ));
20801 assert!(ctx.take_commands().expect("commands").is_empty());
20802 }
20803
20804 #[test]
20805 fn workflow_memo_update_emits_canonical_command_and_replays_once() {
20806 let entries = AvroValue::Map(BTreeMap::from([
20807 ("text".to_string(), AvroValue::String("same".to_string())),
20808 (
20809 "nested".to_string(),
20810 AvroValue::Map(BTreeMap::from([
20811 ("beta".to_string(), AvroValue::Long(2)),
20812 ("alpha".to_string(), AvroValue::Long(1)),
20813 ])),
20814 ),
20815 ("long".to_string(), AvroValue::Long(7)),
20816 ("double".to_string(), AvroValue::Double(7.0)),
20817 ("binary".to_string(), AvroValue::Bytes(b"same".to_vec())),
20818 ]));
20819 let ctx = workflow_context(Vec::new());
20820 ctx.upsert_memo(entries.clone()).expect("valid memo update");
20821 let commands = ctx.take_commands().expect("commands");
20822
20823 assert_eq!(commands.len(), 1);
20824 assert_eq!(commands[0]["type"], "upsert_memo");
20825 let server_entries = json!({
20826 "codec": "avro",
20827 "blob": "wwHioz3/VYAiNw4KDGJpbmFyeQgIc2FtZQxkb3VibGUGAAAAAAAAHEAIbG9uZwQODG5lc3RlZA4ECmFscGhhBAIIYmV0YQQEAAh0ZXh0CghzYW1lAA==",
20828 });
20829 assert_eq!(
20830 commands[0]["entries"]
20831 .as_object()
20832 .expect("entries envelope")
20833 .keys()
20834 .collect::<Vec<_>>(),
20835 vec!["blob", "codec"]
20836 );
20837 assert_eq!(commands[0]["entries"], server_entries);
20838 let wire_entries =
20839 decode_wire_avro_value(&commands[0]["entries"], DEFAULT_CODEC).expect("memo entries");
20840 assert_eq!(wire_entries, entries);
20841
20842 let history = vec![history_event(
20843 "MemoUpserted",
20844 json!({
20845 "sequence": 1,
20846 "entries": server_entries.clone(),
20847 "merged": server_entries,
20848 }),
20849 )];
20850 let replay = workflow_context(history.clone());
20851 replay
20852 .upsert_memo(entries.clone())
20853 .expect("matching replay identity");
20854 assert!(replay.take_commands().expect("replay commands").is_empty());
20855
20856 let changed_types = AvroValue::Map(BTreeMap::from([
20857 ("text".to_string(), AvroValue::Bytes(b"same".to_vec())),
20858 (
20859 "nested".to_string(),
20860 AvroValue::Map(BTreeMap::from([
20861 ("alpha".to_string(), AvroValue::Long(1)),
20862 ("beta".to_string(), AvroValue::Long(2)),
20863 ])),
20864 ),
20865 ("long".to_string(), AvroValue::Double(7.0)),
20866 ("double".to_string(), AvroValue::Long(7)),
20867 ("binary".to_string(), AvroValue::String("same".to_string())),
20868 ]));
20869 let error = workflow_context(history)
20870 .upsert_memo(changed_types)
20871 .expect_err("memo replay identity must preserve Avro value types");
20872 assert!(matches!(
20873 error,
20874 Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20875 ));
20876 }
20877
20878 #[test]
20879 fn workflow_memo_update_rejects_changed_replay_identity_and_invalid_keys() {
20880 let original = encode_value_envelope(&json!({"stage": "original"}), DEFAULT_CODEC)
20881 .expect("memo envelope");
20882 let replay = workflow_context(vec![history_event(
20883 "MemoUpserted",
20884 json!({
20885 "sequence": 1,
20886 "entries": original.clone(),
20887 "merged": original
20888 }),
20889 )]);
20890 let error = replay
20891 .upsert_memo(json!({"stage": "changed"}))
20892 .expect_err("changed memo update must fail replay");
20893 assert!(matches!(
20894 error,
20895 Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20896 ));
20897
20898 let invalid = workflow_context(Vec::new())
20899 .upsert_memo(
20900 json!({"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx": true}),
20901 )
20902 .expect_err("oversized key");
20903 assert!(matches!(invalid, Error::InvalidMemoUpdate(_)));
20904 }
20905
20906 #[test]
20907 fn workflow_memo_replay_distinguishes_signed_zero_identity() {
20908 let negative_zero = AvroValue::Map(BTreeMap::from([(
20909 "reading".to_string(),
20910 AvroValue::Double(-0.0),
20911 )]));
20912 let negative_zero_envelope =
20913 encode_typed_envelope(&negative_zero, DEFAULT_CODEC).expect("negative zero envelope");
20914 let history = vec![history_event(
20915 "MemoUpserted",
20916 json!({
20917 "sequence": 1,
20918 "entries": negative_zero_envelope.clone(),
20919 "merged": negative_zero_envelope,
20920 }),
20921 )];
20922
20923 workflow_context(history.clone())
20924 .upsert_memo(negative_zero)
20925 .expect("matching negative-zero history identity");
20926
20927 let error = workflow_context(history)
20928 .upsert_memo(AvroValue::Map(BTreeMap::from([(
20929 "reading".to_string(),
20930 AvroValue::Double(0.0),
20931 )])))
20932 .expect_err("positive zero must not consume negative-zero memo history");
20933 assert!(matches!(
20934 error,
20935 Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20936 ));
20937 }
20938
20939 #[test]
20940 fn workflow_memo_capability_requires_flag_and_command_advertisement() {
20941 let supported = json!({
20942 "workflow_memo_updates": {"supported": true, "minimum_protocol_version": "1.14"},
20943 "supported_workflow_task_commands": ["complete_workflow", "upsert_memo"]
20944 });
20945 assert!(runtime_supports_workflow_memo_updates(Some(&supported)));
20946 assert!(!runtime_supports_workflow_memo_updates(Some(&json!({
20947 "workflow_memo_updates": {"supported": false},
20948 "supported_workflow_task_commands": ["upsert_memo"]
20949 }))));
20950 assert!(commands_use_workflow_memo_updates(&[json!({
20951 "type": "upsert_memo",
20952 "entries": {"stage": "processing"}
20953 })]));
20954 }
20955
20956 #[test]
20957 fn workflow_task_replay_completes_without_rescheduling_recorded_commands() {
20958 let client = Client::new("http://127.0.0.1:8080").expect("client");
20959 let mut worker = Worker::new(client, "rust-workers");
20960 worker.register_workflow("rust.timer", |ctx, _input| async move {
20961 ctx.sleep(Duration::from_secs(5)).await?;
20962 ctx.activity("after-timer", json!([])).await
20963 });
20964
20965 let task = |history_events| WorkflowTask {
20966 task_id: "wft-rust-timer-1".to_string(),
20967 workflow_command_id: None,
20968 workflow_id: Some("wf-rust-timer".to_string()),
20969 run_id: Some("run-rust-timer".to_string()),
20970 workflow_type: "rust.timer".to_string(),
20971 cancel_requested: false,
20972 payload_codec: DEFAULT_CODEC.to_string(),
20973 arguments: Some(
20974 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
20975 ),
20976 history_events,
20977 total_history_events: None,
20978 history_size_bytes: None,
20979 continue_as_new_recommended: None,
20980 history_budget_pressure: None,
20981 next_history_page_token: None,
20982 workflow_task_attempt: 1,
20983 workflow_signal_id: None,
20984 signal_name: None,
20985 signal_arguments: None,
20986 workflow_update_id: None,
20987 update_name: None,
20988 lease_owner: Some("rust-worker".to_string()),
20989 };
20990
20991 let initial = worker
20992 .execute_workflow_task(task(Vec::new()))
20993 .expect("initial timer task");
20994 assert_eq!(
20995 initial,
20996 vec![json!({"type": "start_timer", "delay_seconds": 5})]
20997 );
20998
20999 let activity_result =
21000 encode_value_envelope(&json!("done"), DEFAULT_CODEC).expect("activity result");
21001 let replayed = worker
21002 .execute_workflow_task(task(vec![
21003 history_event(
21004 "TimerScheduled",
21005 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
21006 ),
21007 history_event(
21008 "TimerFired",
21009 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
21010 ),
21011 history_event(
21012 "ActivityCompleted",
21013 json!({
21014 "sequence": 2,
21015 "activity_type": "after-timer",
21016 "payload_codec": DEFAULT_CODEC,
21017 "result": activity_result,
21018 }),
21019 ),
21020 ]))
21021 .expect("replayed workflow task");
21022 assert_eq!(replayed.len(), 1);
21023 assert_eq!(replayed[0]["type"], "complete_workflow");
21024 assert_eq!(
21025 decode_wire_value(&replayed[0]["result"], DEFAULT_CODEC).expect("result"),
21026 json!("done")
21027 );
21028 }
21029
21030 #[test]
21031 fn workflow_continue_as_new_emits_arguments_type_and_queue_once() {
21032 let client = Client::new("http://127.0.0.1:8080").expect("client");
21033 let mut worker = Worker::new(client, "rust-workers");
21034 worker.register_workflow("rust.continue", |ctx, _input| async move {
21035 ctx.continue_as_new_with_options(
21036 ContinueAsNewOptions::new()
21037 .workflow_type("rust.next")
21038 .task_queue("next-workers"),
21039 json!([2, {"cursor": "next"}]),
21040 )
21041 });
21042
21043 let commands = worker
21044 .execute_workflow_task(workflow_task("rust.continue", Vec::new(), DEFAULT_CODEC))
21045 .expect("continue-as-new command");
21046
21047 assert_eq!(commands.len(), 1);
21048 assert_eq!(commands[0]["type"], "continue_as_new");
21049 assert_eq!(commands[0]["workflow_type"], "rust.next");
21050 assert_eq!(commands[0]["queue"], "next-workers");
21051 assert_eq!(
21052 decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC)
21053 .expect("continue-as-new arguments"),
21054 json!([2, {"cursor": "next"}])
21055 );
21056 }
21057
21058 #[test]
21059 fn continue_as_new_preserves_typed_arguments() {
21060 let client = Client::new("http://127.0.0.1:8080").expect("client");
21061 let mut worker = Worker::new(client, "rust-workers");
21062 worker.register_workflow_avro_value("rust.typed-continue", |ctx, _input| async move {
21063 ctx.continue_as_new(AvroValue::Array(vec![typed_fidelity_probe()]))?;
21064 unreachable!("continue-as-new returns a control-flow error")
21065 });
21066
21067 let commands = worker
21068 .execute_workflow_task(workflow_task(
21069 "rust.typed-continue",
21070 Vec::new(),
21071 DEFAULT_CODEC,
21072 ))
21073 .expect("typed continue-as-new command");
21074
21075 assert_eq!(commands[0]["type"], "continue_as_new");
21076 assert_eq!(
21077 decode_wire_avro_value(&commands[0]["arguments"], DEFAULT_CODEC)
21078 .expect("typed continue arguments"),
21079 AvroValue::Array(vec![typed_fidelity_probe()])
21080 );
21081 }
21082
21083 #[test]
21084 fn recorded_continue_as_new_is_consumed_without_duplicate_successor_command() {
21085 let client = Client::new("http://127.0.0.1:8080").expect("client");
21086 let mut worker = Worker::new(client, "rust-workers");
21087 worker.register_workflow("rust.continue", |ctx, _input| async move {
21088 ctx.continue_as_new(json!([2]))
21089 });
21090 let task = workflow_task(
21091 "rust.continue",
21092 vec![history_event(
21093 "WorkflowContinuedAsNew",
21094 json!({"sequence": 1, "continued_to_run_id": "run-next"}),
21095 )],
21096 DEFAULT_CODEC,
21097 );
21098
21099 for _worker_restart_or_redelivery in 0..2 {
21100 let commands = worker
21101 .execute_workflow_task(task.clone())
21102 .expect("recorded transition replays");
21103 assert!(
21104 commands.is_empty(),
21105 "replay must not emit another successor"
21106 );
21107 }
21108 }
21109
21110 #[test]
21111 fn continue_as_new_rejects_invalid_overrides_before_emitting_a_command() {
21112 let ctx = workflow_context(Vec::new());
21113 let error = ctx
21114 .continue_as_new_with_options(ContinueAsNewOptions::new().task_queue(" "), json!([1]))
21115 .expect_err("blank queue must be rejected");
21116
21117 let Error::InvalidContinueAsNewOptions(error) = error else {
21118 panic!("expected typed continue-as-new validation error");
21119 };
21120 assert_eq!(error.field, "task_queue");
21121 assert!(ctx.take_commands().expect("commands").is_empty());
21122 }
21123
21124 #[test]
21125 fn workflow_context_exposes_server_history_budget() {
21126 let client = Client::new("http://127.0.0.1:8080").expect("client");
21127 let mut worker = Worker::new(client, "rust-workers");
21128 worker.register_workflow("rust.history-budget", |ctx, _input| async move {
21129 let budget = ctx.history_budget()?;
21130 Ok(json!({
21131 "events": budget.event_count,
21132 "bytes": budget.size_bytes,
21133 "recommended": budget.continue_as_new_recommended,
21134 "pressure": budget.pressure,
21135 }))
21136 });
21137 let task: WorkflowTask = serde_json::from_value(json!({
21138 "task_id": "task-history-budget",
21139 "workflow_type": "rust.history-budget",
21140 "payload_codec": DEFAULT_CODEC,
21141 "history_events": [],
21142 "total_history_events": 480,
21143 "history_size_bytes": 1_048_576,
21144 "continue_as_new_recommended": true,
21145 "history_budget_pressure": "continue_as_new_recommended",
21146 }))
21147 .expect("published workflow task");
21148
21149 let commands = worker
21150 .execute_workflow_task(task)
21151 .expect("history-budget workflow");
21152 let result = decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("result");
21153 assert_eq!(result["events"], 480);
21154 assert_eq!(result["bytes"], 1_048_576);
21155 assert_eq!(result["recommended"], true);
21156 assert_eq!(result["pressure"], "continue_as_new_recommended");
21157 }
21158
21159 #[test]
21160 fn uncaught_workflow_handler_error_emits_terminal_failure_command() {
21161 let client = Client::new("http://127.0.0.1:8080").expect("client");
21162 let mut worker = Worker::new(client, "rust-workers");
21163 worker.register_workflow("rust.failing", |_ctx, _input| async move {
21164 Err(Error::Codec("rust_conformance_failure".to_string()))
21165 });
21166 let task = WorkflowTask {
21167 task_id: "wft-rust-failing-1".to_string(),
21168 workflow_command_id: None,
21169 workflow_id: Some("wf-rust-failing".to_string()),
21170 run_id: Some("run-rust-failing".to_string()),
21171 workflow_type: "rust.failing".to_string(),
21172 cancel_requested: false,
21173 payload_codec: DEFAULT_CODEC.to_string(),
21174 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21175 history_events: Vec::new(),
21176 total_history_events: Some(0),
21177 history_size_bytes: None,
21178 continue_as_new_recommended: None,
21179 history_budget_pressure: None,
21180 next_history_page_token: None,
21181 workflow_task_attempt: 1,
21182 workflow_signal_id: None,
21183 signal_name: None,
21184 signal_arguments: None,
21185 workflow_update_id: None,
21186 update_name: None,
21187 lease_owner: Some("rust-worker".to_string()),
21188 };
21189
21190 let commands = worker
21191 .execute_workflow_task(task)
21192 .expect("handler failure becomes a workflow command");
21193
21194 assert_eq!(commands.len(), 1);
21195 assert_eq!(commands[0]["type"], "fail_workflow");
21196 assert_eq!(commands[0]["exception_type"], "RustWorkflowError");
21197 assert_eq!(commands[0]["exception_class"], "durable_workflow::Error");
21198 assert_eq!(commands[0]["non_retryable"], false);
21199 assert_eq!(
21200 commands[0]["message"],
21201 "codec error: rust_conformance_failure"
21202 );
21203 assert_eq!(
21204 commands[0]["exception"]["message"],
21205 "codec error: rust_conformance_failure"
21206 );
21207 }
21208
21209 #[test]
21210 fn ordinary_handler_error_preserves_commands_queued_in_the_same_decision() {
21211 let client = Client::new("http://127.0.0.1:8080").expect("client");
21212 let mut worker = Worker::new(client, "rust-workers");
21213 worker.register_workflow("rust.failing-after-side-effect", |ctx, _input| async move {
21214 let _: String = ctx.side_effect(|| "captured".to_string())?;
21215 Err(Error::WorkerLoop("application failure".to_string()))
21216 });
21217
21218 let commands = worker
21219 .execute_workflow_task(workflow_task(
21220 "rust.failing-after-side-effect",
21221 Vec::new(),
21222 DEFAULT_CODEC,
21223 ))
21224 .expect("ordinary failure remains a workflow decision");
21225
21226 assert_eq!(commands.len(), 2);
21227 assert_eq!(commands[0]["type"], "record_side_effect");
21228 assert_eq!(commands[1]["type"], "fail_workflow");
21229 }
21230
21231 #[test]
21232 fn uncaught_recorded_activity_failure_claims_only_its_persisted_boundary() {
21233 let client = Client::new("http://127.0.0.1:8080").expect("client");
21234 let mut worker = Worker::new(client, "rust-workers");
21235 worker.register_workflow("rust.redrive", |ctx, _input| async move {
21236 ctx.activity("greet", json!([])).await?;
21237 Ok(Value::Null)
21238 });
21239 worker.register_workflow("rust.redrive-translated", |ctx, _input| async move {
21240 match ctx.activity("greet", json!([])).await {
21241 Err(Error::ActivityFailed(_)) => Err(Error::WorkerLoop("translated".to_string())),
21242 result => result,
21243 }
21244 });
21245
21246 let failed = history_event(
21247 "ActivityFailed",
21248 json!({
21249 "sequence": 1,
21250 "activity_type": "greet",
21251 "activity_execution_id": "activity-1",
21252 "message": "failed"
21253 }),
21254 );
21255 let commands = worker
21256 .execute_workflow_task(workflow_task(
21257 "rust.redrive",
21258 vec![failed.clone()],
21259 DEFAULT_CODEC,
21260 ))
21261 .expect("recorded failure becomes a workflow command");
21262 assert_eq!(commands[0]["type"], "fail_workflow");
21263 assert_eq!(commands[0]["failed_step_sequence"], 1);
21264 assert_eq!(commands[0]["failed_activity_execution_id"], "activity-1");
21265
21266 let translated = worker
21267 .execute_workflow_task(workflow_task(
21268 "rust.redrive-translated",
21269 vec![failed],
21270 DEFAULT_CODEC,
21271 ))
21272 .expect("translated failure becomes a workflow command");
21273 assert_eq!(translated[0]["type"], "fail_workflow");
21274 assert!(translated[0].get("failed_step_sequence").is_none());
21275 assert!(translated[0].get("failed_activity_execution_id").is_none());
21276
21277 let without_identity = worker
21278 .execute_workflow_task(workflow_task(
21279 "rust.redrive",
21280 vec![history_event(
21281 "ActivityFailed",
21282 json!({"sequence": 1, "activity_type": "greet", "message": "failed"}),
21283 )],
21284 DEFAULT_CODEC,
21285 ))
21286 .expect("incomplete identity still fails the workflow");
21287 assert!(without_identity[0].get("failed_step_sequence").is_none());
21288
21289 let timed_out = worker
21290 .execute_workflow_task(workflow_task(
21291 "rust.redrive",
21292 vec![history_event(
21293 "ActivityTimedOut",
21294 json!({
21295 "sequence": 1,
21296 "activity_type": "greet",
21297 "activity_execution_id": "activity-timeout",
21298 }),
21299 )],
21300 DEFAULT_CODEC,
21301 ))
21302 .expect("timeout still fails the workflow");
21303 assert!(timed_out[0].get("failed_step_sequence").is_none());
21304 }
21305
21306 #[test]
21307 fn redriven_history_reuses_completed_prefix_and_reschedules_only_failed_step() {
21308 let client = Client::new("http://127.0.0.1:8080").expect("client");
21309 let mut worker = Worker::new(client, "rust-workers");
21310 worker.register_workflow("rust.redrive-successor", |ctx, _input| async move {
21311 let first = ctx.activity("first", json!([])).await?;
21312 let second = ctx.activity("second", json!([first.clone()])).await?;
21313 Ok(json!({"first": first, "second": second}))
21314 });
21315 let first = history_event(
21316 "ActivityCompleted",
21317 json!({
21318 "sequence": 1,
21319 "activity_type": "first",
21320 "result": encode_value_envelope(&json!("recorded"), DEFAULT_CODEC).expect("first result"),
21321 "payload_codec": DEFAULT_CODEC,
21322 "reused_from_run_id": "failed-run",
21323 "reused_activity_execution_id": "original-first",
21324 }),
21325 );
21326
21327 let retry = worker
21328 .execute_workflow_task(workflow_task(
21329 "rust.redrive-successor",
21330 vec![first.clone()],
21331 DEFAULT_CODEC,
21332 ))
21333 .expect("reused result replays before failed step");
21334 assert_eq!(retry.len(), 1);
21335 assert_eq!(retry[0]["type"], "schedule_activity");
21336 assert_eq!(retry[0]["activity_type"], "second");
21337 assert_eq!(
21338 decode_wire_value(&retry[0]["arguments"], DEFAULT_CODEC).expect("second arguments"),
21339 json!(["recorded"]),
21340 );
21341
21342 let second = history_event(
21343 "ActivityCompleted",
21344 json!({
21345 "sequence": 2,
21346 "activity_type": "second",
21347 "result": encode_value_envelope(&json!("retried"), DEFAULT_CODEC).expect("second result"),
21348 "payload_codec": DEFAULT_CODEC,
21349 }),
21350 );
21351 let completed = worker
21352 .execute_workflow_task(workflow_task(
21353 "rust.redrive-successor",
21354 vec![first, second],
21355 DEFAULT_CODEC,
21356 ))
21357 .expect("retried step completes the successor");
21358 assert_eq!(completed.len(), 1);
21359 assert_eq!(completed[0]["type"], "complete_workflow");
21360 assert_eq!(
21361 decode_wire_value(&completed[0]["result"], DEFAULT_CODEC).expect("workflow result"),
21362 json!({"first": "recorded", "second": "retried"}),
21363 );
21364 }
21365
21366 #[test]
21367 fn handler_error_cannot_hide_an_unconsumed_committed_side_effect() {
21368 let client = Client::new("http://127.0.0.1:8080").expect("client");
21369 let mut worker = Worker::new(client, "rust-workers");
21370 worker.register_workflow("rust.removed-side-effect", |_ctx, _input| async move {
21371 Err(Error::WorkerLoop("application failure".to_string()))
21372 });
21373 let result =
21374 encode_value_envelope(&json!("committed"), DEFAULT_CODEC).expect("side-effect result");
21375
21376 let error = worker
21377 .execute_workflow_task(workflow_task(
21378 "rust.removed-side-effect",
21379 vec![history_event(
21380 "SideEffectRecorded",
21381 json!({"sequence": 1, "result": result}),
21382 )],
21383 DEFAULT_CODEC,
21384 ))
21385 .expect_err("removed committed history must not become fail_workflow");
21386
21387 let Error::NonDeterministicReplay(failure) = error else {
21388 panic!("expected typed replay failure");
21389 };
21390 assert_eq!(failure.reason, "recorded_commands_unconsumed");
21391 assert_eq!(failure.sequence, Some(1));
21392 assert_eq!(failure.expected.as_deref(), Some("side effect"));
21393 }
21394
21395 #[test]
21396 fn replay_error_discards_side_effect_queued_before_incompatible_marker_check() {
21397 let client = Client::new("http://127.0.0.1:8080").expect("client");
21398 let mut worker = Worker::new(client, "rust-workers");
21399 worker.register_workflow(
21400 "rust.side-effect-before-marker-error",
21401 |ctx, _input| async move {
21402 assert_eq!(ctx.get_version("restart-safe", 1, 1)?, 1);
21403 let _: String = ctx.side_effect(|| "must-not-commit".to_string())?;
21404 ctx.get_version("restart-safe", 2, 2)?;
21405 Ok(Value::Null)
21406 },
21407 );
21408
21409 let error = worker
21410 .execute_workflow_task(workflow_task(
21411 "rust.side-effect-before-marker-error",
21412 vec![history_event(
21413 "VersionMarkerRecorded",
21414 json!({
21415 "sequence": 1,
21416 "change_id": "restart-safe",
21417 "version": 1,
21418 "min_supported": 1,
21419 "max_supported": 1,
21420 }),
21421 )],
21422 DEFAULT_CODEC,
21423 ))
21424 .expect_err("replay error must return no queued workflow commands");
21425
21426 let Error::NonDeterministicReplay(failure) = error else {
21427 panic!("expected typed replay failure");
21428 };
21429 assert_eq!(failure.reason, "version_marker_incompatible_range");
21430 assert_eq!(failure.sequence, Some(1));
21431 }
21432
21433 #[test]
21434 fn workflow_task_replay_keeps_recorded_unfired_timer_pending_without_rescheduling() {
21435 let client = Client::new("http://127.0.0.1:8080").expect("client");
21436 let mut worker = Worker::new(client, "rust-workers");
21437 worker.register_workflow("rust.timer.pending", |ctx, _input| async move {
21438 ctx.sleep(Duration::from_secs(5)).await?;
21439 Ok(json!({"status": "timer fired"}))
21440 });
21441
21442 let task = WorkflowTask {
21443 task_id: "wft-rust-timer-pending".to_string(),
21444 workflow_command_id: None,
21445 workflow_id: Some("wf-rust-timer".to_string()),
21446 run_id: Some("run-rust-timer".to_string()),
21447 workflow_type: "rust.timer.pending".to_string(),
21448 cancel_requested: false,
21449 payload_codec: DEFAULT_CODEC.to_string(),
21450 arguments: Some(
21451 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
21452 ),
21453 history_events: vec![history_event(
21454 "TimerScheduled",
21455 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
21456 )],
21457 total_history_events: Some(1),
21458 history_size_bytes: None,
21459 continue_as_new_recommended: None,
21460 history_budget_pressure: None,
21461 next_history_page_token: None,
21462 workflow_task_attempt: 1,
21463 workflow_signal_id: None,
21464 signal_name: None,
21465 signal_arguments: None,
21466 workflow_update_id: None,
21467 update_name: None,
21468 lease_owner: Some("rust-worker".to_string()),
21469 };
21470
21471 for _redelivery_or_restart in 0..2 {
21472 let commands = worker
21473 .execute_workflow_task(task.clone())
21474 .expect("recorded timer remains pending");
21475 assert!(
21476 commands.is_empty(),
21477 "recorded timer must not be rescheduled"
21478 );
21479 }
21480 }
21481
21482 #[test]
21483 fn workflow_task_rejects_recorded_command_removed_from_workflow_code() {
21484 let client = Client::new("http://127.0.0.1:8080").expect("client");
21485 let mut worker = Worker::new(client, "rust-workers");
21486 worker.register_workflow("rust.timer.removed", |_ctx, _input| async move {
21487 Ok(json!({"status": "completed"}))
21488 });
21489 let task = WorkflowTask {
21490 task_id: "wft-rust-timer-removed".to_string(),
21491 workflow_command_id: None,
21492 workflow_id: Some("wf-rust-timer".to_string()),
21493 run_id: Some("run-rust-timer".to_string()),
21494 workflow_type: "rust.timer.removed".to_string(),
21495 cancel_requested: false,
21496 payload_codec: DEFAULT_CODEC.to_string(),
21497 arguments: Some(
21498 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
21499 ),
21500 history_events: vec![
21501 history_event(
21502 "TimerScheduled",
21503 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
21504 ),
21505 history_event(
21506 "TimerFired",
21507 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
21508 ),
21509 ],
21510 total_history_events: Some(2),
21511 history_size_bytes: None,
21512 continue_as_new_recommended: None,
21513 history_budget_pressure: None,
21514 next_history_page_token: None,
21515 workflow_task_attempt: 1,
21516 workflow_signal_id: None,
21517 signal_name: None,
21518 signal_arguments: None,
21519 workflow_update_id: None,
21520 update_name: None,
21521 lease_owner: Some("rust-worker".to_string()),
21522 };
21523
21524 let Error::NonDeterministicReplay(failure) = worker
21525 .execute_workflow_task(task)
21526 .expect_err("removed timer must fail replay")
21527 else {
21528 panic!("expected typed replay failure");
21529 };
21530 assert_eq!(failure.reason, "recorded_commands_unconsumed");
21531 assert_eq!(failure.sequence, Some(1));
21532 }
21533
21534 #[test]
21535 fn workflow_context_emits_explicit_child_workflow_contract() {
21536 let ctx = WorkflowContext {
21537 state: Arc::new(Mutex::new(
21538 WorkflowState::new_with_identity(
21539 Vec::new(),
21540 Some("wf-parent".to_string()),
21541 Some("run-parent".to_string()),
21542 "parent-workers".to_string(),
21543 DEFAULT_CODEC.to_string(),
21544 None,
21545 )
21546 .expect("workflow state"),
21547 )),
21548 };
21549 let options = ChildWorkflowOptions::new("python-workers")
21550 .parent_close_policy(ParentClosePolicy::RequestCancel)
21551 .retry_policy(ChildWorkflowRetryPolicy {
21552 max_attempts: Some(3),
21553 backoff_seconds: vec![1, 5],
21554 non_retryable_error_types: vec!["ValidationError".to_string()],
21555 })
21556 .execution_timeout_seconds(600)
21557 .run_timeout_seconds(120);
21558 let mut call = Box::pin(ctx.start_child_workflow(
21559 "python.fulfil-order",
21560 options,
21561 json!([{"order_id": "order-42"}]),
21562 ));
21563 let mut task_context = TaskContext::from_waker(noop_waker_ref());
21564
21565 assert!(matches!(
21566 call.as_mut().poll(&mut task_context),
21567 Poll::Pending
21568 ));
21569 let commands = ctx.take_commands().expect("commands");
21570 assert_eq!(commands.len(), 1);
21571 let command = &commands[0];
21572 assert_eq!(command["type"], "start_child_workflow");
21573 assert_eq!(command["workflow_type"], "python.fulfil-order");
21574 assert_eq!(command["queue"], "python-workers");
21575 assert_eq!(command["parent_close_policy"], "request_cancel");
21576 assert_eq!(command["retry_policy"]["max_attempts"], 3);
21577 assert_eq!(command["execution_timeout_seconds"], 600);
21578 assert_eq!(command["run_timeout_seconds"], 120);
21579 assert_eq!(
21580 decode_wire_value(&command["arguments"], DEFAULT_CODEC).expect("child args"),
21581 json!([{"order_id": "order-42"}])
21582 );
21583 }
21584
21585 fn child_parent_worker() -> Worker {
21586 let client = Client::new("http://127.0.0.1:8080").expect("client");
21587 let mut worker = Worker::new(client, "rust-parent-workers");
21588 worker.register_workflow("rust.parent", |ctx, _input| async move {
21589 let child = ctx
21590 .start_child_workflow(
21591 "python.child",
21592 ChildWorkflowOptions::new("python-child-workers")
21593 .parent_close_policy(ParentClosePolicy::Terminate),
21594 json!([{"codec_probe": [1, true, "rust"]}]),
21595 )
21596 .await?;
21597 Ok(json!({
21598 "parent_workflow_id": child.parent.workflow_id,
21599 "parent_run_id": child.parent.run_id,
21600 "child_workflow_id": child.child.workflow_id,
21601 "child_run_id": child.child.run_id,
21602 "child_workflow_type": child.child_workflow_type,
21603 "result": child.result,
21604 }))
21605 });
21606 worker
21607 }
21608
21609 fn child_parent_task(event_type: &str, payload: Value) -> WorkflowTask {
21610 WorkflowTask {
21611 task_id: "wft-child-parent".to_string(),
21612 workflow_command_id: None,
21613 workflow_id: Some("wf-parent".to_string()),
21614 run_id: Some("run-parent".to_string()),
21615 workflow_type: "rust.parent".to_string(),
21616 cancel_requested: false,
21617 payload_codec: DEFAULT_CODEC.to_string(),
21618 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21619 history_events: vec![
21620 HistoryEvent {
21621 event_type: "ChildWorkflowScheduled".to_string(),
21622 payload: json!({
21623 "sequence": 1,
21624 "child_call_id": "call-child",
21625 "child_workflow_instance_id": "wf-child",
21626 "child_workflow_run_id": "run-child",
21627 "child_workflow_type": "python.child",
21628 }),
21629 raw: HashMap::new(),
21630 },
21631 HistoryEvent {
21632 event_type: event_type.to_string(),
21633 payload,
21634 raw: HashMap::new(),
21635 },
21636 ],
21637 total_history_events: Some(2),
21638 history_size_bytes: None,
21639 continue_as_new_recommended: None,
21640 history_budget_pressure: None,
21641 next_history_page_token: None,
21642 workflow_task_attempt: 1,
21643 workflow_signal_id: None,
21644 signal_name: None,
21645 signal_arguments: None,
21646 workflow_update_id: None,
21647 update_name: None,
21648 lease_owner: Some("rust-worker".to_string()),
21649 }
21650 }
21651
21652 #[test]
21653 fn committed_child_result_replays_without_starting_a_duplicate() {
21654 let worker = child_parent_worker();
21655 let task = child_parent_task(
21656 "ChildRunCompleted",
21657 json!({
21658 "sequence": 1,
21659 "child_call_id": "call-child",
21660 "child_workflow_instance_id": "wf-child",
21661 "child_workflow_run_id": "run-child",
21662 "child_workflow_type": "python.child",
21663 "payload_codec": DEFAULT_CODEC,
21664 "result": fixture_envelope(json!({"from":"python","ok":true})),
21665 }),
21666 );
21667
21668 for _restart in 0..2 {
21669 let commands = worker
21670 .execute_workflow_task(task.clone())
21671 .expect("replayed parent task");
21672 assert_eq!(commands.len(), 1);
21673 assert_eq!(commands[0]["type"], "complete_workflow");
21674 assert!(!commands
21675 .iter()
21676 .any(|command| command["type"] == "start_child_workflow"));
21677 let output =
21678 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
21679 assert_eq!(output["parent_workflow_id"], "wf-parent");
21680 assert_eq!(output["parent_run_id"], "run-parent");
21681 assert_eq!(output["child_workflow_id"], "wf-child");
21682 assert_eq!(output["child_run_id"], "run-child");
21683 assert_eq!(output["result"], json!({"from": "python", "ok": true}));
21684 }
21685 }
21686
21687 #[test]
21688 fn typed_child_arguments_and_results_survive_replay() {
21689 let client = Client::new("http://127.0.0.1:8080").expect("client");
21690 let mut worker = Worker::new(client, "rust-parent-workers");
21691 worker.register_workflow_avro_value("rust.typed-parent", |ctx, _input| async move {
21692 let child = ctx
21693 .start_child_workflow_avro_value(
21694 "python.typed-child",
21695 ChildWorkflowOptions::new("python-workers"),
21696 AvroValue::Array(vec![typed_fidelity_probe()]),
21697 )
21698 .await?;
21699 Ok(child.result)
21700 });
21701
21702 let initial = worker
21703 .execute_workflow_task(workflow_task(
21704 "rust.typed-parent",
21705 Vec::new(),
21706 DEFAULT_CODEC,
21707 ))
21708 .expect("typed child start");
21709 assert_eq!(initial[0]["type"], "start_child_workflow");
21710 assert_eq!(
21711 decode_wire_avro_value(&initial[0]["arguments"], DEFAULT_CODEC)
21712 .expect("typed child arguments"),
21713 AvroValue::Array(vec![typed_fidelity_probe()])
21714 );
21715
21716 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
21717 .expect("typed child result");
21718 let task = workflow_task(
21719 "rust.typed-parent",
21720 vec![
21721 history_event(
21722 "ChildWorkflowScheduled",
21723 json!({
21724 "sequence": 1,
21725 "child_call_id": "call-typed",
21726 "child_workflow_instance_id": "wf-child",
21727 "child_workflow_run_id": "run-child",
21728 "child_workflow_type": "python.typed-child",
21729 }),
21730 ),
21731 history_event(
21732 "ChildRunCompleted",
21733 json!({
21734 "sequence": 1,
21735 "child_call_id": "call-typed",
21736 "child_workflow_instance_id": "wf-child",
21737 "child_workflow_run_id": "run-child",
21738 "child_workflow_type": "python.typed-child",
21739 "payload_codec": DEFAULT_CODEC,
21740 "result": result,
21741 }),
21742 ),
21743 ],
21744 DEFAULT_CODEC,
21745 );
21746
21747 let commands = worker
21748 .execute_workflow_task(task)
21749 .expect("typed child replay");
21750 assert_eq!(commands[0]["type"], "complete_workflow");
21751 assert_eq!(
21752 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
21753 .expect("typed parent result"),
21754 typed_fidelity_probe()
21755 );
21756 }
21757
21758 #[test]
21759 fn pending_child_replays_after_restart_without_starting_a_duplicate() {
21760 let worker = child_parent_worker();
21761 let mut task = child_parent_task("unused", Value::Null);
21762 task.history_events.truncate(1);
21763 task.total_history_events = Some(1);
21764
21765 for _redelivery_or_restart in 0..2 {
21766 let commands = worker
21767 .execute_workflow_task(task.clone())
21768 .expect("recorded child remains pending");
21769 assert!(
21770 commands.is_empty(),
21771 "recorded pending child must not be started again"
21772 );
21773 }
21774 }
21775
21776 #[test]
21777 fn child_cancellation_becomes_stable_parent_failure_command() {
21778 let worker = child_parent_worker();
21779 let task = child_parent_task(
21780 "ChildRunCancelled",
21781 json!({
21782 "sequence": 1,
21783 "child_workflow_instance_id": "wf-child",
21784 "child_workflow_run_id": "run-child",
21785 "child_workflow_type": "python.child",
21786 "failure_id": "failure-child",
21787 "failure_category": "cancelled",
21788 "message": "cancelled by parent-close policy",
21789 }),
21790 );
21791
21792 let commands = worker
21793 .execute_workflow_task(task)
21794 .expect("parent settlement");
21795 assert_eq!(commands.len(), 1);
21796 assert_eq!(commands[0]["type"], "fail_workflow");
21797 assert_eq!(commands[0]["exception_type"], "ChildWorkflowCancelled");
21798 assert_eq!(
21799 commands[0]["exception"]["properties"]["reason"],
21800 "cancelled"
21801 );
21802 assert_eq!(
21803 commands[0]["exception"]["properties"]["child_workflow_run_id"],
21804 "run-child"
21805 );
21806 }
21807
21808 #[test]
21809 fn workflow_can_handle_typed_child_failure() {
21810 let client = Client::new("http://127.0.0.1:8080").expect("client");
21811 let mut worker = Worker::new(client, "rust-parent-workers");
21812 worker.register_workflow("rust.handled-parent", |ctx, _input| async move {
21813 match ctx
21814 .start_child_workflow(
21815 "python.child",
21816 ChildWorkflowOptions::new("python-child-workers"),
21817 json!([]),
21818 )
21819 .await
21820 {
21821 Err(Error::ChildWorkflowFailed(failure)) => Ok(json!({
21822 "reason": failure.reason,
21823 "failure_id": failure.failure_id,
21824 "exception_class": failure.exception_class,
21825 "child_run_id": failure.child_workflow_run_id,
21826 })),
21827 Err(error) => Err(error),
21828 Ok(_) => Err(Error::WorkerLoop(
21829 "child unexpectedly succeeded".to_string(),
21830 )),
21831 }
21832 });
21833 let mut task = child_parent_task(
21834 "ChildRunFailed",
21835 json!({
21836 "sequence": 1,
21837 "child_workflow_instance_id": "wf-child",
21838 "child_workflow_run_id": "run-child",
21839 "child_workflow_type": "python.child",
21840 "failure_id": "failure-child",
21841 "failure_category": "child_workflow",
21842 "message": "payment rejected",
21843 "exception": {
21844 "type": "PaymentRejected",
21845 "class": "payments.PaymentRejected",
21846 "message": "payment rejected"
21847 }
21848 }),
21849 );
21850 task.workflow_type = "rust.handled-parent".to_string();
21851
21852 let commands = worker.execute_workflow_task(task).expect("handled failure");
21853 assert_eq!(commands[0]["type"], "complete_workflow");
21854 let output =
21855 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
21856 assert_eq!(output["reason"], "child_workflow");
21857 assert_eq!(output["failure_id"], "failure-child");
21858 assert_eq!(output["exception_class"], "payments.PaymentRejected");
21859 assert_eq!(output["child_run_id"], "run-child");
21860 }
21861
21862 #[test]
21863 fn rust_hello_world_uses_signal_arguments_from_resume_payload() {
21864 let client = Client::new("http://127.0.0.1:8080").expect("client");
21865 let mut worker = Worker::new(client, "rust-workers");
21866
21867 worker.register_workflow("rust.hello_workflow", |ctx, _input| async move {
21868 let signal = ctx.wait_signal("start").await?;
21869 let name = signal
21870 .first()
21871 .and_then(|value| value.as_str())
21872 .unwrap_or("world");
21873 let greeting = ctx.activity("rust.hello_activity", json!([name])).await?;
21874 Ok(json!({
21875 "greeting": greeting,
21876 "language": "rust"
21877 }))
21878 });
21879
21880 let signal_arguments =
21881 encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC).expect("signal arguments");
21882 let task = WorkflowTask {
21883 task_id: "wft-rust-signal-1".to_string(),
21884 workflow_command_id: None,
21885 workflow_id: Some("wf-rust-hello".to_string()),
21886 run_id: Some("run-rust-hello".to_string()),
21887 workflow_type: "rust.hello_workflow".to_string(),
21888 cancel_requested: false,
21889 payload_codec: DEFAULT_CODEC.to_string(),
21890 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21891 history_events: vec![HistoryEvent {
21892 event_type: "SignalReceived".to_string(),
21893 payload: json!({
21894 "signal_id": "sig-rust-1",
21895 "signal_name": "start"
21896 }),
21897 raw: HashMap::new(),
21898 }],
21899 total_history_events: Some(1),
21900 history_size_bytes: None,
21901 continue_as_new_recommended: None,
21902 history_budget_pressure: None,
21903 next_history_page_token: None,
21904 workflow_task_attempt: 1,
21905 workflow_signal_id: Some("sig-rust-1".to_string()),
21906 signal_name: Some("start".to_string()),
21907 signal_arguments: Some(signal_arguments),
21908 workflow_update_id: None,
21909 update_name: None,
21910 lease_owner: Some("rust-worker".to_string()),
21911 };
21912
21913 let commands = worker.execute_workflow_task(task).expect("workflow task");
21914
21915 assert_eq!(commands.len(), 1);
21916 assert_eq!(commands[0]["type"], "schedule_activity");
21917 assert_eq!(commands[0]["activity_type"], "rust.hello_activity");
21918 assert_eq!(
21919 decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC).expect("activity args"),
21920 json!(["Rust"])
21921 );
21922 }
21923
21924 #[test]
21925 fn workflow_task_appends_paginated_history_events() {
21926 let mut task = WorkflowTask {
21927 task_id: "wft-rust-pages-1".to_string(),
21928 workflow_command_id: None,
21929 workflow_id: Some("wf-rust-pages".to_string()),
21930 run_id: Some("run-rust-pages".to_string()),
21931 workflow_type: "rust.hello_workflow".to_string(),
21932 cancel_requested: false,
21933 payload_codec: DEFAULT_CODEC.to_string(),
21934 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21935 history_events: vec![HistoryEvent {
21936 event_type: "WorkflowStarted".to_string(),
21937 payload: json!({}),
21938 raw: HashMap::new(),
21939 }],
21940 total_history_events: Some(3),
21941 history_size_bytes: None,
21942 continue_as_new_recommended: None,
21943 history_budget_pressure: None,
21944 next_history_page_token: Some("MQ==".to_string()),
21945 workflow_task_attempt: 1,
21946 workflow_signal_id: None,
21947 signal_name: None,
21948 signal_arguments: None,
21949 workflow_update_id: None,
21950 update_name: None,
21951 lease_owner: Some("rust-worker".to_string()),
21952 };
21953
21954 task.append_history_page(WorkflowTaskHistoryPage {
21955 history_events: vec![
21956 HistoryEvent {
21957 event_type: "SignalReceived".to_string(),
21958 payload: json!({
21959 "signal_id": "sig-rust-1",
21960 "signal_name": "start",
21961 "arguments": encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC)
21962 .expect("signal arguments")
21963 }),
21964 raw: HashMap::new(),
21965 },
21966 HistoryEvent {
21967 event_type: "MarkerRecorded".to_string(),
21968 payload: json!({"sequence": 3}),
21969 raw: HashMap::new(),
21970 },
21971 ],
21972 total_history_events: Some(3),
21973 next_history_page_token: None,
21974 });
21975
21976 assert_eq!(task.history_events.len(), 3);
21977 assert_eq!(task.total_history_events, Some(3));
21978 assert_eq!(task.next_history_page_token, None);
21979
21980 let signal = task
21981 .history_events
21982 .iter()
21983 .find(|event| event.event_type == "SignalReceived")
21984 .expect("signal event");
21985 assert_eq!(
21986 decode_signal_event_arguments(signal, DEFAULT_CODEC).expect("signal arguments"),
21987 vec![AvroValue::String("Rust".to_string())]
21988 );
21989 }
21990
21991 #[tokio::test]
21992 async fn query_handler_reads_ordered_cross_codec_signals_without_commands() {
21993 let client = Client::new("http://127.0.0.1:8080").expect("client");
21994 let mut worker = Worker::new(client, "rust-workers");
21995 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21996 worker.register_query("counter", "current", |ctx, _args| async move {
21997 let mut count = 0_i64;
21998 for signal in ctx.signal_events() {
21999 let value = signal
22000 .arguments
22001 .first()
22002 .and_then(Value::as_i64)
22003 .unwrap_or_default();
22004 match signal.name.as_str() {
22005 "increment" => count += value,
22006 "set" => count = value,
22007 _ => {}
22008 }
22009 }
22010 Ok(json!(count))
22011 });
22012
22013 let task = QueryTask {
22014 query_task_id: "query-rust-counter".to_string(),
22015 query_task_attempt: 1,
22016 lease_owner: Some("rust-worker".to_string()),
22017 workflow_id: Some("counter-1".to_string()),
22018 run_id: Some("run-counter-1".to_string()),
22019 workflow_type: "counter".to_string(),
22020 query_name: "current".to_string(),
22021 payload_codec: DEFAULT_CODEC.to_string(),
22022 workflow_arguments: Some(
22023 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
22024 ),
22025 query_arguments: Some(
22026 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("query arguments"),
22027 ),
22028 history_events: vec![
22029 HistoryEvent {
22030 event_type: "SignalReceived".to_string(),
22031 payload: json!({
22032 "signal_id": "php-signal-1",
22033 "signal_name": "increment",
22034 "workflow_sequence": 1,
22035 "payload_codec": DEFAULT_CODEC,
22036 "arguments": encode_value_envelope(&json!([3]), DEFAULT_CODEC).expect("php avro signal")
22037 }),
22038 raw: HashMap::new(),
22039 },
22040 HistoryEvent {
22041 event_type: "SignalReceived".to_string(),
22042 payload: json!({
22043 "signal_id": "python-signal-2",
22044 "signal_name": "increment",
22045 "workflow_sequence": 2,
22046 "payload_codec": DEFAULT_CODEC,
22047 "arguments": encode_value_envelope(&json!([5]), DEFAULT_CODEC).expect("python avro signal")
22048 }),
22049 raw: HashMap::new(),
22050 },
22051 HistoryEvent {
22052 event_type: "SignalReceived".to_string(),
22053 payload: json!({
22054 "signal_id": "rust-signal-3",
22055 "signal_name": "set",
22056 "workflow_sequence": 3,
22057 "payload_codec": DEFAULT_CODEC,
22058 "arguments": encode_value_envelope(&json!([0]), DEFAULT_CODEC).expect("rust avro signal")
22059 }),
22060 raw: HashMap::new(),
22061 },
22062 ],
22063 history_export: None,
22064 run_status: Some("completed".to_string()),
22065 };
22066
22067 let result = worker.execute_query_task(task).await.expect("query result");
22068 assert_eq!(result.into_json().expect("query projection"), json!(0));
22069 }
22070
22071 #[tokio::test]
22072 async fn replayed_queries_read_running_completed_and_cold_restarted_instance_state() {
22073 let worker = replay_counter_worker();
22074 let running_history = json!([
22075 {
22076 "type": "ActivityCompleted",
22077 "payload": {
22078 "sequence": 1,
22079 "activity_type": "load-counter",
22080 "payload_codec": DEFAULT_CODEC,
22081 "result": fixture_envelope(json!("loaded"))
22082 }
22083 },
22084 {
22085 "type": "SignalWaitOpened",
22086 "payload": {
22087 "sequence": 3,
22088 "signal_name": "increment"
22089 }
22090 },
22091 {
22092 "type": "SignalReceived",
22093 "payload": {
22094 "signal_id": "signal-3",
22095 "signal_name": "increment",
22096 "workflow_sequence": 2,
22097 "payload_codec": DEFAULT_CODEC,
22098 "arguments": fixture_envelope(json!([3]))
22099 }
22100 },
22101 {
22102 "type": "SignalApplied",
22103 "payload": {
22104 "sequence": 3,
22105 "signal_id": "signal-3",
22106 "signal_name": "increment",
22107 "payload_codec": DEFAULT_CODEC,
22108 "value": fixture_envelope(json!([3]))
22109 }
22110 }
22111 ]);
22112
22113 let running = worker
22114 .execute_query_task(replay_counter_query(
22115 "current",
22116 running_history.clone(),
22117 "running",
22118 ))
22119 .await
22120 .expect("running replay query");
22121 assert_eq!(
22122 running.clone().into_json().expect("query projection"),
22123 json!({"loaded": "loaded", "count": 3, "finished": false})
22124 );
22125
22126 let detached = worker
22127 .execute_query_task(replay_counter_query(
22128 "detached-mutation",
22129 running_history.clone(),
22130 "running",
22131 ))
22132 .await
22133 .expect("query mutates only its detached state clone");
22134 assert_eq!(detached.into_json().expect("query projection"), json!(999));
22135 let failed = worker
22136 .execute_query_task(replay_counter_query(
22137 "failed-mutation",
22138 running_history.clone(),
22139 "running",
22140 ))
22141 .await
22142 .expect_err("failed query");
22143 assert_eq!(failed.reason, "query_rejected");
22144 let unchanged = worker
22145 .execute_query_task(replay_counter_query("current", running_history, "running"))
22146 .await
22147 .expect("later query reconstructs unchanged state");
22148 assert_eq!(unchanged, running);
22149
22150 let restarted_worker = replay_counter_worker();
22151 let empty_arguments = fixture_envelope(json!([]));
22152 let loaded_result = fixture_envelope(json!("loaded"));
22153 let signal_three = fixture_blob(json!([3]));
22154 let signal_five = fixture_blob(json!([5]));
22155 let restarted_task: QueryTask = serde_json::from_value(json!({
22156 "query_task_id": "query-after-restart",
22157 "workflow_id": "counter-1",
22158 "run_id": "run-counter-1",
22159 "workflow_type": "replay-counter",
22160 "query_name": "current",
22161 "payload_codec": DEFAULT_CODEC,
22162 "workflow_arguments": empty_arguments.clone(),
22163 "query_arguments": empty_arguments,
22164 "history_events": [],
22165 "history_export": {
22166 "payloads": {"codec": DEFAULT_CODEC},
22167 "history_events": [
22168 {
22169 "type": "ActivityCompleted",
22170 "payload": {
22171 "sequence": 1,
22172 "activity_type": "load-counter",
22173 "payload_codec": DEFAULT_CODEC,
22174 "result": null
22175 }
22176 },
22177 {
22178 "type": "SignalWaitOpened",
22179 "payload": {
22180 "sequence": 3,
22181 "signal_name": "increment"
22182 }
22183 },
22184 {
22185 "type": "SignalReceived",
22186 "payload": {
22187 "signal_id": "signal-3",
22188 "signal_name": "increment",
22189 "workflow_sequence": 2
22190 }
22191 },
22192 {
22193 "type": "SignalApplied",
22194 "payload": {
22195 "sequence": 3,
22196 "signal_id": "signal-3",
22197 "signal_name": "increment"
22198 }
22199 },
22200 {
22201 "type": "SignalWaitOpened",
22202 "payload": {
22203 "sequence": 5,
22204 "signal_name": "increment"
22205 }
22206 },
22207 {
22208 "type": "SignalReceived",
22209 "payload": {
22210 "signal_id": "signal-5",
22211 "signal_name": "increment",
22212 "workflow_sequence": 4
22213 }
22214 },
22215 {
22216 "type": "SignalApplied",
22217 "payload": {
22218 "sequence": 5,
22219 "signal_id": "signal-5",
22220 "signal_name": "increment"
22221 }
22222 }
22223 ],
22224 "activities": [{
22225 "sequence": 1,
22226 "activity_type": "load-counter",
22227 "payload_codec": DEFAULT_CODEC,
22228 "result": loaded_result
22229 }],
22230 "signals": [
22231 {
22232 "id": "signal-3",
22233 "name": "increment",
22234 "workflow_sequence": 2,
22235 "payload_codec": DEFAULT_CODEC,
22236 "arguments": signal_three
22237 },
22238 {
22239 "id": "signal-5",
22240 "name": "increment",
22241 "workflow_sequence": 4,
22242 "payload_codec": DEFAULT_CODEC,
22243 "arguments": signal_five
22244 }
22245 ]
22246 },
22247 "run_status": "completed"
22248 }))
22249 .expect("cold replay query task");
22250 let completed = restarted_worker
22251 .execute_query_task(restarted_task)
22252 .await
22253 .expect("completed cold replay query");
22254 assert_eq!(
22255 completed.into_json().expect("query projection"),
22256 json!({"loaded": "loaded", "count": 8, "finished": true})
22257 );
22258 }
22259
22260 #[tokio::test]
22261 async fn replayed_query_replay_failures_are_machine_readable() {
22262 let worker = replay_counter_worker();
22263 let task = replay_counter_query(
22264 "current",
22265 json!([{
22266 "type": "ActivityCompleted",
22267 "payload": {
22268 "sequence": 1,
22269 "payload_codec": DEFAULT_CODEC,
22270 "result": {"codec": DEFAULT_CODEC, "blob": "{"}
22271 }
22272 }]),
22273 "running",
22274 );
22275 let failure = worker
22276 .execute_query_task(task)
22277 .await
22278 .expect_err("invalid replay history payload");
22279 assert_eq!(failure.reason, "query_payload_decode_failed");
22280 assert_eq!(failure.failure_type, "QueryPayloadDecodeFailed");
22281 assert!(failure.message.contains("invalid_payload_framing"));
22282 }
22283
22284 #[tokio::test]
22285 async fn query_task_restores_compact_history_from_export() {
22286 let client = Client::new("http://127.0.0.1:8080").expect("client");
22287 let mut worker = Worker::new(client, "rust-workers");
22288 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
22289 worker.register_query("counter", "current", |ctx, _args| async move {
22290 Ok(json!(ctx.signals("increment")[0][0]))
22291 });
22292 let empty_arguments = fixture_envelope(json!([]));
22293 let exported_signal = fixture_blob(json!([9]));
22294 let task: QueryTask = serde_json::from_value(json!({
22295 "query_task_id": "query-export",
22296 "workflow_type": "counter",
22297 "query_name": "current",
22298 "payload_codec": DEFAULT_CODEC,
22299 "workflow_arguments": empty_arguments.clone(),
22300 "query_arguments": empty_arguments,
22301 "history_events": [],
22302 "history_export": {
22303 "payloads": {"codec": DEFAULT_CODEC},
22304 "history_events": [{
22305 "type": "SignalReceived",
22306 "payload": {"signal_id": "signal-export", "signal_name": "increment"}
22307 }],
22308 "signals": [{
22309 "id": "signal-export",
22310 "name": "increment",
22311 "status": "applied",
22312 "workflow_sequence": 1,
22313 "payload_codec": DEFAULT_CODEC,
22314 "arguments": exported_signal
22315 }]
22316 }
22317 }))
22318 .expect("query task");
22319
22320 let result = worker.execute_query_task(task).await.expect("query result");
22321 assert_eq!(result.into_json().expect("query projection"), json!(9));
22322 }
22323
22324 #[tokio::test]
22325 async fn query_task_failures_have_stable_reasons() {
22326 let client = Client::new("http://127.0.0.1:8080").expect("client");
22327 let mut worker = Worker::new(client, "rust-workers");
22328 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
22329 worker.register_query(
22330 "counter",
22331 "current",
22332 |_ctx, _args| async move { Ok(json!(0)) },
22333 );
22334
22335 let base_task = QueryTask {
22336 query_task_id: "query-errors".to_string(),
22337 query_task_attempt: 1,
22338 lease_owner: None,
22339 workflow_id: Some("counter-errors".to_string()),
22340 run_id: Some("run-errors".to_string()),
22341 workflow_type: "counter".to_string(),
22342 query_name: "missing".to_string(),
22343 payload_codec: DEFAULT_CODEC.to_string(),
22344 workflow_arguments: Some(fixture_envelope(json!([]))),
22345 query_arguments: Some(fixture_envelope(json!([]))),
22346 history_events: Vec::new(),
22347 history_export: None,
22348 run_status: Some("running".to_string()),
22349 };
22350
22351 let unknown = worker
22352 .execute_query_task(base_task.clone())
22353 .await
22354 .expect_err("unknown query");
22355 assert_eq!(unknown.reason, "rejected_unknown_query");
22356
22357 let mut malformed = base_task;
22358 malformed.query_name = "current".to_string();
22359 malformed.query_arguments = Some(json!({"codec": DEFAULT_CODEC, "blob": "{"}));
22360 let malformed = worker
22361 .execute_query_task(malformed)
22362 .await
22363 .expect_err("malformed payload");
22364 assert_eq!(malformed.reason, "query_payload_decode_failed");
22365
22366 let client = Client::new("http://127.0.0.1:8080").expect("client");
22367 let mut unavailable_worker = Worker::new(client, "rust-workers");
22368 unavailable_worker
22369 .register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
22370 let empty_arguments = fixture_envelope(json!([]));
22371 let unavailable_task: QueryTask = serde_json::from_value(json!({
22372 "query_task_id": "query-unavailable",
22373 "workflow_type": "counter",
22374 "query_name": "current",
22375 "payload_codec": DEFAULT_CODEC,
22376 "workflow_arguments": empty_arguments.clone(),
22377 "query_arguments": empty_arguments
22378 }))
22379 .expect("query task");
22380 let unavailable = unavailable_worker
22381 .execute_query_task(unavailable_task)
22382 .await
22383 .expect_err("query handler unavailable");
22384 assert_eq!(unavailable.reason, "query_handler_unavailable");
22385 }
22386
22387 #[tokio::test]
22388 async fn client_query_decodes_result_and_typed_failure() {
22389 let server = MockWorkerServer::start();
22390 let client = Client::builder(server.base_url())
22391 .timeout(Duration::from_secs(2))
22392 .build()
22393 .expect("client");
22394
22395 let result = client
22396 .query_workflow("counter-1", "current", json!([]))
22397 .await
22398 .expect("query result");
22399 assert_eq!(result, json!({"count": 8}));
22400
22401 let error = client
22402 .query_workflow("counter-1", "missing", json!([]))
22403 .await
22404 .expect_err("unknown query");
22405 let Error::QueryFailed(failure) = error else {
22406 panic!("expected typed query failure");
22407 };
22408 assert_eq!(failure.status, 404);
22409 assert_eq!(failure.reason, "rejected_unknown_query");
22410 }
22411
22412 #[tokio::test]
22413 async fn public_client_surfaces_send_and_receive_lossless_avro_values() {
22414 let server = MockWorkerServer::start();
22415 let client = Client::builder(server.base_url())
22416 .timeout(Duration::from_secs(2))
22417 .build()
22418 .expect("client");
22419 let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
22420
22421 client
22422 .start_workflow(
22423 "typed.echo",
22424 "rust-workers",
22425 "typed-start",
22426 arguments.clone(),
22427 )
22428 .await
22429 .expect("typed workflow start");
22430 assert_eq!(
22431 decode_wire_avro_value(
22432 &server.request_body("/api/workflows")["input"],
22433 DEFAULT_CODEC,
22434 )
22435 .expect("typed start input"),
22436 arguments
22437 );
22438
22439 client
22440 .signal_workflow("typed-1", "changed", arguments.clone())
22441 .await
22442 .expect("typed signal");
22443 assert_eq!(
22444 decode_wire_avro_value(
22445 &server.request_body("/api/workflows/typed-1/signal/changed")["input"],
22446 DEFAULT_CODEC,
22447 )
22448 .expect("typed signal input"),
22449 arguments
22450 );
22451
22452 assert_eq!(
22453 client
22454 .query_workflow_avro_value("typed-1", "inspect", arguments.clone())
22455 .await
22456 .expect("typed query"),
22457 typed_fidelity_probe()
22458 );
22459 assert_eq!(
22460 decode_wire_avro_value(
22461 &server.request_body("/api/workflows/typed-1/query/inspect")["input"],
22462 DEFAULT_CODEC,
22463 )
22464 .expect("typed query input"),
22465 arguments
22466 );
22467
22468 assert_eq!(
22469 client
22470 .update_workflow_avro_value(
22471 "typed-1",
22472 "replace",
22473 arguments.clone(),
22474 Some("typed-request"),
22475 )
22476 .await
22477 .expect("typed update"),
22478 typed_fidelity_probe()
22479 );
22480 let update = server.request_body("/api/workflows/typed-1/update/replace");
22481 assert_eq!(update["request_id"], "typed-request");
22482 assert_eq!(
22483 decode_wire_avro_value(&update["input"], DEFAULT_CODEC).expect("typed update input"),
22484 arguments
22485 );
22486
22487 let handle = WorkflowHandle {
22488 client: client.clone(),
22489 workflow_id: "typed-1".to_string(),
22490 run_id: Some("run-typed-1".to_string()),
22491 workflow_type: "typed.echo".to_string(),
22492 };
22493 assert_eq!(
22494 handle
22495 .result_avro_value(WorkflowResultOptions::default())
22496 .await
22497 .expect("typed workflow result"),
22498 typed_fidelity_probe()
22499 );
22500
22501 client
22502 .complete_activity_task(
22503 "activity-typed",
22504 "attempt-typed",
22505 "rust-worker",
22506 typed_fidelity_probe(),
22507 DEFAULT_CODEC,
22508 )
22509 .await
22510 .expect("typed activity completion");
22511 assert_eq!(
22512 decode_wire_avro_value(
22513 &server.request_body("/api/worker/activity-tasks/activity-typed/complete")
22514 ["result"],
22515 DEFAULT_CODEC,
22516 )
22517 .expect("typed activity result"),
22518 typed_fidelity_probe()
22519 );
22520 client
22521 .fail_activity_task(
22522 "activity-typed",
22523 "attempt-typed",
22524 "rust-worker",
22525 "typed failure",
22526 true,
22527 )
22528 .await
22529 .expect("activity failure");
22530 }
22531
22532 #[tokio::test]
22533 async fn lifecycle_commands_support_instance_and_selected_run_targets() {
22534 let server = MockWorkerServer::start();
22535 let client = Client::builder(server.base_url())
22536 .timeout(Duration::from_secs(2))
22537 .build()
22538 .expect("client");
22539
22540 let options = WorkflowCommandOptions::new()
22541 .reason("cleanup requested")
22542 .request_id("cancel-17");
22543 let cancelled = client
22544 .cancel_workflow("wf-lifecycle", options)
22545 .await
22546 .expect("instance cancellation");
22547 assert_eq!(cancelled.command, WorkflowCommandKind::Cancel);
22548 assert_eq!(cancelled.run_id.as_deref(), Some("run-current"));
22549 assert_eq!(cancelled.outcome.as_deref(), Some("cancelled"));
22550 assert_eq!(
22551 server.request_body("/api/workflows/wf-lifecycle/cancel"),
22552 json!({"reason":"cleanup requested","request_id":"cancel-17"})
22553 );
22554
22555 let terminated = client
22556 .terminate_workflow(
22557 "wf-lifecycle",
22558 WorkflowCommandOptions::new().reason("forced stop"),
22559 )
22560 .await
22561 .expect("instance termination");
22562 assert_eq!(terminated.command, WorkflowCommandKind::Terminate);
22563 assert_eq!(terminated.outcome.as_deref(), Some("terminated"));
22564
22565 client
22566 .cancel_workflow_run(
22567 "wf-lifecycle",
22568 "run-current",
22569 WorkflowCommandOptions::default(),
22570 )
22571 .await
22572 .expect("selected run cancellation");
22573 client
22574 .terminate_workflow_run(
22575 "wf-lifecycle",
22576 "run-current",
22577 WorkflowCommandOptions::default(),
22578 )
22579 .await
22580 .expect("selected run termination");
22581
22582 for (command, error) in [
22583 (
22584 WorkflowCommandKind::Cancel,
22585 client
22586 .cancel_workflow_run(
22587 "wf-lifecycle",
22588 "run-stale",
22589 WorkflowCommandOptions::default(),
22590 )
22591 .await
22592 .expect_err("stale cancellation must be rejected"),
22593 ),
22594 (
22595 WorkflowCommandKind::Terminate,
22596 client
22597 .terminate_workflow_run(
22598 "wf-lifecycle",
22599 "run-stale",
22600 WorkflowCommandOptions::default(),
22601 )
22602 .await
22603 .expect_err("stale termination must be rejected"),
22604 ),
22605 ] {
22606 let Error::WorkflowCommandRejected(rejection) = error else {
22607 panic!("expected typed command rejection");
22608 };
22609 assert_eq!(rejection.command, command);
22610 assert_eq!(rejection.status, 409);
22611 assert_eq!(rejection.reason, "historical_run_command_rejected");
22612 assert_eq!(rejection.run_id.as_deref(), Some("run-stale"));
22613 assert_eq!(rejection.target_scope.as_deref(), Some("run"));
22614 }
22615 }
22616
22617 #[tokio::test]
22618 async fn redrive_targets_a_failed_run_and_returns_successor_identity() {
22619 let server = MockWorkerServer::start();
22620 let client = Client::builder(server.base_url())
22621 .timeout(Duration::from_secs(2))
22622 .build()
22623 .expect("client");
22624
22625 let result = client
22626 .redrive_workflow_run("wf-lifecycle", "run-failed", Some("retry-1"))
22627 .await
22628 .expect("redrive accepted");
22629 assert_eq!(result.workflow_id, "wf-lifecycle");
22630 assert_eq!(result.source_run_id, "run-failed");
22631 assert_eq!(result.run_id, "run-successor");
22632 assert_eq!(result.resume_step_sequence, 2);
22633 assert_eq!(
22634 server.request_body("/api/workflows/wf-lifecycle/runs/run-failed/redrive"),
22635 json!({"request_id":"retry-1"})
22636 );
22637
22638 let repeated = client
22639 .redrive_workflow_run("wf-lifecycle", "run-failed-existing", Some("retry-1"))
22640 .await
22641 .expect("idempotent redrive response accepted");
22642 assert_eq!(repeated.run_id, "run-successor");
22643
22644 let error = client
22645 .redrive_workflow_run("wf-lifecycle", "run-completed", None)
22646 .await
22647 .expect_err("completed run must be rejected");
22648 let Error::Http { status, body } = error else {
22649 panic!("expected HTTP redrive rejection");
22650 };
22651 assert_eq!(status.as_u16(), 409);
22652 assert!(body.contains("run_not_failed"));
22653 assert_eq!(
22654 server.request_body("/api/workflows/wf-lifecycle/runs/run-completed/redrive"),
22655 json!({})
22656 );
22657
22658 let handle = WorkflowHandle {
22659 client: client.clone(),
22660 workflow_id: "wf-lifecycle".to_string(),
22661 run_id: Some("run-failed".to_string()),
22662 workflow_type: "test".to_string(),
22663 };
22664 assert_eq!(
22665 handle.redrive(None).await.expect("handle redrive").run_id,
22666 "run-successor"
22667 );
22668 let missing_run = WorkflowHandle {
22669 run_id: None,
22670 ..handle
22671 };
22672 assert!(matches!(
22673 missing_run.redrive(None).await,
22674 Err(Error::Codec(_))
22675 ));
22676 }
22677
22678 #[tokio::test]
22679 async fn workflow_start_options_send_server_enforced_deadlines() {
22680 let server = MockWorkerServer::start();
22681 let client = Client::builder(server.base_url())
22682 .timeout(Duration::from_secs(2))
22683 .build()
22684 .expect("client");
22685
22686 let handle = client
22687 .start_workflow_with_options(
22688 "rust.timeout",
22689 "rust-timeouts",
22690 "wf-start-options",
22691 WorkflowStartOptions::new()
22692 .execution_timeout_seconds(30)
22693 .run_timeout_seconds(1),
22694 json!([]),
22695 )
22696 .await
22697 .expect("workflow start");
22698
22699 assert_eq!(handle.run_id.as_deref(), Some("run-start-options"));
22700 let body = server.request_body("/api/workflows");
22701 assert_eq!(body["execution_timeout_seconds"], 30);
22702 assert_eq!(body["run_timeout_seconds"], 1);
22703
22704 let invalid = client
22705 .start_workflow_with_options(
22706 "rust.timeout",
22707 "rust-timeouts",
22708 "wf-invalid-options",
22709 WorkflowStartOptions::new()
22710 .execution_timeout_seconds(1)
22711 .run_timeout_seconds(2),
22712 json!([]),
22713 )
22714 .await
22715 .expect_err("invalid deadline ordering");
22716 assert!(invalid
22717 .to_string()
22718 .contains("run_timeout_seconds cannot exceed execution_timeout_seconds"));
22719 }
22720
22721 #[tokio::test]
22722 async fn workflow_result_returns_each_typed_terminal_outcome() {
22723 let server = MockWorkerServer::start();
22724 let client = Client::builder(server.base_url())
22725 .timeout(Duration::from_secs(2))
22726 .build()
22727 .expect("client");
22728 let options = WorkflowResultOptions {
22729 poll_interval: Duration::ZERO,
22730 timeout: Duration::from_secs(1),
22731 };
22732
22733 let failed = WorkflowHandle {
22734 client: client.clone(),
22735 workflow_id: "wf-failed".to_string(),
22736 run_id: Some("run-failed".to_string()),
22737 workflow_type: "failure".to_string(),
22738 }
22739 .result(options)
22740 .await
22741 .expect_err("failed outcome");
22742 let Error::WorkflowFailed(failure) = failed else {
22743 panic!("expected WorkflowFailed");
22744 };
22745 assert_eq!(failure.workflow_id, "wf-failed");
22746 assert_eq!(failure.run_id.as_deref(), Some("run-failed"));
22747 assert_eq!(failure.failure_id.as_deref(), Some("failure-17"));
22748 assert_eq!(failure.failure_category.as_deref(), Some("application"));
22749 assert_eq!(failure.exception_type.as_deref(), Some("PaymentError"));
22750 assert_eq!(
22751 failure.exception_class.as_deref(),
22752 Some("billing::PaymentError")
22753 );
22754 assert_eq!(failure.non_retryable, Some(true));
22755
22756 for (workflow_id, expected_kind, expected_reason) in [
22757 (
22758 "wf-cancelled",
22759 WorkflowTerminalKind::Cancelled,
22760 "cleanup requested",
22761 ),
22762 (
22763 "wf-terminated",
22764 WorkflowTerminalKind::Terminated,
22765 "forced stop",
22766 ),
22767 (
22768 "wf-timed-out",
22769 WorkflowTerminalKind::TimedOut,
22770 "run_timeout",
22771 ),
22772 ] {
22773 let error = WorkflowHandle {
22774 client: client.clone(),
22775 workflow_id: workflow_id.to_string(),
22776 run_id: None,
22777 workflow_type: "terminal".to_string(),
22778 }
22779 .result(options)
22780 .await
22781 .expect_err("typed terminal outcome");
22782 let outcome = match error {
22783 Error::WorkflowCancelled(outcome) => outcome,
22784 Error::WorkflowTerminated(outcome) => outcome,
22785 Error::WorkflowTimedOut(outcome) => outcome,
22786 other => panic!("unexpected terminal error: {other}"),
22787 };
22788 assert_eq!(outcome.kind, expected_kind);
22789 assert_eq!(outcome.workflow_id, workflow_id);
22790 assert_eq!(outcome.reason, expected_reason);
22791 }
22792
22793 let wait_timeout = WorkflowHandle {
22794 client,
22795 workflow_id: "wf-waiting".to_string(),
22796 run_id: Some("run-waiting".to_string()),
22797 workflow_type: "waiting".to_string(),
22798 }
22799 .result(WorkflowResultOptions {
22800 poll_interval: Duration::ZERO,
22801 timeout: Duration::ZERO,
22802 })
22803 .await
22804 .expect_err("client wait timeout");
22805 let Error::WorkflowTimedOut(timeout) = wait_timeout else {
22806 panic!("expected typed client timeout");
22807 };
22808 assert_eq!(timeout.reason, "result_wait_timeout");
22809 assert_eq!(timeout.failure_category.as_deref(), Some("client_timeout"));
22810 assert_eq!(timeout.run_id.as_deref(), Some("run-waiting"));
22811 }
22812
22813 #[tokio::test]
22814 async fn workflow_result_follows_chain_and_selected_result_preserves_history() {
22815 let server = MockWorkerServer::start();
22816 let client = Client::builder(server.base_url())
22817 .timeout(Duration::from_secs(2))
22818 .build()
22819 .expect("client");
22820
22821 let handle = WorkflowHandle {
22822 client,
22823 workflow_id: "wf-selected".to_string(),
22824 run_id: Some("run-selected".to_string()),
22825 workflow_type: "selected".to_string(),
22826 };
22827 let options = WorkflowResultOptions {
22828 poll_interval: Duration::ZERO,
22829 timeout: Duration::from_secs(1),
22830 };
22831
22832 let current = handle
22833 .result(options)
22834 .await
22835 .expect("instance result follows the current run");
22836 assert_eq!(current, json!("current run output"));
22837
22838 let error = handle
22839 .result_selected_run(options)
22840 .await
22841 .expect_err("the selected run is cancelled even though the current run completed");
22842
22843 let Error::WorkflowCancelled(outcome) = error else {
22844 panic!("expected selected run cancellation");
22845 };
22846 assert_eq!(outcome.run_id.as_deref(), Some("run-selected"));
22847 assert_eq!(outcome.reason, "selected run cancelled");
22848 assert_eq!(
22849 server.request_count("/api/workflows/wf-selected/runs/run-selected"),
22850 1
22851 );
22852 assert_eq!(server.request_count("/api/workflows/wf-selected"), 1);
22853 }
22854
22855 #[tokio::test]
22856 async fn poll_responses_decode_http_conflict_drain_as_a_stable_stop() {
22857 let server = MockWorkerServer::draining_polls();
22858 let client = Client::builder(server.base_url())
22859 .timeout(Duration::from_secs(2))
22860 .build()
22861 .expect("client");
22862
22863 let workflow = client
22864 .poll_workflow_task_response("draining-worker", "rust-workers", Duration::ZERO)
22865 .await
22866 .expect("workflow drain response");
22867 let activity = client
22868 .poll_activity_task_response("draining-worker", "rust-workers", Duration::ZERO)
22869 .await
22870 .expect("activity drain response");
22871 let query = client
22872 .poll_query_task_response("draining-worker", "rust-workers", Duration::ZERO)
22873 .await
22874 .expect("query drain response");
22875
22876 for outcome in [workflow.outcome(), activity.outcome(), query.outcome()] {
22877 assert_eq!(
22878 outcome,
22879 WorkerPollOutcome::Stop {
22880 poll_status: Some("draining".to_string()),
22881 reason: Some("worker_draining".to_string()),
22882 }
22883 );
22884 }
22885
22886 assert!(client
22887 .poll_workflow_task("draining-worker", "rust-workers", Duration::ZERO)
22888 .await
22889 .expect("compatibility poll")
22890 .is_none());
22891 }
22892
22893 #[tokio::test]
22894 async fn managed_worker_honors_drain_stop_for_every_task_family() {
22895 let server = MockWorkerServer::draining_polls();
22896 let client = Client::builder(server.base_url())
22897 .timeout(Duration::from_secs(2))
22898 .build()
22899 .expect("client");
22900
22901 let mut workflow_worker = Worker::new(client.clone(), "rust-workers")
22902 .worker_id("draining-workflow-worker")
22903 .poll_timeout(Duration::ZERO);
22904 workflow_worker.register_workflow("counter", |_ctx, _args| async { Ok(Value::Null) });
22905 workflow_worker
22906 .run()
22907 .await
22908 .expect("workflow drain is a clean stop");
22909
22910 let mut activity_worker = Worker::new(client.clone(), "rust-workers")
22911 .worker_id("draining-activity-worker")
22912 .poll_timeout(Duration::ZERO);
22913 activity_worker.register_activity("write", |_ctx, _args| async { Ok(Value::Null) });
22914 activity_worker
22915 .run()
22916 .await
22917 .expect("activity drain is a clean stop");
22918
22919 let mut query_worker = Worker::new(client, "rust-workers")
22920 .worker_id("draining-query-worker")
22921 .poll_timeout(Duration::ZERO);
22922 query_worker.register_query("counter", "current", |_ctx, _args| async {
22923 Ok(Value::Null)
22924 });
22925 query_worker
22926 .run()
22927 .await
22928 .expect("query drain is a clean stop");
22929 }
22930
22931 #[tokio::test]
22932 async fn activity_cancellation_and_late_completion_remain_machine_readable() {
22933 let server = MockWorkerServer::start();
22934 let client = Client::builder(server.base_url())
22935 .timeout(Duration::from_secs(2))
22936 .build()
22937 .expect("client");
22938
22939 let heartbeat = client
22940 .heartbeat_activity_task(
22941 "activity-cancel",
22942 "attempt-cancel",
22943 "rust-worker",
22944 typed_fidelity_probe(),
22945 )
22946 .await
22947 .expect("cancellation heartbeat");
22948 assert!(heartbeat.cancel_requested);
22949 assert!(heartbeat.should_stop());
22950 assert_eq!(heartbeat.reason.as_deref(), Some("run_cancelled"));
22951 assert_eq!(heartbeat.run_closed_reason.as_deref(), Some("cancelled"));
22952 let heartbeat_body =
22953 server.request_body("/api/worker/activity-tasks/activity-cancel/heartbeat");
22954 assert_eq!(heartbeat_body["details"]["codec"], DEFAULT_CODEC);
22955 assert_eq!(
22956 decode_wire_avro_value(&heartbeat_body["details"], DEFAULT_CODEC)
22957 .expect("typed heartbeat details"),
22958 typed_fidelity_probe()
22959 );
22960
22961 let error = client
22962 .complete_activity_task(
22963 "activity-cancel",
22964 "attempt-cancel",
22965 "rust-worker",
22966 json!({"late":true}),
22967 DEFAULT_CODEC,
22968 )
22969 .await
22970 .expect_err("late completion must be refused");
22971 assert!(activity_task_rejection_is_final(&error));
22972 let Error::ActivityTaskRejected(rejection) = error else {
22973 panic!("expected typed activity rejection");
22974 };
22975 assert_eq!(rejection.status, 409);
22976 assert_eq!(rejection.reason, "run_cancelled");
22977 assert!(rejection.cancel_requested);
22978 assert_eq!(rejection.can_continue, Some(false));
22979 }
22980
22981 #[tokio::test]
22982 async fn managed_worker_survives_late_completion_and_restart_during_cancellation() {
22983 let server = MockWorkerServer::cancelled_activity();
22984 let client = Client::builder(server.base_url())
22985 .timeout(Duration::from_secs(2))
22986 .build()
22987 .expect("client");
22988 let cancellation_observed = Arc::new(AtomicBool::new(false));
22989 let observed = Arc::clone(&cancellation_observed);
22990 let mut worker = Worker::new(client.clone(), "rust-workers")
22991 .worker_id("rust-cancel-worker")
22992 .poll_timeout(Duration::from_millis(10));
22993 worker.register_activity("cancel-aware", move |ctx, _args| {
22994 let observed = Arc::clone(&observed);
22995 async move {
22996 let heartbeat = ctx.heartbeat(json!({"stage":"running"})).await?;
22997 observed.store(heartbeat.should_stop(), Ordering::SeqCst);
22998 Ok(json!({"late":"completion"}))
22999 }
23000 });
23001
23002 assert_eq!(
23003 worker.run_once().await.expect("cancelled attempt handled"),
23004 1
23005 );
23006 assert!(cancellation_observed.load(Ordering::SeqCst));
23007 assert_eq!(
23008 server.request_count("/api/worker/activity-tasks/activity-cancel/complete"),
23009 1
23010 );
23011
23012 let mut restarted = Worker::new(client, "rust-workers")
23013 .worker_id("rust-cancel-worker-restarted")
23014 .poll_timeout(Duration::from_millis(10));
23015 restarted.register_activity("cancel-aware", |_ctx, _args| async move { Ok(Value::Null) });
23016 assert_eq!(
23017 restarted
23018 .run_once()
23019 .await
23020 .expect("replacement worker continues polling"),
23021 0
23022 );
23023 }
23024
23025 #[tokio::test]
23026 async fn managed_worker_absorbs_selected_run_terminal_timeout_completion_race() {
23027 let response = r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"outcome":"completed","recorded":false,"run_id":"run-selected-timeout","run_status":"failed","created_task_ids":[],"reason":"run_timed_out"}"#;
23028 let server = MockWorkerServer::workflow_completion("409 Conflict", response);
23029 let client = Client::builder(server.base_url())
23030 .timeout(Duration::from_secs(2))
23031 .build()
23032 .expect("client");
23033
23034 let direct_error = client
23035 .complete_workflow_task(
23036 "workflow-timeout-task",
23037 "timeout-worker",
23038 3,
23039 vec![json!({
23040 "type": "complete_workflow",
23041 "result": fixture_envelope(Value::Null)
23042 })],
23043 )
23044 .await
23045 .expect_err("the low-level client preserves the completion rejection");
23046 let Error::Http { status, body } = direct_error else {
23047 panic!("expected the original HTTP completion rejection");
23048 };
23049 assert_eq!(status, reqwest::StatusCode::CONFLICT);
23050 assert_eq!(
23051 serde_json::from_str::<Value>(&body).expect("response body")["reason"],
23052 "run_timed_out"
23053 );
23054
23055 let mut worker = Worker::new(client, "rust-workers")
23056 .worker_id("timeout-worker")
23057 .poll_timeout(Duration::from_millis(10));
23058 worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
23059 Ok(json!({"late": "result"}))
23060 });
23061
23062 assert_eq!(
23063 worker
23064 .run_once()
23065 .await
23066 .expect("authoritative selected-run timeout settles the tick"),
23067 1
23068 );
23069 assert_eq!(
23070 server.request_count("/api/worker/workflow-tasks/workflow-timeout-task/complete"),
23071 2,
23072 "both the direct client proof and managed worker must see the rejection"
23073 );
23074 }
23075
23076 #[tokio::test]
23077 async fn managed_worker_does_not_swallow_nearby_completion_errors() {
23078 for (name, status, response) in [
23079 ("bare conflict", "409 Conflict", r#"{"message":"conflict"}"#),
23080 (
23081 "command was recorded",
23082 "409 Conflict",
23083 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":true,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
23084 ),
23085 (
23086 "lease conflict",
23087 "409 Conflict",
23088 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"lease_expired"}"#,
23089 ),
23090 (
23091 "nonterminal run",
23092 "409 Conflict",
23093 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"waiting","reason":"run_timed_out"}"#,
23094 ),
23095 (
23096 "different selected run",
23097 "409 Conflict",
23098 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-reused-workflow-current","run_status":"failed","reason":"run_timed_out"}"#,
23099 ),
23100 (
23101 "different task attempt",
23102 "409 Conflict",
23103 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":4,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
23104 ),
23105 (
23106 "authentication failure",
23107 "401 Unauthorized",
23108 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
23109 ),
23110 (
23111 "authorization failure",
23112 "403 Forbidden",
23113 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
23114 ),
23115 (
23116 "protocol failure",
23117 "400 Bad Request",
23118 r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.2","requested_version":"1.3"}"#,
23119 ),
23120 (
23121 "malformed command",
23122 "422 Unprocessable Entity",
23123 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
23124 ),
23125 (
23126 "transient server failure",
23127 "503 Service Unavailable",
23128 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
23129 ),
23130 ] {
23131 let server = MockWorkerServer::workflow_completion(status, response);
23132 let client = Client::builder(server.base_url())
23133 .timeout(Duration::from_secs(2))
23134 .build()
23135 .expect("client");
23136 let mut worker = Worker::new(client, "rust-workers")
23137 .worker_id("timeout-worker")
23138 .poll_timeout(Duration::from_millis(10));
23139 worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
23140 Ok(json!({"late": "result"}))
23141 });
23142
23143 let error = worker
23144 .run_once()
23145 .await
23146 .expect_err(&format!("{name} must remain an error"));
23147 assert!(
23148 matches!(error, Error::Http { .. } | Error::Protocol(_)),
23149 "{name} returned an unexpected error variant: {error}"
23150 );
23151 }
23152 }
23153
23154 #[tokio::test]
23155 async fn worker_deregistration_uses_worker_plane_method_path_headers_and_result() {
23156 let server = MockWorkerServer::start();
23157 let client = Client::builder(server.base_url())
23158 .worker_token(Some("worker-secret".to_string()))
23159 .namespace("orders")
23160 .timeout(Duration::from_secs(2))
23161 .build()
23162 .expect("client");
23163 let path = "/api/worker/registrations/worker%2F%CE%B1%20space";
23164
23165 let result = client
23166 .deregister_worker_registration("worker/α space")
23167 .await
23168 .expect("deregister worker registration");
23169
23170 assert_eq!(server.method_for(path).as_deref(), Some("DELETE"));
23171 assert_eq!(
23172 server.worker_protocol_for(path).as_deref(),
23173 Some(WORKER_PROTOCOL_VERSION)
23174 );
23175 assert_eq!(server.control_protocol_for(path), None);
23176 assert_eq!(server.namespace_for(path).as_deref(), Some("orders"));
23177 assert_eq!(
23178 server.authorization_for(path).as_deref(),
23179 Some("Bearer worker-secret")
23180 );
23181 assert_eq!(
23182 result,
23183 WorkerDeregistrationEnvelope {
23184 worker_id: "deregistered-worker".to_string(),
23185 outcome: "deregistered".to_string(),
23186 recovered_workflow_task_count: 2,
23187 }
23188 );
23189 }
23190
23191 #[tokio::test]
23192 async fn low_level_registration_rejects_update_validators_before_transport() {
23193 let server = MockWorkerServer::start();
23194 let client = Client::builder(server.base_url())
23195 .timeout(Duration::from_secs(2))
23196 .build()
23197 .expect("client");
23198
23199 for update_validators in [json!(["approve"]), json!("approve")] {
23200 let error = client
23201 .register_worker_with_command_contracts(
23202 "validator-claiming-worker",
23203 "rust-workers",
23204 vec!["orders".to_string()],
23205 vec![],
23206 1,
23207 1,
23208 vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
23209 json!({
23210 "orders": {
23211 "queries": ["current"],
23212 "updates": ["approve"],
23213 "update_validators": update_validators,
23214 },
23215 }),
23216 )
23217 .await
23218 .expect_err("unsupported validator claims must fail before registration");
23219
23220 let Error::UnsupportedUpdateValidators { workflow_type } = error else {
23221 panic!("expected typed unsupported-validator failure");
23222 };
23223 assert_eq!(workflow_type, "orders");
23224 }
23225 assert_eq!(server.request_count("/api/worker/register"), 0);
23226 }
23227
23228 #[tokio::test]
23229 async fn low_level_registration_preserves_query_and_update_contracts() {
23230 let server = MockWorkerServer::start();
23231 let client = Client::builder(server.base_url())
23232 .timeout(Duration::from_secs(2))
23233 .build()
23234 .expect("client");
23235 let contracts = json!({
23236 "orders": {
23237 "queries": ["current"],
23238 "updates": ["approve"],
23239 "update_validators": [],
23240 },
23241 "payments": {
23242 "queries": ["status"],
23243 "updates": ["capture"],
23244 },
23245 });
23246
23247 client
23248 .register_worker_with_command_contracts(
23249 "command-worker",
23250 "rust-workers",
23251 vec!["orders".to_string(), "payments".to_string()],
23252 vec![],
23253 1,
23254 1,
23255 vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
23256 contracts.clone(),
23257 )
23258 .await
23259 .expect("query and update contracts must remain supported");
23260
23261 assert_eq!(
23262 server.request_body("/api/worker/register")["workflow_command_contracts"],
23263 contracts
23264 );
23265 assert!(server
23266 .request_body("/api/worker/register")
23267 .get("workflow_definition_fingerprints")
23268 .is_none());
23269 }
23270
23271 #[tokio::test]
23272 async fn rust_worker_advertises_source_backed_definition_and_clears_it_on_reregistration() {
23273 let server = MockWorkerServer::start();
23274 let client = Client::builder(server.base_url())
23275 .timeout(Duration::from_secs(2))
23276 .build()
23277 .expect("client");
23278 let mut worker = Worker::new(client, "rust-workers");
23279 worker.register_workflow("orders", |_ctx, _input| async { Ok(Value::Null) });
23280
23281 assert!(worker
23282 .set_workflow_definition_sources("missing", &[include_str!("lib.rs")])
23283 .is_err());
23284 assert!(worker
23285 .set_workflow_definition_sources("orders", &[])
23286 .is_err());
23287 worker
23288 .set_workflow_definition_sources("orders", &[include_str!("lib.rs")])
23289 .expect("embed workflow source");
23290 worker.register().await.expect("register worker");
23291 let first = server.request_bodies("/api/worker/register")[0]
23292 ["workflow_definition_fingerprints"]["orders"]
23293 .as_str()
23294 .expect("source-backed fingerprint")
23295 .to_string();
23296 assert!(first.starts_with("sha256:"));
23297
23298 worker
23299 .set_workflow_definition_sources("orders", &[include_str!("lib.rs"), "changed helper"])
23300 .expect("change embedded helper source");
23301 worker.register().await.expect("register changed source");
23302 let changed = server.request_bodies("/api/worker/register")[1]
23303 ["workflow_definition_fingerprints"]["orders"]
23304 .as_str()
23305 .expect("changed fingerprint")
23306 .to_string();
23307 assert_ne!(first, changed);
23308
23309 worker.register_workflow("orders", |_ctx, _input| async { Ok(Value::Null) });
23310 worker
23311 .register()
23312 .await
23313 .expect("register replacement handler");
23314 assert_eq!(
23315 server.request_bodies("/api/worker/register")[2]["workflow_definition_fingerprints"],
23316 json!({})
23317 );
23318 }
23319
23320 #[tokio::test]
23321 async fn role_scoped_tokens_are_never_used_for_the_opposite_plane() {
23322 let server = MockWorkerServer::start();
23323 let control_only = Client::builder(server.base_url())
23324 .control_token(Some("control-secret".to_string()))
23325 .build()
23326 .expect("control client");
23327
23328 let error = control_only
23329 .register_worker("worker", "queue", vec![], vec![], 1, 1)
23330 .await
23331 .expect_err("control token must not authorize a worker request");
23332 assert!(matches!(
23333 error,
23334 Error::MissingRoleCredentials { role: "worker", .. }
23335 ));
23336 assert_eq!(server.request_count("/api/worker/register"), 0);
23337
23338 let worker_only = Client::builder(server.base_url())
23339 .worker_token(Some("worker-secret".to_string()))
23340 .build()
23341 .expect("worker client");
23342 let error = worker_only
23343 .health()
23344 .await
23345 .expect_err("worker token must not authorize a control request");
23346 assert!(matches!(
23347 error,
23348 Error::MissingRoleCredentials {
23349 role: "control",
23350 ..
23351 }
23352 ));
23353 assert_eq!(server.request_count("/api/health"), 0);
23354 }
23355
23356 #[tokio::test]
23357 async fn shared_token_supports_worker_and_control_planes() {
23358 let server = MockWorkerServer::start();
23359 let client = Client::builder(server.base_url())
23360 .token(Some("shared-secret".to_string()))
23361 .build()
23362 .expect("client");
23363
23364 client.health().await.expect("control request");
23365 client
23366 .register_worker("worker", "queue", vec![], vec![], 1, 1)
23367 .await
23368 .expect("worker request");
23369
23370 assert_eq!(
23371 server.authorization_for("/api/health").as_deref(),
23372 Some("Bearer shared-secret")
23373 );
23374 assert_eq!(
23375 server.control_protocol_for("/api/health").as_deref(),
23376 Some(CONTROL_PLANE_VERSION)
23377 );
23378 assert_eq!(
23379 server.authorization_for("/api/worker/register").as_deref(),
23380 Some("Bearer shared-secret")
23381 );
23382 assert_eq!(
23383 server
23384 .worker_protocol_for("/api/worker/register")
23385 .as_deref(),
23386 Some(WORKER_PROTOCOL_VERSION)
23387 );
23388 }
23389
23390 #[tokio::test]
23391 async fn baseline_worker_endpoints_send_the_baseline_protocol() {
23392 let server = MockWorkerServer::start();
23393 let client = Client::builder(server.base_url())
23394 .timeout(Duration::from_secs(2))
23395 .build()
23396 .expect("client");
23397
23398 client
23399 .register_worker("capture-worker", "capture", vec![], vec![], 1, 1)
23400 .await
23401 .expect("register");
23402 client
23403 .heartbeat_worker("capture-worker", 1, 1)
23404 .await
23405 .expect("heartbeat");
23406 client
23407 .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
23408 .await
23409 .expect("workflow poll");
23410 client
23411 .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
23412 .await
23413 .expect("activity poll");
23414
23415 for path in [
23416 "/api/worker/register",
23417 "/api/worker/heartbeat",
23418 "/api/worker/workflow-tasks/poll",
23419 "/api/worker/activity-tasks/poll",
23420 ] {
23421 assert_eq!(
23422 server.worker_protocol_for(path).as_deref(),
23423 Some(WORKER_PROTOCOL_VERSION),
23424 "unexpected protocol for {path}"
23425 );
23426 }
23427
23428 assert_eq!(
23429 server.request_body("/api/worker/workflow-tasks/poll")["timeout_seconds"],
23430 1
23431 );
23432 assert_eq!(
23433 server.request_body("/api/worker/activity-tasks/poll")["timeout_seconds"],
23434 1
23435 );
23436 assert!(
23437 server.request_body("/api/worker/workflow-tasks/poll")["poll_request_id"]
23438 .as_str()
23439 .is_some_and(|id| id.starts_with("rust-workflow-poll-"))
23440 );
23441 assert!(
23442 server.request_body("/api/worker/activity-tasks/poll")["poll_request_id"]
23443 .as_str()
23444 .is_some_and(|id| id.starts_with("rust-activity-poll-"))
23445 );
23446 }
23447
23448 #[tokio::test]
23449 async fn query_task_endpoints_send_the_query_feature_protocol() {
23450 let server = MockWorkerServer::start();
23451 let client = Client::builder(server.base_url())
23452 .timeout(Duration::from_secs(2))
23453 .build()
23454 .expect("client");
23455
23456 client
23457 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
23458 .await
23459 .expect("query poll");
23460 client
23461 .complete_query_task(
23462 "query-capture",
23463 "capture-worker",
23464 1,
23465 json!(8),
23466 DEFAULT_CODEC,
23467 )
23468 .await
23469 .expect("query complete");
23470 client
23471 .fail_query_task(
23472 "query-capture",
23473 "capture-worker",
23474 1,
23475 "failed",
23476 "query_rejected",
23477 "QueryFailed",
23478 )
23479 .await
23480 .expect("query fail");
23481
23482 for path in [
23483 "/api/worker/query-tasks/poll",
23484 "/api/worker/query-tasks/query-capture/complete",
23485 "/api/worker/query-tasks/query-capture/fail",
23486 ] {
23487 assert_eq!(
23488 server.worker_protocol_for(path).as_deref(),
23489 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
23490 "unexpected protocol for {path}"
23491 );
23492 }
23493
23494 assert_eq!(
23495 server.request_body("/api/worker/query-tasks/poll")["timeout_seconds"],
23496 1
23497 );
23498 assert!(
23499 server.request_body("/api/worker/query-tasks/poll")["poll_request_id"]
23500 .as_str()
23501 .is_some_and(|id| id.starts_with("rust-query-poll-"))
23502 );
23503 }
23504
23505 #[tokio::test]
23506 async fn disconnected_client_polls_retry_once_with_the_same_request_id() {
23507 let server = MockWorkerServer::transient_worker_failures();
23508 let client = Client::builder(server.base_url())
23509 .timeout(Duration::from_secs(2))
23510 .build()
23511 .expect("client");
23512
23513 client
23514 .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
23515 .await
23516 .expect("workflow poll retry");
23517 client
23518 .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
23519 .await
23520 .expect("activity poll retry");
23521 client
23522 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
23523 .await
23524 .expect("query poll retry");
23525
23526 for path in [
23527 "/api/worker/workflow-tasks/poll",
23528 "/api/worker/activity-tasks/poll",
23529 "/api/worker/query-tasks/poll",
23530 ] {
23531 let bodies = server.request_bodies(path);
23532 assert_eq!(bodies.len(), 2, "{path} must be retried once");
23533 assert_eq!(
23534 bodies[0]["poll_request_id"], bodies[1]["poll_request_id"],
23535 "{path} must preserve the request binding across retry"
23536 );
23537 }
23538 }
23539
23540 #[tokio::test]
23541 async fn worker_poll_retries_preserve_request_id_across_consecutive_failures() {
23542 let server = MockWorkerServer::consecutive_poll_failures(2);
23543 let client = Client::builder(server.base_url())
23544 .timeout(Duration::from_secs(2))
23545 .build()
23546 .expect("client");
23547 let mut worker = Worker::new(client, "capture")
23548 .worker_id("capture-worker")
23549 .poll_timeout(Duration::from_millis(10))
23550 .retry_policy(WorkerRetryPolicy {
23551 max_retries: 2,
23552 initial_backoff: Duration::from_millis(1),
23553 max_backoff: Duration::from_millis(1),
23554 });
23555 worker.register_workflow(
23556 "capture.workflow",
23557 |_ctx, _input| async move { Ok(Value::Null) },
23558 );
23559 worker.register_activity(
23560 "capture.activity",
23561 |_ctx, _input| async move { Ok(Value::Null) },
23562 );
23563 worker.register_query("capture.workflow", "current", |_ctx, _args| async move {
23564 Ok(Value::Null)
23565 });
23566
23567 assert_eq!(worker.run_once().await.expect("poll retries"), 0);
23568
23569 for path in [
23570 "/api/worker/workflow-tasks/poll",
23571 "/api/worker/activity-tasks/poll",
23572 "/api/worker/query-tasks/poll",
23573 ] {
23574 let bodies = server.request_bodies(path);
23575 assert_eq!(bodies.len(), 3, "{path} must use exactly two retries");
23576 assert!(
23577 bodies
23578 .iter()
23579 .all(|body| body["poll_request_id"] == bodies[0]["poll_request_id"]),
23580 "{path} must preserve one request binding across every retry"
23581 );
23582 }
23583 }
23584
23585 fn storage_refusal(poll_id: Option<&str>, unavailable: bool, mid_poll: bool) -> Value {
23586 let reason = if unavailable {
23587 "storage_admission_unavailable"
23588 } else {
23589 "storage_pressure"
23590 };
23591 let mut body = json!({
23592 "reason": reason,
23593 "storage_state": if unavailable { "fenced" } else { "draining" },
23594 "retryable": true,
23595 "retry_after_seconds": 1,
23596 });
23597 if !mid_poll {
23598 body["request_admitted"] = json!(false);
23599 }
23600 if let Some(id) = poll_id {
23601 body["task"] = Value::Null;
23602 body["poll_status"] = json!(reason);
23603 body["poll_request_id"] = json!(id);
23604 body["retry_same_poll_request_id"] = json!(true);
23605 body["claim_admitted"] = json!(false);
23606 }
23607 body
23608 }
23609
23610 fn backend_refusal(path: &str, request: &str) -> Option<Value> {
23611 let operation = match path {
23612 "/api/worker/register" => "register_worker",
23613 "/api/worker/heartbeat" => "heartbeat_worker",
23614 "/api/worker/workflow-tasks/poll" => "poll_workflow_task",
23615 "/api/worker/activity-tasks/poll" => "poll_activity_task",
23616 "/api/worker/query-tasks/poll" => "poll_query_task",
23617 "/api/worker/update-validation-tasks/poll" => "poll_update_validation_task",
23618 _ => return None,
23619 };
23620 let request: Value = serde_json::from_str(request).ok()?;
23621 let mut response = json!({
23622 "reason": "backend_unavailable",
23623 "operation": operation,
23624 "outcome": "unknown",
23625 "retryable": true,
23626 "retry_after_seconds": 1,
23627 "worker_id": request["worker_id"],
23628 "task_queue": request.get("task_queue"),
23629 });
23630 if path.ends_with("/poll") {
23631 response["task"] = Value::Null;
23632 response["poll_status"] = json!("backend_unavailable");
23633 response["poll_request_id"] = request["poll_request_id"].clone();
23634 response["retry_same_poll_request_id"] = json!(true);
23635 }
23636 Some(response)
23637 }
23638
23639 fn backend_retry_override(
23640 path: &str,
23641 request: &str,
23642 number: usize,
23643 ) -> Option<(&'static str, String)> {
23644 if number > 8 {
23645 return None;
23646 }
23647 Some((
23648 "503 Service Unavailable",
23649 backend_refusal(path, request)?.to_string(),
23650 ))
23651 }
23652
23653 fn backend_unavailable_override(
23654 path: &str,
23655 request: &str,
23656 _number: usize,
23657 ) -> Option<(&'static str, String)> {
23658 Some((
23659 "503 Service Unavailable",
23660 backend_refusal(path, request)?.to_string(),
23661 ))
23662 }
23663
23664 fn storage_worker(server: &MockWorkerServer) -> Worker {
23665 Worker::new(Client::new(server.base_url()).expect("client"), "storage")
23666 .worker_id("storage-worker")
23667 .retry_policy(WorkerRetryPolicy {
23668 max_retries: 1,
23669 initial_backoff: Duration::from_millis(1),
23670 max_backoff: Duration::from_millis(1),
23671 })
23672 }
23673
23674 fn assert_identical_requests(server: &MockWorkerServer, path: &str, count: usize) {
23675 let requests = server.requests.lock().expect("requests");
23676 let bodies: Vec<_> = requests
23677 .iter()
23678 .filter(|request| request.path == path)
23679 .map(|request| &request.body)
23680 .collect();
23681 assert_eq!(bodies.len(), count, "{path}");
23682 assert!(bodies.iter().all(|body| body == &bodies[0]), "{path}");
23683 }
23684
23685 #[test]
23686 fn backend_recovery_requires_the_explicit_worker_contract() {
23687 for (path, operation) in [
23688 ("/worker/register", "register_worker"),
23689 ("/worker/heartbeat", "heartbeat_worker"),
23690 ("/worker/workflow-tasks/poll", "poll_workflow_task"),
23691 ("/worker/activity-tasks/poll", "poll_activity_task"),
23692 ("/worker/query-tasks/poll", "poll_query_task"),
23693 (
23694 "/worker/update-validation-tasks/poll",
23695 "poll_update_validation_task",
23696 ),
23697 ] {
23698 let poll_id = path.ends_with("/poll").then_some("same-poll");
23699 let request = json!({"worker_id":"same-worker","task_queue":"same-queue","poll_request_id":poll_id});
23700 let body = backend_refusal(&format!("/api{path}"), &request.to_string())
23701 .expect("worker operation");
23702 assert_eq!(body["operation"], operation);
23703 let error = Error::Http {
23704 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23705 body: body.to_string(),
23706 };
23707 assert_eq!(
23708 worker_backend_unavailable_retry_after(&error, path, poll_id),
23709 Some(Duration::from_secs(1))
23710 );
23711 assert!(!worker_operation_is_retryable(&error));
23712 for (field, value) in [
23713 ("operation", json!("wrong_operation")),
23714 ("outcome", json!("completed")),
23715 ("retryable", json!(false)),
23716 ("retry_after_seconds", json!(0)),
23717 ("worker_id", json!("")),
23718 ] {
23719 let mut invalid = body.clone();
23720 invalid[field] = value;
23721 let error = Error::Http {
23722 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23723 body: invalid.to_string(),
23724 };
23725 assert!(
23726 worker_backend_unavailable_retry_after(&error, path, poll_id).is_none(),
23727 "{path}: {field}"
23728 );
23729 assert!(!worker_operation_is_retryable(&error));
23730 }
23731 if let Some(poll_id) = poll_id {
23732 for (field, value) in [
23733 ("poll_request_id", json!("wrong-poll")),
23734 ("poll_status", json!("empty")),
23735 ("task", json!({"task_id":"claimed"})),
23736 ("retry_same_poll_request_id", json!(false)),
23737 ] {
23738 let mut invalid = body.clone();
23739 invalid[field] = value;
23740 let error = Error::Http {
23741 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23742 body: invalid.to_string(),
23743 };
23744 assert!(
23745 worker_backend_unavailable_retry_after(&error, path, Some(poll_id))
23746 .is_none(),
23747 "{path}: {field}"
23748 );
23749 }
23750 }
23751 let error = Error::Http {
23752 status: reqwest::StatusCode::UNAUTHORIZED,
23753 body: body.to_string(),
23754 };
23755 assert!(worker_backend_unavailable_retry_after(&error, path, poll_id).is_none());
23756 assert!(!worker_operation_is_retryable(&error));
23757 }
23758 }
23759
23760 #[tokio::test]
23761 async fn backend_recovery_outlives_generic_retry_budget_and_preserves_polls() {
23762 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23763 request_override: Some(backend_retry_override),
23764 ..MockWorkerBehavior::default()
23765 });
23766 let mut worker =
23767 storage_worker(&server).with_storage_admission(Arc::new(AtomicBool::new(false)));
23768 worker.register_workflow("backend.workflow", |_, _| async { Ok(Value::Null) });
23769 worker.register_activity("backend.activity", |_, _| async { Ok(Value::Null) });
23770 worker.register_query("backend.workflow", "state", |_, _| async {
23771 Ok(Value::Null)
23772 });
23773 worker.register().await.expect("registration recovery");
23774 worker
23775 .client
23776 .heartbeat_worker("storage-worker", 1, 1)
23777 .await
23778 .expect("heartbeat recovery");
23779 assert_eq!(worker.run_once().await.expect("poll recovery"), 0);
23780 for path in [
23781 "/api/worker/register",
23782 "/api/worker/heartbeat",
23783 "/api/worker/workflow-tasks/poll",
23784 "/api/worker/activity-tasks/poll",
23785 "/api/worker/query-tasks/poll",
23786 ] {
23787 assert_identical_requests(&server, path, 9);
23788 }
23789 }
23790
23791 #[tokio::test]
23792 async fn backend_recovery_wait_stops_with_worker() {
23793 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23794 request_override: Some(backend_unavailable_override),
23795 ..MockWorkerBehavior::default()
23796 });
23797 let mut worker = storage_worker(&server).retry_policy(WorkerRetryPolicy {
23798 max_retries: 1,
23799 initial_backoff: Duration::from_secs(2),
23800 max_backoff: Duration::from_secs(2),
23801 });
23802 worker.register_workflow("backend.workflow", |_, _| async { Ok(Value::Null) });
23803 let started = tokio::time::Instant::now();
23804 let _ = tokio::time::timeout(
23805 Duration::from_secs(1),
23806 worker.run_until(tokio::time::sleep(Duration::from_millis(100))),
23807 )
23808 .await
23809 .expect("shutdown interrupts backend wait");
23810 assert!(started.elapsed() < Duration::from_secs(1));
23811 assert!(server.request_count("/api/worker/register") >= 1);
23812 }
23813
23814 #[test]
23815 fn storage_admission_requires_an_explicit_identity_preserving_contract() {
23816 for unavailable in [false, true] {
23817 for mid_poll in [false, true] {
23818 let body = storage_refusal(Some("same-poll"), unavailable, mid_poll);
23819 let error = Error::Http {
23820 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23821 body: body.to_string(),
23822 };
23823 assert_eq!(
23824 worker_storage_admission_retry_after(&error, Some("same-poll")),
23825 Some(Duration::from_secs(1))
23826 );
23827 assert!(
23828 !worker_operation_is_retryable(&error),
23829 "storage is not a bounded generic retry"
23830 );
23831 for (field, value) in [
23832 ("poll_request_id", json!("wrong-poll")),
23833 ("task", json!({"task_id":"claimed"})),
23834 ("retryable", json!(false)),
23835 ("retry_after_seconds", json!(0)),
23836 ("retry_after_seconds", json!(true)),
23837 ("retry_after_seconds", json!(1.0)),
23838 ("storage_state", json!("normal")),
23839 ("poll_status", json!("empty")),
23840 ("claim_admitted", json!(true)),
23841 ("retry_same_poll_request_id", json!(false)),
23842 ("request_admitted", json!(true)),
23843 ] {
23844 let mut invalid = body.clone();
23845 invalid[field] = value;
23846 let error = Error::Http {
23847 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23848 body: invalid.to_string(),
23849 };
23850 assert!(
23851 worker_storage_admission_retry_after(&error, Some("same-poll")).is_none(),
23852 "{field}"
23853 );
23854 }
23855 }
23856 }
23857 let body = storage_refusal(None, false, false);
23858 let error = Error::Http {
23859 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23860 body: body.to_string(),
23861 };
23862 assert!(worker_storage_admission_retry_after(&error, None).is_some());
23863 assert!(worker_storage_admission_retry_after(&error, Some("")).is_none());
23864 let error = Error::Http {
23865 status: reqwest::StatusCode::FORBIDDEN,
23866 body: body.to_string(),
23867 };
23868 assert!(worker_storage_admission_retry_after(&error, None).is_none());
23869 let body = storage_refusal(None, false, true);
23870 let error = Error::Http {
23871 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23872 body: body.to_string(),
23873 };
23874 assert!(worker_storage_admission_retry_after(&error, None).is_none());
23875 }
23876
23877 #[tokio::test]
23878 async fn storage_poll_recovery_preserves_ambiguous_claim_identity() {
23879 for unavailable in [false, true] {
23880 for mid_poll in [false, true] {
23881 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23882 poll_failures_per_path: 1,
23883 storage_refusals: 7,
23884 storage_path: Some("/poll"),
23885 storage_unavailable: unavailable,
23886 storage_mid_poll: mid_poll,
23887 ..MockWorkerBehavior::default()
23888 });
23889 let mut worker = storage_worker(&server);
23890 worker.register_query("unused", "state", |_, _| async { Ok(Value::Null) });
23891 assert_eq!(worker.run_once().await.expect("storage recovery"), 0);
23892 for path in [
23893 "/api/worker/workflow-tasks/poll",
23894 "/api/worker/activity-tasks/poll",
23895 "/api/worker/query-tasks/poll",
23896 ] {
23897 assert_identical_requests(&server, path, 9);
23898 }
23899 }
23900 }
23901 }
23902
23903 #[tokio::test]
23904 async fn storage_refused_mutations_do_not_reserialize_or_change_client_scope() {
23905 struct CountedBody(Arc<AtomicUsize>);
23906 impl Serialize for CountedBody {
23907 fn serialize<S: Serializer>(
23908 &self,
23909 serializer: S,
23910 ) -> std::result::Result<S::Ok, S::Error> {
23911 let count = self.0.fetch_add(1, Ordering::SeqCst);
23912 json!({"serialization":count,"lease_owner":"worker","attempt":7})
23913 .serialize(serializer)
23914 }
23915 }
23916 for path in [
23917 "/api/worker/register",
23918 "/api/worker/heartbeat",
23919 "/api/worker/workflow-tasks/storage-task/complete",
23920 "/api/worker/workflow-tasks/storage-task/fail",
23921 "/api/worker/activity-tasks/storage-task/complete",
23922 "/api/worker/activity-tasks/storage-task/fail",
23923 "/api/worker/activity-tasks/storage-task/heartbeat",
23924 "/api/worker/query-tasks/storage-task/complete",
23925 "/api/worker/query-tasks/storage-task/fail",
23926 ] {
23927 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23928 storage_refusals: 7,
23929 storage_path: Some(path),
23930 ..MockWorkerBehavior::default()
23931 });
23932 let worker =
23933 storage_worker(&server).with_storage_admission(Arc::new(AtomicBool::new(false)));
23934 let calls = Arc::new(AtomicUsize::new(0));
23935 let _: Value = worker
23936 .client
23937 .request_json(
23938 reqwest::Method::POST,
23939 &path[4..],
23940 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
23941 Some(&CountedBody(Arc::clone(&calls))),
23942 )
23943 .await
23944 .expect("prepared request recovery");
23945 assert_eq!(calls.load(Ordering::SeqCst), 1);
23946 assert_identical_requests(&server, path, 8);
23947 }
23948 for worker_scope in [false, true] {
23949 let path = if worker_scope {
23950 "/api/health"
23951 } else {
23952 "/api/worker/register"
23953 };
23954 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23955 storage_refusals: usize::MAX,
23956 storage_path: Some(path),
23957 ..MockWorkerBehavior::default()
23958 });
23959 let worker = storage_worker(&server);
23960 let client = worker.client.clone();
23961 let worker = worker.with_storage_admission(Arc::new(AtomicBool::new(false)));
23962 let error = if worker_scope {
23963 worker
23964 .client
23965 .health()
23966 .await
23967 .expect_err("control plane is not retried")
23968 } else {
23969 client
23970 .request_json::<Value, Value>(
23971 reqwest::Method::POST,
23972 "/worker/register",
23973 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
23974 Some(&json!({})),
23975 )
23976 .await
23977 .expect_err("direct client is not retried")
23978 };
23979 assert!(worker_storage_admission_body(&error).is_some());
23980 assert_eq!(server.request_count(path), 1);
23981 }
23982 }
23983
23984 #[tokio::test]
23985 async fn storage_activity_outcome_is_retained_without_reexecuting_handler() {
23986 for fail in [false, true] {
23987 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23988 storage_activity: true,
23989 storage_refusals: 7,
23990 storage_path: Some("/storage-activity/"),
23991 ..MockWorkerBehavior::default()
23992 });
23993 let mut worker = storage_worker(&server);
23994 let calls = Arc::new(AtomicUsize::new(0));
23995 let observed = Arc::clone(&calls);
23996 worker.register_activity("storage.activity", move |ctx, _| {
23997 let calls = Arc::clone(&observed);
23998 async move {
23999 calls.fetch_add(1, Ordering::SeqCst);
24000 ctx.heartbeat(json!({"step":2})).await?;
24001 if fail {
24002 Err(Error::WorkerLoop("intentional handler failure".to_string()))
24003 } else {
24004 Ok(json!({"receipt":true}))
24005 }
24006 }
24007 });
24008 assert_eq!(worker.run_once().await.expect("activity settled"), 1);
24009 assert_eq!(calls.load(Ordering::SeqCst), 1);
24010 assert_identical_requests(
24011 &server,
24012 "/api/worker/activity-tasks/storage-activity/heartbeat",
24013 8,
24014 );
24015 let suffix = if fail { "fail" } else { "complete" };
24016 assert_identical_requests(
24017 &server,
24018 &format!("/api/worker/activity-tasks/storage-activity/{suffix}"),
24019 8,
24020 );
24021 let other = if fail { "complete" } else { "fail" };
24022 assert_eq!(
24023 server.request_count(&format!(
24024 "/api/worker/activity-tasks/storage-activity/{other}"
24025 )),
24026 0
24027 );
24028 }
24029 }
24030
24031 #[tokio::test]
24032 async fn storage_waits_are_interruptible_without_false_activity_failure() {
24033 for path in [
24034 "/api/worker/register",
24035 "/api/worker/heartbeat",
24036 "/api/worker/activity-tasks/poll",
24037 "/api/worker/activity-tasks/storage-activity/heartbeat",
24038 "/api/worker/activity-tasks/storage-activity/complete",
24039 ] {
24040 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
24041 storage_activity: true,
24042 storage_refusals: usize::MAX,
24043 storage_path: Some(path),
24044 ..MockWorkerBehavior::default()
24045 });
24046 let mut worker = storage_worker(&server).retry_policy(WorkerRetryPolicy::default());
24047 worker.register_activity("storage.activity", |ctx, _| async move {
24048 ctx.heartbeat(json!({"step":2})).await?;
24049 Ok(json!({"receipt":true}))
24050 });
24051 let shutdown = async {
24052 while server.request_count(path) == 0 {
24053 tokio::time::sleep(Duration::from_millis(1)).await;
24054 }
24055 };
24056 let result = tokio::time::timeout(Duration::from_secs(2), worker.run_until(shutdown))
24057 .await
24058 .expect("shutdown interrupts admission");
24059 assert!(
24060 result.is_err(),
24061 "a refused operation must not appear acknowledged: {path}, {result:?}"
24062 );
24063 assert_eq!(server.request_count(path), 1);
24064 assert_eq!(
24065 server.request_count("/api/worker/activity-tasks/storage-activity/fail"),
24066 0
24067 );
24068 assert_eq!(
24069 server.request_count("/api/worker/registrations/mock-worker"),
24070 usize::from(!path.ends_with("/register"))
24071 );
24072 }
24073 }
24074
24075 #[tokio::test]
24076 async fn storage_query_outcome_is_retained_without_reexecuting_handler() {
24077 for fail in [false, true] {
24078 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
24079 storage_query: true,
24080 storage_refusals: 7,
24081 storage_path: Some("/storage-query/"),
24082 ..MockWorkerBehavior::default()
24083 });
24084 let mut worker = storage_worker(&server);
24085 worker.register_workflow("storage.workflow", |_, _| async { Ok(Value::Null) });
24086 let calls = Arc::new(AtomicUsize::new(0));
24087 let observed = Arc::clone(&calls);
24088 worker.register_query("storage.workflow", "state", move |_, _| {
24089 let calls = Arc::clone(&observed);
24090 async move {
24091 calls.fetch_add(1, Ordering::SeqCst);
24092 if fail {
24093 Err(Error::WorkerLoop("intentional query failure".to_string()))
24094 } else {
24095 Ok(json!({"state":"waiting"}))
24096 }
24097 }
24098 });
24099 assert_eq!(worker.run_once().await.expect("query settled"), 1);
24100 assert_eq!(calls.load(Ordering::SeqCst), 1);
24101 let suffix = if fail { "fail" } else { "complete" };
24102 assert_identical_requests(
24103 &server,
24104 &format!("/api/worker/query-tasks/storage-query/{suffix}"),
24105 8,
24106 );
24107 let other = if fail { "complete" } else { "fail" };
24108 assert_eq!(
24109 server.request_count(&format!("/api/worker/query-tasks/storage-query/{other}")),
24110 0
24111 );
24112 }
24113 }
24114
24115 #[tokio::test]
24116 async fn storage_recovery_does_not_override_auth_lease_or_invalid_contract() {
24117 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
24118 storage_refusals: 7,
24119 storage_path: Some("/poll"),
24120 unauthorized_polls: true,
24121 ..MockWorkerBehavior::default()
24122 });
24123 let error = storage_worker(&server)
24124 .run_once()
24125 .await
24126 .expect_err("auth remains terminal");
24127 assert!(matches!(
24128 error,
24129 Error::Http {
24130 status: reqwest::StatusCode::UNAUTHORIZED,
24131 ..
24132 }
24133 ));
24134 assert_identical_requests(&server, "/api/worker/workflow-tasks/poll", 8);
24135
24136 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
24137 storage_refusals: 7,
24138 storage_path: Some("/activity-cancel/complete"),
24139 ..MockWorkerBehavior::default()
24140 });
24141 let worker =
24142 storage_worker(&server).with_storage_admission(Arc::new(AtomicBool::new(false)));
24143 let error = worker
24144 .client
24145 .complete_activity_task(
24146 "activity-cancel",
24147 "attempt-cancel",
24148 "worker",
24149 json!({}),
24150 DEFAULT_CODEC,
24151 )
24152 .await
24153 .expect_err("cancellation remains terminal");
24154 assert!(activity_task_rejection_is_final(&error));
24155 assert_identical_requests(
24156 &server,
24157 "/api/worker/activity-tasks/activity-cancel/complete",
24158 8,
24159 );
24160
24161 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
24162 storage_refusals: usize::MAX,
24163 storage_path: Some("/poll"),
24164 storage_wrong_poll_id: true,
24165 ..MockWorkerBehavior::default()
24166 });
24167 assert!(storage_worker(&server).run_once().await.is_err());
24168 assert_eq!(server.request_count("/api/worker/workflow-tasks/poll"), 1);
24169 }
24170
24171 #[tokio::test]
24172 async fn storage_pollers_stop_when_the_run_future_is_aborted() {
24173 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
24174 storage_refusals: usize::MAX,
24175 storage_path: Some("/poll"),
24176 ..MockWorkerBehavior::default()
24177 });
24178 let mut worker = storage_worker(&server).retry_policy(WorkerRetryPolicy::default());
24179 worker.register_activity("unused", |_, _| async { Ok(Value::Null) });
24180 let run = tokio::spawn(async move { worker.run().await });
24181 tokio::time::timeout(Duration::from_secs(2), async {
24182 while server.request_count("/api/worker/activity-tasks/poll") == 0 {
24183 tokio::time::sleep(Duration::from_millis(1)).await;
24184 }
24185 })
24186 .await
24187 .expect("poll started");
24188 run.abort();
24189 assert!(run.await.expect_err("cancelled run").is_cancelled());
24190 tokio::time::sleep(Duration::from_millis(250)).await;
24191 assert_eq!(server.request_count("/api/worker/activity-tasks/poll"), 1);
24192 }
24193
24194 #[tokio::test]
24195 async fn query_protocol_rejection_from_older_server_is_typed() {
24196 let server = MockWorkerServer::reject_query_protocol();
24197 let client = Client::builder(server.base_url())
24198 .timeout(Duration::from_secs(2))
24199 .build()
24200 .expect("client");
24201
24202 let error = client
24203 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
24204 .await
24205 .expect_err("server below query protocol floor must reject");
24206 let Error::Protocol(failure) = error else {
24207 panic!("expected typed protocol failure");
24208 };
24209
24210 assert_eq!(failure.status, 400);
24211 assert_eq!(failure.reason, "unsupported_protocol_version");
24212 assert_eq!(failure.supported_version.as_deref(), Some("1.7"));
24213 assert_eq!(
24214 failure.requested_version.as_deref(),
24215 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
24216 );
24217 assert_eq!(
24218 server
24219 .worker_protocol_for("/api/worker/query-tasks/poll")
24220 .as_deref(),
24221 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
24222 );
24223 }
24224
24225 #[tokio::test]
24226 async fn run_once_without_query_handlers_keeps_pre_query_server_compatibility() {
24227 let server = MockWorkerServer::reject_query_protocol();
24228 let client = Client::builder(server.base_url())
24229 .timeout(Duration::from_secs(2))
24230 .build()
24231 .expect("client");
24232 let mut worker = Worker::new(client, "rust-workers")
24233 .worker_id("baseline-worker")
24234 .poll_timeout(Duration::from_millis(10));
24235
24236 worker.register_workflow("baseline.workflow", |_ctx, _input| async move {
24237 Ok(Value::Null)
24238 });
24239
24240 assert_eq!(worker.run_once().await.expect("baseline run once"), 0);
24241 assert_eq!(
24242 server
24243 .worker_protocol_for("/api/worker/workflow-tasks/poll")
24244 .as_deref(),
24245 Some(WORKER_PROTOCOL_VERSION)
24246 );
24247 assert_eq!(
24248 server.worker_protocol_for("/api/worker/query-tasks/poll"),
24249 None,
24250 "a worker without query handlers must not use the query-task endpoint"
24251 );
24252 }
24253
24254 #[tokio::test]
24255 async fn completion_time_query_rejection_is_typed_without_stopping_worker() {
24256 let server = MockWorkerServer::reject_query_completion();
24257 let client = Client::builder(server.base_url())
24258 .timeout(Duration::from_secs(2))
24259 .build()
24260 .expect("client");
24261
24262 let error = client
24263 .complete_query_task("query-late", "late-worker", 1, json!(8), DEFAULT_CODEC)
24264 .await
24265 .expect_err("expired completion must be rejected");
24266 let Error::QueryFailed(failure) = error else {
24267 panic!("expected typed query failure");
24268 };
24269 assert_eq!(failure.status, 409);
24270 assert_eq!(failure.reason, "query_task_timed_out");
24271
24272 let mut worker = Worker::new(client, "rust-workers")
24273 .worker_id("late-worker")
24274 .poll_timeout(Duration::from_millis(10));
24275 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
24276 worker.register_query(
24277 "counter",
24278 "current",
24279 |_ctx, _args| async move { Ok(json!(8)) },
24280 );
24281
24282 assert_eq!(worker.run_once().await.expect("late task is handled"), 1);
24283 assert_eq!(
24284 worker
24285 .run_once()
24286 .await
24287 .expect("worker continues after late completion"),
24288 0
24289 );
24290 assert_eq!(
24291 server.request_count("/api/worker/query-tasks/query-late/complete"),
24292 2
24293 );
24294 assert_eq!(
24295 server.request_count("/api/worker/query-tasks/query-late/fail"),
24296 0,
24297 "a server completion rejection must not be reported as an encoding failure"
24298 );
24299 }
24300
24301 #[tokio::test]
24302 async fn normal_shutdown_joins_pollers_and_deregisters_once() {
24303 let server = MockWorkerServer::start();
24304 let client = Client::builder(server.base_url())
24305 .timeout(Duration::from_secs(2))
24306 .build()
24307 .expect("client");
24308 let mut worker = Worker::new(client, "rust-workers")
24309 .worker_id("joined-worker")
24310 .poll_timeout(Duration::from_millis(10));
24311 worker.register_workflow(
24312 "joined.workflow",
24313 |_ctx, _input| async move { Ok(Value::Null) },
24314 );
24315 worker.register_activity(
24316 "joined.activity",
24317 |_ctx, _input| async move { Ok(Value::Null) },
24318 );
24319 worker.register_query("joined.workflow", "state", |_ctx, _input| async move {
24320 Ok(Value::Null)
24321 });
24322
24323 worker
24324 .run_until(tokio::time::sleep(Duration::from_millis(20)))
24325 .await
24326 .expect("normal shutdown");
24327
24328 let deregistration_path = "/api/worker/registrations/mock-worker";
24329 assert_eq!(server.request_count(deregistration_path), 1);
24330 for poll_path in [
24331 "/api/worker/workflow-tasks/poll",
24332 "/api/worker/activity-tasks/poll",
24333 "/api/worker/query-tasks/poll",
24334 ] {
24335 assert!(server.request_count(poll_path) > 0, "missing {poll_path}");
24336 }
24337 assert_eq!(
24338 server.captured_paths().last().map(String::as_str),
24339 Some(deregistration_path),
24340 "deregistration must start only after every poller has joined"
24341 );
24342 }
24343
24344 #[tokio::test]
24345 async fn registration_failure_does_not_deregister() {
24346 let server = MockWorkerServer::rejected_registration();
24347 let client = Client::builder(server.base_url())
24348 .timeout(Duration::from_secs(2))
24349 .build()
24350 .expect("client");
24351 let worker = Worker::new(client, "rust-workers").worker_id("never-registered");
24352
24353 let error = worker
24354 .run_until(async {})
24355 .await
24356 .expect_err("registration must fail");
24357 assert!(matches!(
24358 error,
24359 Error::Http {
24360 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
24361 ..
24362 }
24363 ));
24364 assert!(server
24365 .captured_paths()
24366 .iter()
24367 .all(|path| !path.starts_with("/api/worker/registrations/")));
24368 }
24369
24370 #[tokio::test]
24371 async fn protocol_116_server_rejects_occurrence_identity_worker_registration() {
24372 let server = MockWorkerServer::rejected_registration_protocol();
24373 let client = Client::builder(server.base_url())
24374 .timeout(Duration::from_secs(2))
24375 .build()
24376 .expect("client");
24377 let worker = Worker::new(client, "rust-workers").worker_id("protocol-117-worker");
24378
24379 let error = worker
24380 .run_until(async {})
24381 .await
24382 .expect_err("a protocol 1.16 server must reject this worker");
24383 let Error::Protocol(failure) = error else {
24384 panic!("expected typed protocol rejection");
24385 };
24386 assert_eq!(failure.reason, "unsupported_protocol_version");
24387 assert_eq!(failure.supported_version.as_deref(), Some("1.16"));
24388 assert_eq!(failure.requested_version.as_deref(), Some("1.17"));
24389 assert_eq!(
24390 server
24391 .worker_protocol_for("/api/worker/register")
24392 .as_deref(),
24393 Some(WORKER_PROTOCOL_VERSION)
24394 );
24395 }
24396
24397 #[tokio::test]
24398 async fn declined_registration_does_not_deregister() {
24399 let server = MockWorkerServer::declined_registration();
24400 let client = Client::builder(server.base_url())
24401 .timeout(Duration::from_secs(2))
24402 .build()
24403 .expect("client");
24404 let worker = Worker::new(client, "rust-workers").worker_id("declined-worker");
24405
24406 let error = worker
24407 .run_until(async {})
24408 .await
24409 .expect_err("declined registration must fail");
24410 assert!(matches!(error, Error::WorkerLoop(_)));
24411 assert!(error.to_string().contains("was not accepted"));
24412 assert!(server
24413 .captured_paths()
24414 .iter()
24415 .all(|path| !path.starts_with("/api/worker/registrations/")));
24416 }
24417
24418 #[tokio::test]
24419 async fn deregistration_http_failure_is_returned_after_normal_shutdown() {
24420 let server = MockWorkerServer::rejected_deregistration();
24421 let client = Client::builder(server.base_url())
24422 .timeout(Duration::from_secs(2))
24423 .build()
24424 .expect("client");
24425 let worker = Worker::new(client, "rust-workers").worker_id("forbidden-cleanup");
24426
24427 let error = worker
24428 .run_until(async {})
24429 .await
24430 .expect_err("deregistration must fail");
24431 assert!(matches!(
24432 error,
24433 Error::Http {
24434 status: reqwest::StatusCode::FORBIDDEN,
24435 ..
24436 }
24437 ));
24438 assert_eq!(
24439 server.request_count("/api/worker/registrations/mock-worker"),
24440 1
24441 );
24442 }
24443
24444 #[tokio::test]
24445 async fn deregistration_protocol_failure_is_returned_after_normal_shutdown() {
24446 let server = MockWorkerServer::rejected_deregistration_protocol();
24447 let client = Client::builder(server.base_url())
24448 .timeout(Duration::from_secs(2))
24449 .build()
24450 .expect("client");
24451 let worker = Worker::new(client, "rust-workers").worker_id("protocol-cleanup");
24452
24453 let error = worker
24454 .run_until(async {})
24455 .await
24456 .expect_err("protocol rejection must fail shutdown");
24457 let Error::Protocol(failure) = error else {
24458 panic!("expected typed protocol failure");
24459 };
24460 assert_eq!(failure.reason, "unsupported_protocol_version");
24461 assert_eq!(
24462 failure.requested_version.as_deref(),
24463 Some(WORKER_PROTOCOL_VERSION)
24464 );
24465 assert_eq!(
24466 server.request_count("/api/worker/registrations/mock-worker"),
24467 1
24468 );
24469 }
24470
24471 #[tokio::test]
24472 async fn primary_poller_error_retains_deregistration_failure_context() {
24473 let server = MockWorkerServer::unauthorized_polls_and_rejected_deregistration();
24474 let client = Client::builder(server.base_url())
24475 .timeout(Duration::from_secs(2))
24476 .build()
24477 .expect("client");
24478 let mut worker = Worker::new(client, "rust-workers")
24479 .worker_id("combined-failure")
24480 .poll_timeout(Duration::from_millis(10));
24481 worker.register_workflow("combined.workflow", |_ctx, _input| async move {
24482 Ok(Value::Null)
24483 });
24484
24485 let error = worker
24486 .run()
24487 .await
24488 .expect_err("worker and cleanup must fail");
24489 let summary = error.to_string();
24490 assert!(summary.contains("authentication_failed"));
24491 assert!(summary.contains("worker cannot deregister"));
24492 let Error::WorkerShutdown {
24493 primary,
24494 deregistration,
24495 } = error
24496 else {
24497 panic!("expected combined worker shutdown error");
24498 };
24499 assert!(matches!(
24500 *primary,
24501 Error::Http {
24502 status: reqwest::StatusCode::UNAUTHORIZED,
24503 ..
24504 }
24505 ));
24506 assert!(matches!(
24507 *deregistration,
24508 Error::Http {
24509 status: reqwest::StatusCode::FORBIDDEN,
24510 ..
24511 }
24512 ));
24513 assert_eq!(
24514 server.request_count("/api/worker/registrations/mock-worker"),
24515 1
24516 );
24517 }
24518
24519 #[tokio::test]
24520 async fn activity_only_worker_can_shutdown_without_workflow_poller() {
24521 let server = MockWorkerServer::start();
24522 let client = Client::builder(server.base_url())
24523 .timeout(Duration::from_secs(2))
24524 .build()
24525 .expect("client");
24526 let mut worker = Worker::new(client, "rust-workers")
24527 .worker_id("activity-only-worker")
24528 .poll_timeout(Duration::from_millis(10));
24529
24530 worker.register_activity(
24531 "activity.only",
24532 |_ctx, _args| async move { Ok(Value::Null) },
24533 );
24534
24535 worker.run_until(async {}).await.expect("run worker");
24536 }
24537
24538 #[tokio::test]
24539 async fn workflow_only_worker_can_shutdown_without_activity_poller() {
24540 let server = MockWorkerServer::start();
24541 let client = Client::builder(server.base_url())
24542 .timeout(Duration::from_secs(2))
24543 .build()
24544 .expect("client");
24545 let mut worker = Worker::new(client, "rust-workers")
24546 .worker_id("workflow-only-worker")
24547 .poll_timeout(Duration::from_millis(10));
24548
24549 worker.register_workflow(
24550 "workflow.only",
24551 |_ctx, _input| async move { Ok(Value::Null) },
24552 );
24553
24554 worker.run_until(async {}).await.expect("run worker");
24555 }
24556
24557 #[tokio::test]
24558 async fn worker_heartbeat_observer_receives_server_acknowledgements() {
24559 let server = MockWorkerServer::start();
24560 let client = Client::builder(server.base_url())
24561 .timeout(Duration::from_secs(2))
24562 .build()
24563 .expect("client");
24564 let observations = Arc::new(Mutex::new(Vec::new()));
24565 let observed = Arc::clone(&observations);
24566 let mut worker = Worker::new(client, "rust-workers")
24567 .worker_id("observed-heartbeat-worker")
24568 .poll_timeout(Duration::from_millis(10))
24569 .on_worker_heartbeat(move |observation| {
24570 observed
24571 .lock()
24572 .expect("heartbeat observations")
24573 .push(observation.clone());
24574 });
24575
24576 worker.register_workflow("workflow.observed", |_ctx, _input| async move {
24577 Ok(Value::Null)
24578 });
24579 let acknowledged = Arc::clone(&observations);
24580 worker
24581 .run_until(async move {
24582 tokio::time::timeout(Duration::from_secs(2), async move {
24583 loop {
24584 if !acknowledged
24585 .lock()
24586 .expect("heartbeat observations")
24587 .is_empty()
24588 {
24589 break;
24590 }
24591 tokio::time::sleep(Duration::from_millis(1)).await;
24592 }
24593 })
24594 .await
24595 .expect("heartbeat acknowledgement within timeout");
24596 })
24597 .await
24598 .expect("run worker");
24599
24600 let observations = observations.lock().expect("heartbeat observations");
24601 let first = observations.first().expect("heartbeat acknowledgement");
24602 assert_eq!(first.worker_id, "observed-heartbeat-worker");
24603 assert_eq!(first.task_queue, "rust-workers");
24604 assert!(first.acknowledged_at_unix_millis > 0);
24605 assert_eq!(first.acknowledgement, json!({}));
24606 }
24607
24608 #[tokio::test]
24609 async fn delayed_worker_heartbeat_keeps_cadence_and_pollers_live() {
24610 let server = MockWorkerServer::delayed_heartbeat_worker();
24611 let client = Client::builder(server.base_url())
24612 .timeout(Duration::from_secs(3))
24613 .build()
24614 .expect("client");
24615 let observations = Arc::new(Mutex::new(Vec::new()));
24616 let observed = Arc::clone(&observations);
24617 let mut worker = Worker::new(client, "rust-snapshot-workers")
24618 .worker_id("rust-snapshot-worker")
24619 .poll_timeout(Duration::from_millis(10))
24620 .on_worker_heartbeat(move |observation| {
24621 observed
24622 .lock()
24623 .expect("heartbeat observations")
24624 .push(observation.clone());
24625 });
24626
24627 worker.register_workflow("snapshot", |ctx, _input| async move {
24628 ctx.wait_signal("finish").await?;
24629 Ok(json!({"status": "finished"}))
24630 });
24631 worker.register_query("snapshot", "current", |ctx, _args| async move {
24632 Ok(json!(ctx
24633 .signals("increment")
24634 .iter()
24635 .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
24636 .sum::<i64>()))
24637 });
24638 worker.register_activity("cancel-aware", |_ctx, _args| async move {
24639 Ok(json!({"late": "completion"}))
24640 });
24641
24642 worker
24643 .run_until(tokio::time::sleep(Duration::from_millis(3_800)))
24644 .await
24645 .expect("delayed heartbeat must allow a clean worker shutdown");
24646
24647 let observations = observations.lock().expect("heartbeat observations");
24648 assert!(
24649 observations.len() >= 3,
24650 "the immediate heartbeat, delayed acknowledgement, and next cadence heartbeat must complete"
24651 );
24652 assert!(
24653 observations.windows(2).all(|pair| {
24654 pair[1].acknowledged_at_unix_millis
24655 .saturating_sub(pair[0].acknowledged_at_unix_millis)
24656 >= 850
24657 }),
24658 "successful acknowledgements must not catch up faster than the advertised one-second cadence: {observations:?}"
24659 );
24660 drop(observations);
24661
24662 let heartbeat_times = server.request_times("/api/worker/heartbeat");
24663 let delayed_request_at = *heartbeat_times
24664 .get(1)
24665 .expect("intentionally delayed heartbeat request");
24666 let delay_window_start = delayed_request_at + Duration::from_millis(100);
24667 let delay_window_end = delayed_request_at + Duration::from_millis(1_400);
24668 for path in [
24669 "/api/worker/workflow-tasks/poll",
24670 "/api/worker/activity-tasks/poll",
24671 "/api/worker/query-tasks/poll",
24672 ] {
24673 assert!(
24674 server
24675 .request_times(path)
24676 .iter()
24677 .any(|received_at| *received_at >= delay_window_start
24678 && *received_at <= delay_window_end),
24679 "{path} must keep polling while a heartbeat acknowledgement is delayed"
24680 );
24681 }
24682 assert!(
24683 server.request_count("/api/worker/workflow-tasks/snapshot-wait-3/fail") >= 1,
24684 "workflow work must be settled"
24685 );
24686 assert!(
24687 server.request_count("/api/worker/activity-tasks/activity-cancel/complete") >= 1,
24688 "activity work must be settled"
24689 );
24690 assert!(
24691 server.request_count("/api/worker/query-tasks/snapshot-current/complete") >= 1,
24692 "query work must be settled"
24693 );
24694 }
24695
24696 #[tokio::test]
24697 async fn retried_worker_heartbeat_restarts_the_advertised_cadence() {
24698 let server = MockWorkerServer::heartbeat_retry_worker();
24699 let client = Client::builder(server.base_url())
24700 .timeout(Duration::from_secs(2))
24701 .build()
24702 .expect("client");
24703 let observations = Arc::new(Mutex::new(Vec::new()));
24704 let observed = Arc::clone(&observations);
24705 let worker = Worker::new(client, "rust-workers")
24706 .worker_id("heartbeat-retry-worker")
24707 .retry_policy(WorkerRetryPolicy {
24708 max_retries: 1,
24709 initial_backoff: Duration::from_millis(300),
24710 max_backoff: Duration::from_millis(300),
24711 })
24712 .on_worker_heartbeat(move |observation| {
24713 observed
24714 .lock()
24715 .expect("heartbeat observations")
24716 .push(observation.clone());
24717 });
24718
24719 worker
24720 .run_until(tokio::time::sleep(Duration::from_millis(2_700)))
24721 .await
24722 .expect("retryable heartbeat failure must remain bounded and recover");
24723
24724 let observations = observations.lock().expect("heartbeat observations");
24725 assert!(observations.len() >= 3, "heartbeat retry must recover");
24726 assert!(
24727 observations.windows(2).all(|pair| {
24728 pair[1]
24729 .acknowledged_at_unix_millis
24730 .saturating_sub(pair[0].acknowledged_at_unix_millis)
24731 >= 850
24732 }),
24733 "a successful retry must start a fresh advertised cadence: {observations:?}"
24734 );
24735 assert_eq!(
24736 server.request_count("/api/worker/heartbeat"),
24737 observations.len() + 1,
24738 "one retryable failure must add exactly one bounded request"
24739 );
24740 }
24741
24742 #[tokio::test]
24743 async fn query_enabled_worker_ignores_unmatched_signals_then_completes_once() {
24744 let server = MockWorkerServer::waiting_query_worker();
24745 let client = Client::builder(server.base_url())
24746 .timeout(Duration::from_secs(2))
24747 .build()
24748 .expect("client");
24749 let observations = Arc::new(Mutex::new(Vec::new()));
24750 let observed = Arc::clone(&observations);
24751 let mut worker = Worker::new(client, "rust-snapshot-workers")
24752 .worker_id("rust-snapshot-worker")
24753 .poll_timeout(Duration::from_millis(10))
24754 .on_worker_heartbeat(move |observation| {
24755 observed
24756 .lock()
24757 .expect("heartbeat observations")
24758 .push(observation.clone());
24759 });
24760
24761 worker.register_workflow("snapshot", |ctx, _input| async move {
24762 ctx.wait_signal("finish").await?;
24763 Ok(json!({"status": "finished"}))
24764 });
24765 worker.register_query("snapshot", "current", |ctx, _args| async move {
24766 let current = ctx
24767 .signals("increment")
24768 .iter()
24769 .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
24770 .sum::<i64>();
24771 Ok(json!(current))
24772 });
24773 worker.register_update("snapshot", "replace", |_ctx, args| async move { Ok(args) });
24774
24775 worker
24776 .run_until(tokio::time::sleep(Duration::from_millis(3_200)))
24777 .await
24778 .expect("pending workflow and query poller must remain live until shutdown");
24779
24780 assert!(
24781 observations.lock().expect("heartbeat observations").len() >= 4,
24782 "the immediate heartbeat and at least three advertised one-second intervals must be acknowledged"
24783 );
24784 assert!(
24785 server.request_count("/api/worker/workflow-tasks/poll") >= 3,
24786 "workflow polling must continue after empty replay acknowledgements"
24787 );
24788 assert!(
24789 server.request_count("/api/worker/query-tasks/poll") >= 2,
24790 "query polling must continue after serving the current query"
24791 );
24792 assert_eq!(
24793 server.request_body("/api/worker/register")["capabilities"],
24794 json!([
24795 CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY,
24796 DURABLE_SELECTION_CAPABILITY,
24797 MEMO_UPSERTS_CAPABILITY,
24798 TYPED_SEARCH_ATTRIBUTES_CAPABILITY,
24799 QUERY_TASKS_CAPABILITY,
24800 WORKFLOW_UPDATES_CAPABILITY,
24801 MESSAGE_STREAMS_CAPABILITY
24802 ])
24803 );
24804 assert_eq!(
24805 server.request_body("/api/worker/register")["workflow_command_contracts"]["snapshot"],
24806 json!({
24807 "queries": ["current"],
24808 "query_contracts": [],
24809 "signals": [],
24810 "signal_contracts": [],
24811 "updates": ["replace"],
24812 "update_contracts": [],
24813 "update_validators": [],
24814 })
24815 );
24816
24817 let opened = server.request_body("/api/worker/workflow-tasks/snapshot-open/complete");
24818 assert_eq!(
24819 opened["commands"],
24820 json!([{
24821 "type": "open_signal_wait",
24822 "signal_name": "finish",
24823 }])
24824 );
24825
24826 for task_id in ["snapshot-wait-3", "snapshot-wait-5"] {
24827 let fail_path = format!("/api/worker/workflow-tasks/{task_id}/fail");
24828 let completion_path = format!("/api/worker/workflow-tasks/{task_id}/complete");
24829 let failure = server.request_body(&fail_path);
24830 assert_eq!(
24831 failure["failure"]["type"],
24832 WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE
24833 );
24834 assert_eq!(server.request_count(&completion_path), 0);
24835 }
24836
24837 let query_completion =
24838 server.request_body("/api/worker/query-tasks/snapshot-current/complete");
24839 assert_eq!(query_completion["result"], json!(8));
24840
24841 let terminal_path = "/api/worker/workflow-tasks/snapshot-finish/complete";
24842 assert_eq!(
24843 server.request_count(terminal_path),
24844 1,
24845 "the matching signal must settle the workflow exactly once"
24846 );
24847 let terminal = server.request_body(terminal_path);
24848 assert_eq!(terminal["commands"].as_array().map(Vec::len), Some(1));
24849 assert_eq!(terminal["commands"][0]["type"], "complete_workflow");
24850 assert_eq!(
24851 decode_wire_value(&terminal["commands"][0]["result"], DEFAULT_CODEC)
24852 .expect("terminal workflow result"),
24853 json!({"status": "finished"})
24854 );
24855 }
24856
24857 #[tokio::test]
24858 async fn worker_retries_poll_and_heartbeat_transport_failures_independently() {
24859 let server = MockWorkerServer::transient_worker_failures();
24860 let client = Client::builder(server.base_url())
24861 .timeout(Duration::from_secs(2))
24862 .build()
24863 .expect("client");
24864 let mut worker = Worker::new(client, "rust-workers")
24865 .worker_id("retry-worker")
24866 .poll_timeout(Duration::from_millis(10))
24867 .retry_policy(WorkerRetryPolicy {
24868 max_retries: 2,
24869 initial_backoff: Duration::from_millis(1),
24870 max_backoff: Duration::from_millis(1),
24871 });
24872 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
24873 worker.register_activity(
24874 "counter.activity",
24875 |_ctx, _input| async move { Ok(Value::Null) },
24876 );
24877 worker.register_query(
24878 "counter",
24879 "current",
24880 |_ctx, _args| async move { Ok(json!(8)) },
24881 );
24882
24883 worker
24884 .run_until(tokio::time::sleep(Duration::from_millis(75)))
24885 .await
24886 .expect("transient failures must not stop the worker");
24887
24888 for path in [
24889 "/api/worker/heartbeat",
24890 "/api/worker/workflow-tasks/poll",
24891 "/api/worker/activity-tasks/poll",
24892 "/api/worker/query-tasks/poll",
24893 ] {
24894 assert!(
24895 server.request_count(path) >= 2,
24896 "{path} must continue after its transient failure"
24897 );
24898 }
24899 }
24900
24901 #[tokio::test]
24902 async fn worker_continues_after_long_poll_capacity_backpressure() {
24903 let server = MockWorkerServer::capacity_limited_activity_poll();
24904 let client = Client::builder(server.base_url())
24905 .timeout(Duration::from_secs(2))
24906 .build()
24907 .expect("client");
24908 let mut worker = Worker::new(client, "rust-workers")
24909 .worker_id("capacity-worker")
24910 .poll_timeout(Duration::from_millis(10))
24911 .retry_policy(WorkerRetryPolicy {
24912 max_retries: 0,
24913 initial_backoff: Duration::from_millis(1),
24914 max_backoff: Duration::from_millis(1),
24915 });
24916 worker.register_activity("capacity.activity", |_ctx, _input| async move {
24917 Ok(json!({"handled": true}))
24918 });
24919
24920 worker
24921 .run_until(tokio::time::sleep(Duration::from_millis(50)))
24922 .await
24923 .expect("capacity backpressure must not stop the worker");
24924
24925 assert!(
24926 server.request_count("/api/worker/activity-tasks/poll") >= 2,
24927 "the activity poller must continue after capacity backpressure"
24928 );
24929 assert_eq!(
24930 server.request_count("/api/worker/activity-tasks/capacity-activity/complete"),
24931 1,
24932 "the worker must complete work returned after capacity recovers"
24933 );
24934 }
24935
24936 #[test]
24937 fn worker_poll_capacity_backpressure_requires_the_typed_retryable_contract() {
24938 let capacity = Error::Http {
24939 status: reqwest::StatusCode::TOO_MANY_REQUESTS,
24940 body: r#"{"poll_status":"long_poll_capacity_exhausted","retryable":true,"retry_after_seconds":3}"#.to_string(),
24941 };
24942 assert_eq!(
24943 worker_poll_capacity_retry_after(&capacity),
24944 Some(Duration::from_secs(3))
24945 );
24946
24947 let rejected_capacity = Error::Http {
24948 status: reqwest::StatusCode::TOO_MANY_REQUESTS,
24949 body: r#"{"reason":"long_poll_capacity_exhausted","retryable":false,"retry_after_seconds":3}"#.to_string(),
24950 };
24951 assert_eq!(worker_poll_capacity_retry_after(&rejected_capacity), None);
24952 assert!(!worker_operation_is_retryable(&rejected_capacity));
24953
24954 let ordinary_rate_limit = Error::Http {
24955 status: reqwest::StatusCode::TOO_MANY_REQUESTS,
24956 body: r#"{"reason":"rate_limited","retryable":true,"retry_after_seconds":3}"#
24957 .to_string(),
24958 };
24959 assert_eq!(worker_poll_capacity_retry_after(&ordinary_rate_limit), None);
24960 assert!(worker_operation_is_retryable(&ordinary_rate_limit));
24961 }
24962
24963 #[tokio::test]
24964 async fn worker_bounds_transport_retries() {
24965 let server = MockWorkerServer::unavailable_polls();
24966 let client = Client::builder(server.base_url())
24967 .timeout(Duration::from_secs(2))
24968 .build()
24969 .expect("client");
24970 let mut worker = Worker::new(client, "rust-workers")
24971 .worker_id("bounded-retry-worker")
24972 .poll_timeout(Duration::from_millis(10))
24973 .retry_policy(WorkerRetryPolicy {
24974 max_retries: 2,
24975 initial_backoff: Duration::from_millis(1),
24976 max_backoff: Duration::from_millis(1),
24977 });
24978 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
24979
24980 let error = worker.run().await.expect_err("retry bound must terminate");
24981 assert!(matches!(error, Error::Transport(_)));
24982 assert_eq!(
24983 server.request_count("/api/worker/workflow-tasks/poll"),
24984 3,
24985 "one initial request plus exactly two retries"
24986 );
24987 }
24988
24989 #[tokio::test]
24990 async fn worker_retry_policy_can_disable_poll_retries() {
24991 let server = MockWorkerServer::unavailable_polls();
24992 let client = Client::builder(server.base_url())
24993 .timeout(Duration::from_secs(2))
24994 .build()
24995 .expect("client");
24996 let mut worker = Worker::new(client, "rust-workers")
24997 .worker_id("no-retry-worker")
24998 .poll_timeout(Duration::from_millis(10))
24999 .retry_policy(WorkerRetryPolicy {
25000 max_retries: 0,
25001 initial_backoff: Duration::from_millis(1),
25002 max_backoff: Duration::from_millis(1),
25003 });
25004 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
25005
25006 let error = worker
25007 .run_once()
25008 .await
25009 .expect_err("disabled retries must return the first transport failure");
25010 assert!(matches!(error, Error::Transport(_)));
25011 assert_eq!(
25012 server.request_count("/api/worker/workflow-tasks/poll"),
25013 1,
25014 "max_retries=0 must send only the initial request"
25015 );
25016 }
25017
25018 #[tokio::test]
25019 async fn worker_does_not_retry_authentication_failures() {
25020 let server = MockWorkerServer::unauthorized_polls();
25021 let client = Client::builder(server.base_url())
25022 .timeout(Duration::from_secs(2))
25023 .build()
25024 .expect("client");
25025 let mut worker = Worker::new(client, "rust-workers")
25026 .worker_id("unauthorized-worker")
25027 .poll_timeout(Duration::from_millis(10));
25028 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
25029
25030 let error = worker
25031 .run()
25032 .await
25033 .expect_err("authentication must terminate");
25034 let Error::Http { status, body } = error else {
25035 panic!("expected stable HTTP authentication error");
25036 };
25037 assert_eq!(status, reqwest::StatusCode::UNAUTHORIZED);
25038 assert!(body.contains("authentication_failed"));
25039 assert_eq!(
25040 server.request_count("/api/worker/workflow-tasks/poll"),
25041 1,
25042 "authentication failures must not be retried"
25043 );
25044 }
25045
25046 #[derive(Clone, Debug)]
25047 struct CapturedRequest {
25048 headers: String,
25049 method: String,
25050 path: String,
25051 authorization: Option<String>,
25052 namespace: Option<String>,
25053 worker_protocol: Option<String>,
25054 control_protocol: Option<String>,
25055 body: String,
25056 received_at: Instant,
25057 }
25058
25059 struct MockWorkerServer {
25060 addr: SocketAddr,
25061 stop: Arc<AtomicBool>,
25062 requests: Arc<Mutex<Vec<CapturedRequest>>>,
25063 thread: Option<thread::JoinHandle<()>>,
25064 }
25065
25066 type RequestOverride = fn(&str, &str, usize) -> Option<(&'static str, String)>;
25067
25068 #[derive(Clone, Copy, Default)]
25069 struct MockWorkerBehavior {
25070 response_override: Option<fn(&str) -> Option<(&'static str, String)>>,
25071 request_override: Option<RequestOverride>,
25072 storage_refusals: usize,
25073 storage_path: Option<&'static str>,
25074 storage_unavailable: bool,
25075 storage_mid_poll: bool,
25076 storage_activity: bool,
25077 storage_query: bool,
25078 storage_wrong_poll_id: bool,
25079 reject_query_protocol: bool,
25080 reject_query_completion: bool,
25081 waiting_query_worker: bool,
25082 decline_registration: bool,
25083 complete_named_signal: bool,
25084 poll_failures_per_path: usize,
25085 long_poll_capacity_responses_per_path: usize,
25086 heartbeat_failures: usize,
25087 heartbeat_failure_request: Option<usize>,
25088 delayed_heartbeat_request: Option<usize>,
25089 heartbeat_response_delay: Duration,
25090 concurrent_requests: bool,
25091 unauthorized_polls: bool,
25092 reject_registration: bool,
25093 reject_registration_protocol: bool,
25094 reject_deregistration: bool,
25095 reject_deregistration_protocol: bool,
25096 cancelled_activity: bool,
25097 draining_polls: bool,
25098 invalid_task_payload_codec: Option<InvalidTaskPayloadCodec>,
25099 workflow_completion_status: Option<&'static str>,
25100 workflow_completion_body: Option<&'static str>,
25101 }
25102
25103 impl MockWorkerServer {
25104 fn start() -> Self {
25105 Self::start_with_behavior(MockWorkerBehavior::default())
25106 }
25107
25108 fn reject_query_protocol() -> Self {
25109 Self::start_with_behavior(MockWorkerBehavior {
25110 reject_query_protocol: true,
25111 ..MockWorkerBehavior::default()
25112 })
25113 }
25114
25115 fn reject_query_completion() -> Self {
25116 Self::start_with_behavior(MockWorkerBehavior {
25117 reject_query_completion: true,
25118 ..MockWorkerBehavior::default()
25119 })
25120 }
25121
25122 fn waiting_query_worker() -> Self {
25123 Self::start_with_behavior(MockWorkerBehavior {
25124 waiting_query_worker: true,
25125 complete_named_signal: true,
25126 ..MockWorkerBehavior::default()
25127 })
25128 }
25129
25130 fn transient_worker_failures() -> Self {
25131 Self::start_with_behavior(MockWorkerBehavior {
25132 poll_failures_per_path: 1,
25133 heartbeat_failures: 1,
25134 ..MockWorkerBehavior::default()
25135 })
25136 }
25137
25138 fn consecutive_poll_failures(count: usize) -> Self {
25139 Self::start_with_behavior(MockWorkerBehavior {
25140 poll_failures_per_path: count,
25141 ..MockWorkerBehavior::default()
25142 })
25143 }
25144
25145 fn capacity_limited_activity_poll() -> Self {
25146 Self::start_with_behavior(MockWorkerBehavior {
25147 long_poll_capacity_responses_per_path: 1,
25148 ..MockWorkerBehavior::default()
25149 })
25150 }
25151
25152 fn delayed_heartbeat_worker() -> Self {
25153 Self::start_with_behavior(MockWorkerBehavior {
25154 waiting_query_worker: true,
25155 delayed_heartbeat_request: Some(2),
25156 heartbeat_response_delay: Duration::from_millis(1_500),
25157 concurrent_requests: true,
25158 cancelled_activity: true,
25159 ..MockWorkerBehavior::default()
25160 })
25161 }
25162
25163 fn heartbeat_retry_worker() -> Self {
25164 Self::start_with_behavior(MockWorkerBehavior {
25165 waiting_query_worker: true,
25166 heartbeat_failure_request: Some(2),
25167 concurrent_requests: true,
25168 ..MockWorkerBehavior::default()
25169 })
25170 }
25171
25172 fn unavailable_polls() -> Self {
25173 Self::start_with_behavior(MockWorkerBehavior {
25174 poll_failures_per_path: usize::MAX,
25175 ..MockWorkerBehavior::default()
25176 })
25177 }
25178
25179 fn unauthorized_polls() -> Self {
25180 Self::start_with_behavior(MockWorkerBehavior {
25181 unauthorized_polls: true,
25182 ..MockWorkerBehavior::default()
25183 })
25184 }
25185
25186 fn rejected_registration() -> Self {
25187 Self::start_with_behavior(MockWorkerBehavior {
25188 reject_registration: true,
25189 ..MockWorkerBehavior::default()
25190 })
25191 }
25192
25193 fn rejected_registration_protocol() -> Self {
25194 Self::start_with_behavior(MockWorkerBehavior {
25195 reject_registration_protocol: true,
25196 ..MockWorkerBehavior::default()
25197 })
25198 }
25199
25200 fn declined_registration() -> Self {
25201 Self::start_with_behavior(MockWorkerBehavior {
25202 decline_registration: true,
25203 ..MockWorkerBehavior::default()
25204 })
25205 }
25206
25207 fn rejected_deregistration() -> Self {
25208 Self::start_with_behavior(MockWorkerBehavior {
25209 reject_deregistration: true,
25210 ..MockWorkerBehavior::default()
25211 })
25212 }
25213
25214 fn rejected_deregistration_protocol() -> Self {
25215 Self::start_with_behavior(MockWorkerBehavior {
25216 reject_deregistration_protocol: true,
25217 ..MockWorkerBehavior::default()
25218 })
25219 }
25220
25221 fn unauthorized_polls_and_rejected_deregistration() -> Self {
25222 Self::start_with_behavior(MockWorkerBehavior {
25223 unauthorized_polls: true,
25224 reject_deregistration: true,
25225 ..MockWorkerBehavior::default()
25226 })
25227 }
25228
25229 fn cancelled_activity() -> Self {
25230 Self::start_with_behavior(MockWorkerBehavior {
25231 cancelled_activity: true,
25232 ..MockWorkerBehavior::default()
25233 })
25234 }
25235
25236 fn draining_polls() -> Self {
25237 Self::start_with_behavior(MockWorkerBehavior {
25238 draining_polls: true,
25239 ..MockWorkerBehavior::default()
25240 })
25241 }
25242
25243 fn invalid_task_payload_codec(codec: InvalidTaskPayloadCodec) -> Self {
25244 Self::start_with_behavior(MockWorkerBehavior {
25245 invalid_task_payload_codec: Some(codec),
25246 ..MockWorkerBehavior::default()
25247 })
25248 }
25249
25250 fn workflow_completion(status: &'static str, body: &'static str) -> Self {
25251 Self::start_with_behavior(MockWorkerBehavior {
25252 workflow_completion_status: Some(status),
25253 workflow_completion_body: Some(body),
25254 ..MockWorkerBehavior::default()
25255 })
25256 }
25257
25258 fn start_with_behavior(behavior: MockWorkerBehavior) -> Self {
25259 let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock server");
25260 listener
25261 .set_nonblocking(true)
25262 .expect("configure mock listener");
25263 let addr = listener.local_addr().expect("mock server address");
25264 let stop = Arc::new(AtomicBool::new(false));
25265 let server_stop = Arc::clone(&stop);
25266 let requests = Arc::new(Mutex::new(Vec::new()));
25267 let server_requests = Arc::clone(&requests);
25268 let thread = thread::spawn(move || {
25269 let mut request_threads = Vec::new();
25270 while !server_stop.load(Ordering::SeqCst) {
25271 match listener.accept() {
25272 Ok((mut stream, _)) => {
25273 if behavior.concurrent_requests {
25274 let requests = Arc::clone(&server_requests);
25275 request_threads.push(thread::spawn(move || {
25276 handle_mock_worker_request(&mut stream, &requests, behavior)
25277 }));
25278 } else {
25279 handle_mock_worker_request(&mut stream, &server_requests, behavior);
25280 }
25281 }
25282 Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
25283 let mut index = 0;
25284 while index < request_threads.len() {
25285 if request_threads[index].is_finished() {
25286 request_threads
25287 .swap_remove(index)
25288 .join()
25289 .expect("join mock request");
25290 } else {
25291 index += 1;
25292 }
25293 }
25294 thread::sleep(Duration::from_millis(5));
25295 }
25296 Err(_) => break,
25297 }
25298 }
25299 for request_thread in request_threads {
25300 request_thread.join().expect("join mock request");
25301 }
25302 });
25303
25304 Self {
25305 addr,
25306 stop,
25307 requests,
25308 thread: Some(thread),
25309 }
25310 }
25311
25312 fn base_url(&self) -> String {
25313 format!("http://{}", self.addr)
25314 }
25315
25316 fn worker_protocol_for(&self, path: &str) -> Option<String> {
25317 self.requests
25318 .lock()
25319 .expect("captured requests")
25320 .iter()
25321 .find(|request| request.path == path)
25322 .and_then(|request| request.worker_protocol.clone())
25323 }
25324
25325 fn control_protocol_for(&self, path: &str) -> Option<String> {
25326 self.requests
25327 .lock()
25328 .expect("captured requests")
25329 .iter()
25330 .find(|request| request.path == path)
25331 .and_then(|request| request.control_protocol.clone())
25332 }
25333
25334 fn method_for(&self, path: &str) -> Option<String> {
25335 self.requests
25336 .lock()
25337 .expect("captured requests")
25338 .iter()
25339 .find(|request| request.path == path)
25340 .map(|request| request.method.clone())
25341 }
25342
25343 fn authorization_for(&self, path: &str) -> Option<String> {
25344 self.requests
25345 .lock()
25346 .expect("captured requests")
25347 .iter()
25348 .find(|request| request.path == path)
25349 .and_then(|request| request.authorization.clone())
25350 }
25351
25352 fn namespace_for(&self, path: &str) -> Option<String> {
25353 self.requests
25354 .lock()
25355 .expect("captured requests")
25356 .iter()
25357 .find(|request| request.path == path)
25358 .and_then(|request| request.namespace.clone())
25359 }
25360
25361 fn request_count(&self, path: &str) -> usize {
25362 self.requests
25363 .lock()
25364 .expect("captured requests")
25365 .iter()
25366 .filter(|request| request.path == path)
25367 .count()
25368 }
25369
25370 fn captured_paths(&self) -> Vec<String> {
25371 self.requests
25372 .lock()
25373 .expect("captured requests")
25374 .iter()
25375 .map(|request| request.path.clone())
25376 .collect()
25377 }
25378
25379 fn request_times(&self, path: &str) -> Vec<Instant> {
25380 self.requests
25381 .lock()
25382 .expect("captured requests")
25383 .iter()
25384 .filter(|request| request.path == path)
25385 .map(|request| request.received_at)
25386 .collect()
25387 }
25388
25389 fn request_body(&self, path: &str) -> Value {
25390 let requests = self.requests.lock().expect("captured requests");
25391 let body = &requests
25392 .iter()
25393 .find(|request| request.path == path)
25394 .unwrap_or_else(|| panic!("missing request for {path}"))
25395 .body;
25396 serde_json::from_str(body).unwrap_or_else(|error| {
25397 panic!("invalid JSON request body for {path}: {error}: {body:?}")
25398 })
25399 }
25400
25401 fn request_bodies(&self, path: &str) -> Vec<Value> {
25402 self.requests
25403 .lock()
25404 .expect("captured requests")
25405 .iter()
25406 .filter(|request| request.path == path)
25407 .map(|request| {
25408 serde_json::from_str(&request.body).unwrap_or_else(|error| {
25409 panic!(
25410 "invalid JSON request body for {path}: {error}: {:?}",
25411 request.body
25412 )
25413 })
25414 })
25415 .collect()
25416 }
25417 }
25418
25419 impl Drop for MockWorkerServer {
25420 fn drop(&mut self) {
25421 self.stop.store(true, Ordering::SeqCst);
25422 let _ = TcpStream::connect(self.addr);
25423
25424 if let Some(thread) = self.thread.take() {
25425 thread.join().expect("join mock server");
25426 }
25427 }
25428 }
25429
25430 fn handle_mock_worker_request(
25431 stream: &mut TcpStream,
25432 requests: &Arc<Mutex<Vec<CapturedRequest>>>,
25433 behavior: MockWorkerBehavior,
25434 ) {
25435 let _ = stream.set_read_timeout(Some(Duration::from_millis(200)));
25436 let mut buffer = [0_u8; 8192];
25437 let mut request = Vec::new();
25438
25439 loop {
25440 match stream.read(&mut buffer) {
25441 Ok(0) => break,
25442 Ok(read) => {
25443 request.extend_from_slice(&buffer[..read]);
25444 if mock_request_is_complete(&request) {
25445 break;
25446 }
25447 }
25448 Err(error)
25449 if matches!(
25450 error.kind(),
25451 std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
25452 ) =>
25453 {
25454 break;
25455 }
25456 Err(_) => return,
25457 }
25458 }
25459
25460 let request = String::from_utf8_lossy(&request);
25461 let body = request
25462 .split_once("\r\n\r\n")
25463 .map(|(_, body)| body)
25464 .unwrap_or_default();
25465 let path = request
25466 .lines()
25467 .next()
25468 .and_then(|line| line.split_whitespace().nth(1))
25469 .unwrap_or_default();
25470 let method = request
25471 .lines()
25472 .next()
25473 .and_then(|line| line.split_whitespace().next())
25474 .unwrap_or_default();
25475 let authorization = request.lines().find_map(|line| {
25476 let (name, value) = line.split_once(':')?;
25477 name.eq_ignore_ascii_case("Authorization")
25478 .then(|| value.trim().to_string())
25479 });
25480 let namespace = request.lines().find_map(|line| {
25481 let (name, value) = line.split_once(':')?;
25482 name.eq_ignore_ascii_case("X-Namespace")
25483 .then(|| value.trim().to_string())
25484 });
25485 let worker_protocol = request.lines().find_map(|line| {
25486 let (name, value) = line.split_once(':')?;
25487 name.eq_ignore_ascii_case("X-Durable-Workflow-Protocol-Version")
25488 .then(|| value.trim().to_string())
25489 });
25490 let control_protocol = request.lines().find_map(|line| {
25491 let (name, value) = line.split_once(':')?;
25492 name.eq_ignore_ascii_case("X-Durable-Workflow-Control-Plane-Version")
25493 .then(|| value.trim().to_string())
25494 });
25495 let request_number = {
25496 let mut requests = requests.lock().expect("captured requests");
25497 requests.push(CapturedRequest {
25498 headers: request
25499 .split_once("\r\n\r\n")
25500 .map_or("", |(headers, _)| headers)
25501 .to_owned(),
25502 method: method.to_string(),
25503 path: path.to_string(),
25504 authorization,
25505 namespace,
25506 worker_protocol: worker_protocol.clone(),
25507 control_protocol,
25508 body: body.to_string(),
25509 received_at: Instant::now(),
25510 });
25511 requests
25512 .iter()
25513 .filter(|request| request.path == path)
25514 .count()
25515 };
25516
25517 if let Some(response) = behavior
25518 .request_override
25519 .and_then(|handler| handler(path, body, request_number))
25520 {
25521 write_mock_response(stream, response.0, &response.1);
25522 return;
25523 }
25524 if let Some(response) = behavior.response_override.and_then(|handler| handler(path)) {
25525 write_mock_response(stream, response.0, &response.1);
25526 return;
25527 }
25528 if path.ends_with("/poll") && request_number <= behavior.poll_failures_per_path {
25529 return;
25530 }
25531 let pressure_path = behavior
25532 .storage_path
25533 .is_some_and(|part| path.contains(part));
25534 let prior_failures = if path.ends_with("/poll") {
25535 behavior.poll_failures_per_path
25536 } else {
25537 0
25538 };
25539 if pressure_path
25540 && request_number.saturating_sub(prior_failures) <= behavior.storage_refusals
25541 {
25542 let request_body: Value = serde_json::from_str(body).unwrap_or(Value::Null);
25543 let poll_id = path
25544 .ends_with("/poll")
25545 .then(|| request_body["poll_request_id"].as_str().unwrap_or(""));
25546 let mut refusal = storage_refusal(
25547 poll_id,
25548 behavior.storage_unavailable,
25549 behavior.storage_mid_poll,
25550 );
25551 if behavior.storage_wrong_poll_id {
25552 refusal["poll_request_id"] = json!("wrong-poll");
25553 }
25554 write_mock_response(stream, "503 Service Unavailable", &refusal.to_string());
25555 return;
25556 }
25557 if path.contains("/storage-task/")
25558 || path.contains("/storage-activity/")
25559 || path.contains("/storage-query/")
25560 {
25561 write_mock_response(stream, "200 OK", "{}");
25562 return;
25563 }
25564 if behavior.storage_query && path == "/api/worker/query-tasks/poll" && request_number == 1 {
25565 write_mock_response(stream, "200 OK", &json!({"task":{
25566 "query_task_id":"storage-query", "query_task_attempt":7, "workflow_type":"storage.workflow",
25567 "query_name":"state", "workflow_id":"workflow", "run_id":"run", "payload_codec":"avro",
25568 "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC).unwrap(),
25569 "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC).unwrap(),
25570 "history_events":[], "run_status":"waiting", "lease_owner":"storage-worker"
25571 }}).to_string());
25572 return;
25573 }
25574 if behavior.storage_activity
25575 && path == "/api/worker/activity-tasks/poll"
25576 && request_number == 1
25577 {
25578 write_mock_response(stream, "200 OK", &json!({"task":{
25579 "task_id":"storage-activity", "activity_attempt_id":"storage-attempt", "activity_type":"storage.activity",
25580 "payload_codec":"avro", "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC).unwrap(),
25581 "attempt_number":7, "lease_owner":"storage-worker"
25582 }}).to_string());
25583 return;
25584 }
25585
25586 if path == "/api/worker/register" {
25587 if behavior.reject_registration_protocol {
25588 write_mock_response(
25589 stream,
25590 "400 Bad Request",
25591 r#"{"reason":"unsupported_protocol_version","message":"condition-wait occurrence identity requires worker protocol 1.17","supported_version":"1.16","requested_version":"1.17"}"#,
25592 );
25593 return;
25594 }
25595 if behavior.reject_registration {
25596 write_mock_response(
25597 stream,
25598 "503 Service Unavailable",
25599 r#"{"reason":"registration_unavailable","message":"registration failed"}"#,
25600 );
25601 return;
25602 }
25603 }
25604
25605 if path.starts_with("/api/worker/registrations/") {
25606 if behavior.reject_deregistration_protocol {
25607 write_mock_response(
25608 stream,
25609 "400 Bad Request",
25610 r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.17","requested_version":"1.19"}"#,
25611 );
25612 } else if behavior.reject_deregistration {
25613 write_mock_response(
25614 stream,
25615 "403 Forbidden",
25616 r#"{"reason":"authorization_failed","message":"worker cannot deregister"}"#,
25617 );
25618 } else {
25619 write_mock_response(
25620 stream,
25621 "200 OK",
25622 r#"{"worker_id":"deregistered-worker","outcome":"deregistered","recovered_workflow_task_count":2}"#,
25623 );
25624 }
25625 return;
25626 }
25627
25628 let is_poll = matches!(
25629 path,
25630 "/api/worker/workflow-tasks/poll"
25631 | "/api/worker/activity-tasks/poll"
25632 | "/api/worker/query-tasks/poll"
25633 );
25634 if is_poll && request_number <= behavior.long_poll_capacity_responses_per_path {
25635 write_mock_response(
25636 stream,
25637 "429 Too Many Requests",
25638 r#"{"task":null,"poll_status":"long_poll_capacity_exhausted","reason":"long_poll_capacity_exhausted","retryable":true,"retry_after_seconds":1}"#,
25639 );
25640 return;
25641 }
25642 if is_poll && request_number <= behavior.poll_failures_per_path {
25643 return;
25644 }
25645 if path == "/api/worker/heartbeat" && request_number <= behavior.heartbeat_failures {
25646 return;
25647 }
25648 if path == "/api/worker/heartbeat"
25649 && behavior.heartbeat_failure_request == Some(request_number)
25650 {
25651 return;
25652 }
25653 if path == "/api/worker/heartbeat"
25654 && behavior.delayed_heartbeat_request == Some(request_number)
25655 {
25656 thread::sleep(behavior.heartbeat_response_delay);
25657 }
25658 if behavior.unauthorized_polls && is_poll {
25659 write_mock_response(
25660 stream,
25661 "401 Unauthorized",
25662 r#"{"reason":"authentication_failed","message":"invalid worker token"}"#,
25663 );
25664 return;
25665 }
25666 if behavior.draining_polls && is_poll {
25667 write_mock_response(
25668 stream,
25669 "409 Conflict",
25670 r#"{"task":null,"poll_status":"draining","reason":"worker_draining","worker_status":"draining","drain_intent":"draining"}"#,
25671 );
25672 return;
25673 }
25674
25675 if let Some(codec_case) = behavior.invalid_task_payload_codec {
25676 if is_poll && request_number == 1 {
25677 let mut task = match path {
25678 "/api/worker/workflow-tasks/poll" => json!({
25679 "task_id": "codec-workflow",
25680 "workflow_type": "codec.workflow",
25681 "payload_codec": DEFAULT_CODEC,
25682 "workflow_task_attempt": 1,
25683 "lease_owner": "codec-worker"
25684 }),
25685 "/api/worker/activity-tasks/poll" => json!({
25686 "task_id": "codec-activity",
25687 "activity_attempt_id": "codec-activity-attempt",
25688 "activity_type": "codec.activity",
25689 "payload_codec": DEFAULT_CODEC,
25690 "attempt_number": 1,
25691 "lease_owner": "codec-worker"
25692 }),
25693 "/api/worker/query-tasks/poll" => json!({
25694 "query_task_id": "codec-query",
25695 "query_task_attempt": 1,
25696 "workflow_type": "codec.workflow",
25697 "query_name": "known",
25698 "payload_codec": DEFAULT_CODEC,
25699 "lease_owner": "codec-worker"
25700 }),
25701 _ => unreachable!("is_poll limits task codec probe paths"),
25702 };
25703 codec_case.apply(&mut task);
25704 write_mock_response(stream, "200 OK", &json!({"task": task}).to_string());
25705 return;
25706 }
25707
25708 if matches!(
25709 path,
25710 "/api/worker/workflow-tasks/codec-workflow/fail"
25711 | "/api/worker/activity-tasks/codec-activity/fail"
25712 | "/api/worker/query-tasks/codec-query/fail"
25713 ) {
25714 write_mock_response(stream, "200 OK", r#"{"outcome":"failed"}"#);
25715 return;
25716 }
25717 }
25718
25719 if behavior.reject_query_protocol && path.starts_with("/api/worker/query-tasks/") {
25720 let requested_version = worker_protocol.as_deref().unwrap_or("missing");
25721 let body = format!(
25722 r#"{{"reason":"unsupported_protocol_version","message":"Query tasks require worker protocol 1.8 or newer.","supported_version":"1.7","requested_version":"{requested_version}"}}"#
25723 );
25724 write_mock_response(stream, "400 Bad Request", &body);
25725 return;
25726 }
25727
25728 if behavior.reject_query_completion && path == "/api/worker/query-tasks/query-late/complete"
25729 {
25730 write_mock_response(
25731 stream,
25732 "409 Conflict",
25733 r#"{"reason":"query_task_timed_out","message":"query task timed out before completion"}"#,
25734 );
25735 return;
25736 }
25737
25738 if behavior.workflow_completion_status.is_some()
25739 && path == "/api/worker/workflow-tasks/poll"
25740 && request_number == 1
25741 {
25742 write_mock_response(
25743 stream,
25744 "200 OK",
25745 r#"{"task":{"task_id":"workflow-timeout-task","workflow_id":"reused-workflow-id","run_id":"run-selected-timeout","workflow_type":"timeout.workflow","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"workflow_task_attempt":3,"lease_owner":"timeout-worker"}}"#,
25746 );
25747 return;
25748 }
25749
25750 if path == "/api/worker/workflow-tasks/workflow-timeout-task/complete" {
25751 if let (Some(status), Some(body)) = (
25752 behavior.workflow_completion_status,
25753 behavior.workflow_completion_body,
25754 ) {
25755 write_mock_response(stream, status, body);
25756 return;
25757 }
25758 }
25759
25760 if behavior.waiting_query_worker {
25761 if behavior.complete_named_signal
25762 && path == "/api/worker/workflow-tasks/poll"
25763 && request_number == 1
25764 {
25765 let body = json!({
25766 "task": {
25767 "task_id": "snapshot-open",
25768 "workflow_id": "snapshot-1",
25769 "run_id": "snapshot-run-1",
25770 "workflow_type": "snapshot",
25771 "payload_codec": DEFAULT_CODEC,
25772 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25773 .expect("Avro workflow arguments"),
25774 "history_events": [],
25775 "workflow_task_attempt": 1,
25776 "lease_owner": "rust-snapshot-worker"
25777 }
25778 })
25779 .to_string();
25780 write_mock_response(stream, "200 OK", &body);
25781 return;
25782 }
25783
25784 let signal_request = request_number - usize::from(behavior.complete_named_signal);
25785 let signal_request_limit = 2 + usize::from(behavior.complete_named_signal);
25786 if path == "/api/worker/workflow-tasks/poll"
25787 && signal_request >= 1
25788 && signal_request <= signal_request_limit
25789 {
25790 let finish = behavior.complete_named_signal && signal_request == 3;
25791 let amounts = if signal_request == 1 {
25792 vec![3]
25793 } else {
25794 vec![3, 5]
25795 };
25796 let task_id = if signal_request == 1 {
25797 "snapshot-wait-3"
25798 } else if finish {
25799 "snapshot-finish"
25800 } else {
25801 "snapshot-wait-5"
25802 };
25803 let mut history_events = std::iter::once(json!({
25804 "event_type": "SignalWaitOpened",
25805 "payload": {"sequence": 1, "signal_name": "finish"}
25806 }))
25807 .chain(amounts.iter().enumerate().map(|(index, amount)| {
25808 json!({
25809 "event_type": "SignalReceived",
25810 "payload": {
25811 "signal_id": format!("increment-{amount}"),
25812 "signal_name": "increment",
25813 "workflow_sequence": index + 2,
25814 "payload_codec": DEFAULT_CODEC,
25815 "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
25816 .expect("Avro signal envelope")
25817 }
25818 })
25819 }))
25820 .collect::<Vec<_>>();
25821 let (resume_id, resume_name, resume_arguments) = if finish {
25822 history_events.push(json!({
25823 "event_type": "SignalReceived",
25824 "payload": {
25825 "signal_id": "finish",
25826 "signal_name": "finish",
25827 "workflow_sequence": 4,
25828 "payload_codec": DEFAULT_CODEC,
25829 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25830 .expect("Avro finish signal envelope")
25831 }
25832 }));
25833 (
25834 "finish".to_string(),
25835 "finish".to_string(),
25836 encode_value_envelope(&json!([]), DEFAULT_CODEC)
25837 .expect("Avro finish resume signal"),
25838 )
25839 } else {
25840 let amount = amounts.last().expect("amount");
25841 (
25842 format!("increment-{amount}"),
25843 "increment".to_string(),
25844 encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
25845 .expect("Avro increment resume signal"),
25846 )
25847 };
25848 let body = json!({
25849 "task": {
25850 "task_id": task_id,
25851 "workflow_id": "snapshot-1",
25852 "run_id": "snapshot-run-1",
25853 "workflow_type": "snapshot",
25854 "payload_codec": DEFAULT_CODEC,
25855 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25856 .expect("Avro workflow arguments"),
25857 "history_events": history_events,
25858 "workflow_task_attempt": 1,
25859 "workflow_signal_id": resume_id,
25860 "signal_name": resume_name,
25861 "signal_arguments": resume_arguments,
25862 "lease_owner": "rust-snapshot-worker"
25863 }
25864 })
25865 .to_string();
25866 write_mock_response(stream, "200 OK", &body);
25867 return;
25868 }
25869
25870 if path == "/api/worker/query-tasks/poll" && request_number == 1 {
25871 let history_events = [3, 5]
25872 .into_iter()
25873 .enumerate()
25874 .map(|(index, amount)| {
25875 json!({
25876 "event_type": "SignalReceived",
25877 "payload": {
25878 "signal_id": format!("increment-{amount}"),
25879 "signal_name": "increment",
25880 "workflow_sequence": index + 2,
25881 "payload_codec": DEFAULT_CODEC,
25882 "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
25883 .expect("Avro query signal envelope")
25884 }
25885 })
25886 })
25887 .collect::<Vec<_>>();
25888 let body = json!({
25889 "task": {
25890 "query_task_id": "snapshot-current",
25891 "query_task_attempt": 1,
25892 "lease_owner": "rust-snapshot-worker",
25893 "workflow_id": "snapshot-1",
25894 "run_id": "snapshot-run-1",
25895 "workflow_type": "snapshot",
25896 "query_name": "current",
25897 "payload_codec": DEFAULT_CODEC,
25898 "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25899 .expect("Avro workflow arguments"),
25900 "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25901 .expect("Avro query arguments"),
25902 "history_events": history_events,
25903 "run_status": "waiting"
25904 }
25905 })
25906 .to_string();
25907 write_mock_response(stream, "200 OK", &body);
25908 return;
25909 }
25910
25911 if path == "/api/worker/workflow-tasks/snapshot-wait-3/fail"
25912 || path == "/api/worker/workflow-tasks/snapshot-wait-5/fail"
25913 {
25914 write_mock_response(
25915 stream,
25916 "200 OK",
25917 r#"{"outcome":"waiting_for_history","recorded":true}"#,
25918 );
25919 return;
25920 }
25921
25922 if path == "/api/worker/workflow-tasks/snapshot-open/complete" {
25923 write_mock_response(stream, "200 OK", r#"{"outcome":"waiting","recorded":true}"#);
25924 return;
25925 }
25926
25927 if path == "/api/worker/workflow-tasks/snapshot-finish/complete" {
25928 write_mock_response(
25929 stream,
25930 "200 OK",
25931 r#"{"outcome":"completed","run_status":"completed","recorded":true}"#,
25932 );
25933 return;
25934 }
25935
25936 if path == "/api/worker/query-tasks/snapshot-current/complete" {
25937 write_mock_response(stream, "200 OK", r#"{"outcome":"completed"}"#);
25938 return;
25939 }
25940 }
25941
25942 if matches!(
25943 path,
25944 "/api/workflows/typed-1/query/inspect" | "/api/workflows/typed-1/update/replace"
25945 ) {
25946 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
25947 .expect("typed mock result");
25948 let body = json!({
25949 "result": typed_fidelity_probe().into_json().expect("result projection"),
25950 "result_envelope": result,
25951 })
25952 .to_string();
25953 write_mock_response(stream, "200 OK", &body);
25954 return;
25955 }
25956
25957 if path == "/api/workflows/typed-1" {
25958 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
25959 .expect("typed mock result");
25960 let body = json!({
25961 "workflow_id": "typed-1",
25962 "run_id": "run-typed-1",
25963 "workflow_type": "typed.echo",
25964 "status": "completed",
25965 "output": typed_fidelity_probe().into_json().expect("output projection"),
25966 "output_envelope": result,
25967 })
25968 .to_string();
25969 write_mock_response(stream, "200 OK", &body);
25970 return;
25971 }
25972
25973 let (status, body) = match path {
25974 "/api/cluster/info" => ("200 OK", r#"{"limits":{"max_payload_bytes":2097152}}"#),
25975 "/api/health" => ("200 OK", r#"{"status":"ok"}"#),
25976 "/api/workflows" => (
25977 "201 Created",
25978 r#"{"workflow_id":"wf-start-options","run_id":"run-start-options","workflow_type":"rust.timeout"}"#,
25979 ),
25980 "/api/worker/register" if behavior.decline_registration => (
25981 "200 OK",
25982 r#"{"worker_id":"declined-worker","registered":false}"#,
25983 ),
25984 "/api/worker/register" if behavior.waiting_query_worker => (
25985 "200 OK",
25986 r#"{"worker_id":"rust-snapshot-worker","registered":true,"heartbeat_interval_seconds":1}"#,
25987 ),
25988 "/api/worker/register" => (
25989 "200 OK",
25990 r#"{"worker_id":"mock-worker","registered":true,"heartbeat_interval_seconds":3600}"#,
25991 ),
25992 "/api/worker/heartbeat" => ("200 OK", "{}"),
25993 "/api/worker/activity-tasks/poll"
25994 if behavior.cancelled_activity && request_number == 1 =>
25995 {
25996 (
25997 "200 OK",
25998 r#"{"task":{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","activity_type":"cancel-aware","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"rust-cancel-worker"}}"#,
25999 )
26000 }
26001 "/api/worker/activity-tasks/poll"
26002 if behavior.long_poll_capacity_responses_per_path > 0
26003 && request_number
26004 == behavior
26005 .long_poll_capacity_responses_per_path
26006 .saturating_add(1) =>
26007 {
26008 (
26009 "200 OK",
26010 r#"{"task":{"task_id":"capacity-activity","activity_attempt_id":"capacity-attempt","activity_type":"capacity.activity","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"capacity-worker"}}"#,
26011 )
26012 }
26013 "/api/worker/activity-tasks/poll" | "/api/worker/workflow-tasks/poll" => {
26014 ("200 OK", r#"{"task":null}"#)
26015 }
26016 "/api/worker/query-tasks/poll"
26017 if behavior.reject_query_completion && request_number == 1 =>
26018 {
26019 (
26020 "200 OK",
26021 r#"{"task":{"query_task_id":"query-late","query_task_attempt":1,"lease_owner":"late-worker","workflow_id":"counter-late","run_id":"run-late","workflow_type":"counter","query_name":"current","payload_codec":"avro","workflow_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"query_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"run_status":"running"}}"#,
26022 )
26023 }
26024 "/api/worker/query-tasks/poll" => ("200 OK", r#"{"task":null}"#),
26025 "/api/worker/query-tasks/query-capture/complete"
26026 | "/api/worker/query-tasks/query-capture/fail" => ("200 OK", "{}"),
26027 "/api/worker/activity-tasks/activity-cancel/heartbeat" => (
26028 "200 OK",
26029 r#"{"activity_attempt_id":"attempt-cancel","cancel_requested":true,"can_continue":false,"reason":"run_cancelled","run_closed_reason":"cancelled","heartbeat_recorded":false}"#,
26030 ),
26031 "/api/worker/activity-tasks/activity-cancel/complete" => (
26032 "409 Conflict",
26033 r#"{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","reason":"run_cancelled","cancel_requested":true,"can_continue":false,"run_closed_reason":"cancelled"}"#,
26034 ),
26035 "/api/worker/activity-tasks/activity-typed/complete"
26036 | "/api/worker/activity-tasks/activity-typed/fail"
26037 | "/api/worker/activity-tasks/capacity-activity/complete"
26038 | "/api/workflows/typed-1/signal/changed" => ("200 OK", "{}"),
26039 "/api/workflows/counter-1/query/current" => (
26040 "200 OK",
26041 r#"{"workflow_id":"counter-1","query_name":"current","result":{"count":8},"result_envelope":{"codec":"avro","blob":"wwHioz3/VYAiNw4CCmNvdW50BBAA"}}"#,
26042 ),
26043 "/api/workflows/counter-1/query/missing" => (
26044 "404 Not Found",
26045 r#"{"workflow_id":"counter-1","query_name":"missing","reason":"rejected_unknown_query","message":"unknown query"}"#,
26046 ),
26047 "/api/workflows/wf-lifecycle/cancel" => (
26048 "200 OK",
26049 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","reason":"cleanup requested","command_status":"accepted"}"#,
26050 ),
26051 "/api/workflows/wf-lifecycle/terminate" => (
26052 "200 OK",
26053 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","reason":"forced stop","command_status":"accepted"}"#,
26054 ),
26055 "/api/workflows/wf-lifecycle/runs/run-current/cancel" => (
26056 "200 OK",
26057 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","command_status":"accepted"}"#,
26058 ),
26059 "/api/workflows/wf-lifecycle/runs/run-current/terminate" => (
26060 "200 OK",
26061 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","command_status":"accepted"}"#,
26062 ),
26063 "/api/workflows/wf-lifecycle/runs/run-failed/redrive" => (
26064 "202 Accepted",
26065 r#"{"workflow_id":"wf-lifecycle","continued_from_run_id":"run-failed","run_id":"run-successor","outcome":"redriven","command_status":"accepted","resume_step_sequence":2}"#,
26066 ),
26067 "/api/workflows/wf-lifecycle/runs/run-failed-existing/redrive" => (
26068 "200 OK",
26069 r#"{"workflow_id":"wf-lifecycle","continued_from_run_id":"run-failed-existing","run_id":"run-successor","outcome":"redriven","command_status":"accepted","resume_step_sequence":2}"#,
26070 ),
26071 "/api/workflows/wf-lifecycle/runs/run-completed/redrive" => (
26072 "409 Conflict",
26073 r#"{"workflow_id":"wf-lifecycle","run_id":"run-completed","reason":"run_not_failed","message":"Run is not failed."}"#,
26074 ),
26075 "/api/workflows/wf-lifecycle/runs/run-stale/cancel"
26076 | "/api/workflows/wf-lifecycle/runs/run-stale/terminate" => (
26077 "409 Conflict",
26078 r#"{"workflow_id":"wf-lifecycle","run_id":"run-stale","reason":"historical_run_command_rejected","target_scope":"run","message":"Commands cannot target historical runs."}"#,
26079 ),
26080 "/api/workflows/wf-failed" | "/api/workflows/wf-failed/runs/run-failed" => (
26081 "200 OK",
26082 r#"{"workflow_id":"wf-failed","run_id":"run-failed","status":"failed","closed_reason":"failed","error":"payment failed","failure":{"message":"payment failed","failure_category":"application","exception_type":"PaymentError","exception_class":"billing::PaymentError","non_retryable":true,"exception":{"type":"PaymentError","class":"billing::PaymentError","message":"payment failed"},"failures":[{"id":"failure-17","failure_category":"application"}]}}"#,
26083 ),
26084 "/api/workflows/wf-cancelled" => (
26085 "200 OK",
26086 r#"{"workflow_id":"wf-cancelled","run_id":"run-cancelled","status":"cancelled","closed_reason":"cancelled","reason":"cleanup requested"}"#,
26087 ),
26088 "/api/workflows/wf-terminated" => (
26089 "200 OK",
26090 r#"{"workflow_id":"wf-terminated","run_id":"run-terminated","status":"terminated","closed_reason":"terminated","reason":"forced stop"}"#,
26091 ),
26092 "/api/workflows/wf-timed-out" => (
26093 "200 OK",
26094 r#"{"workflow_id":"wf-timed-out","run_id":"run-timed-out","status":"failed","closed_reason":"timed_out","reason":"run_timeout"}"#,
26095 ),
26096 "/api/workflows/wf-waiting" | "/api/workflows/wf-waiting/runs/run-waiting" => (
26097 "200 OK",
26098 r#"{"workflow_id":"wf-waiting","run_id":"run-waiting","status":"waiting"}"#,
26099 ),
26100 "/api/workflows/wf-selected" => (
26101 "200 OK",
26102 r#"{"workflow_id":"wf-selected","run_id":"run-current","status":"completed","output":"current run output"}"#,
26103 ),
26104 "/api/workflows/wf-selected/runs/run-selected" => (
26105 "200 OK",
26106 r#"{"workflow_id":"wf-selected","run_id":"run-selected","status":"cancelled","closed_reason":"cancelled","reason":"selected run cancelled"}"#,
26107 ),
26108 _ => ("404 Not Found", r#"{"message":"not found"}"#),
26109 };
26110 write_mock_response(stream, status, body);
26111 }
26112
26113 fn mock_request_is_complete(request: &[u8]) -> bool {
26114 let Some(header_end) = request
26115 .windows(4)
26116 .position(|window| window == b"\r\n\r\n")
26117 .map(|position| position + 4)
26118 else {
26119 return false;
26120 };
26121 let headers = String::from_utf8_lossy(&request[..header_end]);
26122 let content_length = headers.lines().find_map(|line| {
26123 let (name, value) = line.split_once(':')?;
26124 name.eq_ignore_ascii_case("content-length")
26125 .then(|| value.trim().parse::<usize>().ok())
26126 .flatten()
26127 });
26128
26129 request.len() >= header_end + content_length.unwrap_or(0)
26130 }
26131
26132 fn write_mock_response(stream: &mut TcpStream, status: &str, body: &str) {
26133 let response = format!(
26134 "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
26135 body.len()
26136 );
26137
26138 let _ = stream.write_all(response.as_bytes());
26139 let _ = stream.flush();
26140 }
26141}