1#![doc = include_str!("../README.md")]
2
3mod runtime_payloads;
4mod runtime_uploads;
5
6use std::{
7 any::{type_name, Any, TypeId},
8 collections::{BTreeMap, HashMap},
9 future::Future,
10 io::{self, Read},
11 pin::Pin,
12 sync::{
13 atomic::{AtomicBool, Ordering},
14 Arc, Mutex, OnceLock,
15 },
16 task::{Context as TaskContext, Poll},
17 time::{Duration, Instant, SystemTime, UNIX_EPOCH},
18};
19
20use apache_avro::{from_avro_datum, to_avro_datum, types::Value as AvroDatum, Schema};
21use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
22use chrono::DateTime;
23use futures_util::{future::OptionFuture, task::noop_waker_ref};
24use serde::{
25 de::DeserializeOwned,
26 ser::{SerializeMap, SerializeSeq},
27 Deserialize, Deserializer, Serialize, Serializer,
28};
29pub use serde_json::{json, Value};
30use sha2::{Digest, Sha256};
31use thiserror::Error;
32pub use uuid::Uuid;
33
34pub const WORKER_PROTOCOL_VERSION: &str = "1.19";
35pub const PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION: &str = "1.18";
37pub const CONTROL_PLANE_VERSION: &str = "2";
38pub const DEFAULT_CODEC: &str = "avro";
39pub const SDK_VERSION: &str = concat!("durable-workflow-rust/", env!("CARGO_PKG_VERSION"));
40pub const CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY: &str =
42 "condition_wait_occurrence_identity";
43pub const MEMO_UPSERTS_CAPABILITY: &str = "memo_upserts";
45pub const QUERY_TASKS_CAPABILITY: &str = "query_tasks";
47pub const TYPED_SEARCH_ATTRIBUTES_CAPABILITY: &str = "typed_search_attributes";
49pub const WORKFLOW_UPDATES_CAPABILITY: &str = "workflow_updates";
51pub const MESSAGE_STREAMS_CAPABILITY: &str = "message_streams";
53pub const DURABLE_SELECTION_CAPABILITY: &str = "durable_selection";
55pub const MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.15";
56pub const MESSAGE_STREAM_SIGNAL: &str = "__durable_workflow_message_stream";
57pub const MESSAGE_STREAM_SCHEMA: &str = "durable-workflow.v2.message-stream.message";
58pub const MESSAGE_STREAM_CURSOR_SCHEMA: &str = "durable-workflow.v2.message-stream.cursor";
59pub const MESSAGE_STREAM_MAX_BATCH: usize = 100;
60pub const QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
62pub const SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
64pub const MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.14";
66pub const TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.16";
68pub const CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.9";
70pub const CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.17";
72pub const DURABLE_SELECTION_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.19";
74
75pub fn worker_protocol_supports_message_streams(version: &str) -> bool {
76 let Some((major, minor)) = version.split_once('.') else {
77 return false;
78 };
79 major == "1" && minor.parse::<u64>().is_ok_and(|minor| minor >= 15)
80}
81
82fn validate_user_signal_name(signal_name: &str) -> Result<()> {
83 if signal_name == MESSAGE_STREAM_SIGNAL {
84 return Err(Error::Codec(format!(
85 "signal name {MESSAGE_STREAM_SIGNAL:?} is reserved by the workflow runtime"
86 )));
87 }
88 Ok(())
89}
90
91const MAX_LONG_POLL_TIMEOUT_SECONDS: u64 = 60;
92const WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE: &str =
93 "Workflow task waiting for scheduled history.";
94const WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE: &str = "WorkflowTaskWaitingForHistory";
95const MISSING_TASK_PAYLOAD_CODEC: &str = "\0missing-task-payload-codec";
96const NULL_TASK_PAYLOAD_CODEC: &str = "\0null-task-payload-codec";
97const NON_STRING_TASK_PAYLOAD_CODEC: &str = "\0non-string-task-payload-codec";
98const MAX_MEMO_ENTRIES: usize = 100;
99const MAX_MEMO_VALUE_SIZE_BYTES: usize = 10_240;
100const MAX_MEMO_TOTAL_SIZE_BYTES: usize = 65_536;
101
102const QUERY_TASK_FINAL_REJECTION_REASONS: &[&str] = &[
103 "lease_expired",
104 "query_task_not_found",
105 "query_task_not_leased",
106 "query_task_timed_out",
107];
108
109pub fn portable_worker_affinity_capability_manifest() -> Value {
111 json!({
112 "local_activities": {
113 "supported": false,
114 "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
115 "reason": "rust_worker_does_not_execute_record_local_activity",
116 },
117 "worker_sessions": {
118 "supported": false,
119 "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
120 "reason": "rust_worker_has_no_typed_session_lifecycle",
121 },
122 "sticky_execution": {
123 "supported": false,
124 "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
125 "reason": "rust_worker_uses_complete_durable_history_replay",
126 },
127 })
128}
129
130pub const AVRO_VALUE_SCHEMA_JSON: &str =
132 include_str!("../schema/durable_workflow.protocol.Value.v1.avsc");
133pub const AVRO_VALUE_SCHEMA_FINGERPRINT_HEX: &str = "e2a33dff55802237";
134pub const AVRO_VALUE_SCHEMA_FINGERPRINT: [u8; 8] = [0xe2, 0xa3, 0x3d, 0xff, 0x55, 0x80, 0x22, 0x37];
135const AVRO_SINGLE_OBJECT_MAGIC: [u8; 2] = [0xc3, 0x01];
136
137static AVRO_VALUE_SCHEMA: OnceLock<std::result::Result<Schema, String>> = OnceLock::new();
138static AVRO_VALUE_ORDERED_MAP_ENCODING_SCHEMA: OnceLock<std::result::Result<Schema, String>> =
139 OnceLock::new();
140
141#[derive(Clone, Copy)]
142enum RequestProtocol {
143 ControlPlane,
144 Worker(&'static str),
145}
146
147pub type Result<T> = std::result::Result<T, Error>;
148
149#[derive(Debug, Error)]
150pub enum Error {
151 #[error("transport error: {0}")]
152 Transport(#[from] reqwest::Error),
153 #[error(
154 "invalid Durable Workflow base URL: omit the SDK-owned /api suffix and pass the Server or Cloud runtime base URL; the SDK appends /api automatically"
155 )]
156 InvalidBaseUrl,
157 #[error("json error: {0}")]
158 Json(#[from] serde_json::Error),
159 #[error("http {status}: {body}")]
160 Http {
161 status: reqwest::StatusCode,
162 body: String,
163 },
164 #[error("codec error: {0}")]
165 Codec(String),
166 #[error(transparent)]
167 QueryFailed(QueryFailure),
168 #[error(transparent)]
169 Protocol(ProtocolFailure),
170 #[error(transparent)]
171 NonDeterministicReplay(ReplayFailure),
172 #[error(transparent)]
173 ChildWorkflowFailed(ChildWorkflowFailure),
174 #[error(transparent)]
175 ActivityFailed(ActivityFailure),
176 #[error(transparent)]
177 ParallelFailed(ParallelFailure),
178 #[error(transparent)]
179 SagaCompensationFailed(SagaCompensationFailure),
180 #[error(transparent)]
181 InvalidParallelGroup(ParallelGroupError),
182 #[error(transparent)]
183 DurableOperationCancelled(DurableOperationCancelled),
184 #[error(transparent)]
185 WorkflowCancellationRequested(WorkflowCancellationRequested),
186 #[error(transparent)]
187 WorkflowCommandRejected(WorkflowCommandRejection),
188 #[error(transparent)]
189 WorkflowFailed(WorkflowTerminalOutcome),
190 #[error(transparent)]
191 WorkflowCancelled(WorkflowTerminalOutcome),
192 #[error(transparent)]
193 WorkflowTerminated(WorkflowTerminalOutcome),
194 #[error(transparent)]
195 WorkflowTimedOut(WorkflowTerminalOutcome),
196 #[error(transparent)]
197 ActivityTaskRejected(ActivityTaskRejection),
198 #[error("workflow handler {0:?} is not registered")]
199 WorkflowNotRegistered(String),
200 #[error("activity handler {0:?} is not registered")]
201 ActivityNotRegistered(String),
202 #[error(
203 "{handler_kind} handler {handler_name:?} {value_kind} type {rust_type} is incompatible with the fixed Avro Value codec: {message}"
204 )]
205 HandlerType {
206 handler_kind: HandlerKind,
207 handler_name: String,
208 value_kind: HandlerValueKind,
209 rust_type: &'static str,
210 message: String,
211 },
212 #[error("workflow future yielded without emitting a durable command")]
213 WorkflowYieldedWithoutCommand,
214 #[error(
215 "workflow_stream_command_identity_missing: workflow stream authoring requires a non-empty server-provided workflow_command_id"
216 )]
217 MissingWorkflowCommandIdentity,
218 #[error("workflow state lock is poisoned")]
219 WorkflowStatePoisoned,
220 #[error("timer duration is too large for the worker protocol")]
221 TimerDurationOverflow,
222 #[error(transparent)]
223 InvalidConditionWaitOptions(#[from] ConditionWaitOptionsError),
224 #[error(transparent)]
225 InvalidSearchAttributeUpdate(#[from] SearchAttributeUpdateError),
226 #[error("operation timed out")]
227 Timeout,
228 #[error(
229 "missing {role}-plane credentials: configure ClientBuilder::{role}_token or ClientBuilder::token; a {opposite_role}-plane token cannot authorize this request"
230 )]
231 MissingRoleCredentials {
232 role: &'static str,
233 opposite_role: &'static str,
234 },
235 #[error("worker loop error: {0}")]
236 WorkerLoop(String),
237 #[error(
238 "workflow command contract for {workflow_type:?} declares update validators, but this Rust SDK cannot execute synchronous pre-accept update validation"
239 )]
240 UnsupportedUpdateValidators { workflow_type: String },
241 #[error("{primary}; worker deregistration also failed: {deregistration}")]
242 WorkerShutdown {
243 primary: Box<Error>,
244 deregistration: Box<Error>,
245 },
246 #[error("invalid child workflow options: {0}")]
247 InvalidChildWorkflowOptions(String),
248 #[error("invalid workflow memo update: {0}")]
249 InvalidMemoUpdate(String),
250 #[error(
251 "workflow_memo_updates_unavailable: the connected runtime did not advertise workflow memo update support"
252 )]
253 WorkflowMemoUpdatesUnavailable,
254 #[error(transparent)]
255 InvalidActivityOptions(ActivityOptionsError),
256 #[error(transparent)]
257 InvalidContinueAsNewOptions(#[from] ContinueAsNewOptionsError),
258 #[doc(hidden)]
259 #[error("workflow requested continue as new")]
260 ContinueAsNew(ContinueAsNewRequest),
261}
262
263#[derive(Clone, Debug, Error, PartialEq, Eq)]
265pub enum ConditionWaitOptionsError {
266 #[error("condition_key must be non-empty")]
267 EmptyKey,
268 #[error("condition_definition_fingerprint must be non-empty")]
269 EmptyPredicateIdentity,
270 #[error("condition timeout is too large for the worker protocol")]
271 TimeoutOverflow,
272}
273
274#[derive(Clone, Debug, PartialEq, Eq)]
281pub struct ConditionWaitOptions {
282 condition_key: String,
283 predicate_identity: String,
284 timeout: Option<Duration>,
285}
286
287impl ConditionWaitOptions {
288 pub fn new(condition_key: impl Into<String>, predicate_identity: impl Into<String>) -> Self {
289 Self {
290 condition_key: condition_key.into(),
291 predicate_identity: predicate_identity.into(),
292 timeout: None,
293 }
294 }
295
296 pub fn timeout(mut self, timeout: Duration) -> Self {
297 self.timeout = Some(timeout);
298 self
299 }
300
301 fn validate(
302 &self,
303 ) -> std::result::Result<ValidatedConditionWaitOptions, ConditionWaitOptionsError> {
304 let condition_key = self.condition_key.trim();
305 if condition_key.is_empty() {
306 return Err(ConditionWaitOptionsError::EmptyKey);
307 }
308 let predicate_identity = self.predicate_identity.trim();
309 if predicate_identity.is_empty() {
310 return Err(ConditionWaitOptionsError::EmptyPredicateIdentity);
311 }
312 let timeout_seconds = self
313 .timeout
314 .map(|timeout| {
315 timeout
316 .as_secs()
317 .checked_add(u64::from(timeout.subsec_nanos() > 0))
318 .ok_or(ConditionWaitOptionsError::TimeoutOverflow)
319 })
320 .transpose()?;
321
322 Ok(ValidatedConditionWaitOptions {
323 condition_key: condition_key.to_string(),
324 predicate_identity: predicate_identity.to_string(),
325 timeout_seconds,
326 })
327 }
328}
329
330#[derive(Clone, Debug, PartialEq, Eq)]
331struct ValidatedConditionWaitOptions {
332 condition_key: String,
333 predicate_identity: String,
334 timeout_seconds: Option<u64>,
335}
336
337const CONDITION_WAIT_OCCURRENCE_PREFIX: &str = "rust:condition-wait:";
338
339#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
341#[serde(rename_all = "snake_case")]
342pub enum ConditionWaitResult {
343 Satisfied,
344 TimedOut,
345}
346
347impl ConditionWaitResult {
348 pub fn is_satisfied(self) -> bool {
349 self == Self::Satisfied
350 }
351
352 pub fn is_timed_out(self) -> bool {
353 self == Self::TimedOut
354 }
355}
356
357#[doc(hidden)]
359pub fn __condition_definition_fingerprint(source: &str) -> String {
360 let mut digest = Sha256::new();
361 digest.update(b"durable-workflow-rust.wait-condition.v1\0");
362 digest.update(source.as_bytes());
363 format!("sha256:{:x}", digest.finalize())
364}
365
366#[macro_export]
372macro_rules! wait_condition {
373 ($ctx:expr, $key:expr, timeout: $timeout:expr, $predicate:expr $(,)?) => {{
374 $ctx.wait_condition(
375 $crate::ConditionWaitOptions::new(
376 $key,
377 $crate::__condition_definition_fingerprint(concat!(
378 module_path!(),
379 "\0",
380 stringify!($predicate)
381 )),
382 )
383 .timeout($timeout),
384 $predicate,
385 )
386 }};
387 ($ctx:expr, $key:expr, $predicate:expr $(,)?) => {{
388 $ctx.wait_condition(
389 $crate::ConditionWaitOptions::new(
390 $key,
391 $crate::__condition_definition_fingerprint(concat!(
392 module_path!(),
393 "\0",
394 stringify!($predicate)
395 )),
396 ),
397 $predicate,
398 )
399 }};
400}
401
402const MAX_SEARCH_ATTRIBUTES_PER_UPDATE: usize = 100;
403const MAX_SEARCH_ATTRIBUTE_KEY_LENGTH: usize = 64;
404const MAX_SEARCH_ATTRIBUTE_STRING_LENGTH: usize = 2_048;
405const MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH: usize = 255;
406const MAX_SEARCH_ATTRIBUTE_UPDATE_BYTES: usize = 65_536;
407
408#[derive(Clone, Debug, Error, PartialEq, Eq)]
410pub enum SearchAttributeUpdateError {
411 #[error("search-attribute update requires at least one attribute")]
412 Empty,
413 #[error("search attribute key {0:?} must be 1-64 URL-safe ASCII characters")]
414 InvalidKey(String),
415 #[error("search-attribute update exceeds the limit of 100 attributes")]
416 TooManyAttributes,
417 #[error("search attribute {key:?} {kind} value exceeds {limit} bytes")]
418 ValueTooLong {
419 key: String,
420 kind: &'static str,
421 limit: usize,
422 },
423 #[error(
424 "search attribute {0:?} must not contain an empty string value; use delete() to remove it"
425 )]
426 EmptyString(String),
427 #[error("search attribute {0:?} has a non-finite float value")]
428 NonFiniteFloat(String),
429 #[error("search attribute {0:?} must use an RFC 3339 datetime with an explicit timezone")]
430 InvalidDateTime(String),
431 #[error("search-attribute update exceeds the 65536-byte protocol limit")]
432 PayloadTooLarge,
433}
434
435#[derive(Clone, Debug, PartialEq)]
437pub enum SearchAttributeValue {
438 String(String),
439 Keyword(String),
440 KeywordList(Vec<String>),
441 Int(i64),
442 Float(f64),
443 Bool(bool),
444 DateTime(String),
445 Delete,
446}
447
448impl SearchAttributeValue {
449 fn type_name(&self) -> Option<&'static str> {
450 match self {
451 Self::String(_) => Some("string"),
452 Self::Keyword(_) => Some("keyword"),
453 Self::KeywordList(_) => Some("keyword_list"),
454 Self::Int(_) => Some("int"),
455 Self::Float(_) => Some("float"),
456 Self::Bool(_) => Some("bool"),
457 Self::DateTime(_) => Some("datetime"),
458 Self::Delete => None,
459 }
460 }
461
462 fn normalized(self, key: &str) -> std::result::Result<Self, SearchAttributeUpdateError> {
463 let normalize_string = |value: String, kind: &'static str, limit: usize| {
464 let value = value.trim().to_string();
465 if value.is_empty() {
466 return Err(SearchAttributeUpdateError::EmptyString(key.to_string()));
467 }
468 if value.len() > limit {
469 return Err(SearchAttributeUpdateError::ValueTooLong {
470 key: key.to_string(),
471 kind,
472 limit,
473 });
474 }
475 Ok(value)
476 };
477
478 match self {
479 Self::String(value) => Ok(Self::String(normalize_string(
480 value,
481 "string",
482 MAX_SEARCH_ATTRIBUTE_STRING_LENGTH,
483 )?)),
484 Self::Keyword(value) => Ok(Self::Keyword(normalize_string(
485 value,
486 "keyword",
487 MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH,
488 )?)),
489 Self::KeywordList(values) => {
490 let values = values
491 .into_iter()
492 .map(|value| {
493 let value = value.trim().to_string();
494 if value.len() > MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH {
495 return Err(SearchAttributeUpdateError::ValueTooLong {
496 key: key.to_string(),
497 kind: "keyword-list entry",
498 limit: MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH,
499 });
500 }
501 Ok(value)
502 })
503 .collect::<std::result::Result<Vec<_>, _>>()?;
504 Ok(Self::KeywordList(values))
505 }
506 Self::Float(value) if !value.is_finite() => {
507 Err(SearchAttributeUpdateError::NonFiniteFloat(key.to_string()))
508 }
509 Self::DateTime(value) => {
510 let value =
511 normalize_string(value, "datetime", MAX_SEARCH_ATTRIBUTE_STRING_LENGTH)?;
512 if DateTime::parse_from_rfc3339(&value).is_err() {
513 return Err(SearchAttributeUpdateError::InvalidDateTime(key.to_string()));
514 }
515 Ok(Self::DateTime(value))
516 }
517 value => Ok(value),
518 }
519 }
520
521 fn into_json(self) -> Value {
522 match self {
523 Self::String(value) | Self::Keyword(value) | Self::DateTime(value) => {
524 Value::String(value)
525 }
526 Self::KeywordList(values) => {
527 Value::Array(values.into_iter().map(Value::String).collect())
528 }
529 Self::Int(value) => json!(value),
530 Self::Float(value) => json!(value),
531 Self::Bool(value) => json!(value),
532 Self::Delete => Value::Null,
533 }
534 }
535}
536
537#[derive(Clone, Debug, Default, PartialEq)]
539pub struct SearchAttributeUpdate {
540 attributes: BTreeMap<String, SearchAttributeValue>,
541}
542
543impl SearchAttributeUpdate {
544 pub fn new() -> Self {
545 Self::default()
546 }
547
548 pub fn set(
549 mut self,
550 key: impl Into<String>,
551 value: SearchAttributeValue,
552 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
553 let key = key.into();
554 validate_search_attribute_key(&key)?;
555 if !self.attributes.contains_key(&key)
556 && self.attributes.len() >= MAX_SEARCH_ATTRIBUTES_PER_UPDATE
557 {
558 return Err(SearchAttributeUpdateError::TooManyAttributes);
559 }
560 self.attributes.insert(key.clone(), value.normalized(&key)?);
561 self.validate_size()?;
562 Ok(self)
563 }
564
565 pub fn string(
566 self,
567 key: impl Into<String>,
568 value: impl Into<String>,
569 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
570 self.set(key, SearchAttributeValue::String(value.into()))
571 }
572
573 pub fn keyword(
574 self,
575 key: impl Into<String>,
576 value: impl Into<String>,
577 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
578 self.set(key, SearchAttributeValue::Keyword(value.into()))
579 }
580
581 pub fn keyword_list<I, V>(
582 self,
583 key: impl Into<String>,
584 values: I,
585 ) -> std::result::Result<Self, SearchAttributeUpdateError>
586 where
587 I: IntoIterator<Item = V>,
588 V: Into<String>,
589 {
590 self.set(
591 key,
592 SearchAttributeValue::KeywordList(values.into_iter().map(Into::into).collect()),
593 )
594 }
595
596 pub fn int(
597 self,
598 key: impl Into<String>,
599 value: i64,
600 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
601 self.set(key, SearchAttributeValue::Int(value))
602 }
603
604 pub fn float(
605 self,
606 key: impl Into<String>,
607 value: f64,
608 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
609 self.set(key, SearchAttributeValue::Float(value))
610 }
611
612 pub fn bool(
613 self,
614 key: impl Into<String>,
615 value: bool,
616 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
617 self.set(key, SearchAttributeValue::Bool(value))
618 }
619
620 pub fn datetime(
621 self,
622 key: impl Into<String>,
623 value: impl Into<String>,
624 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
625 self.set(key, SearchAttributeValue::DateTime(value.into()))
626 }
627
628 pub fn delete(
629 self,
630 key: impl Into<String>,
631 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
632 self.set(key, SearchAttributeValue::Delete)
633 }
634
635 fn validate_size(&self) -> std::result::Result<(), SearchAttributeUpdateError> {
636 let (attributes, _) = self.clone().into_wire_parts();
637 if serde_json::to_vec(&attributes)
638 .map(|payload| payload.len() > MAX_SEARCH_ATTRIBUTE_UPDATE_BYTES)
639 .unwrap_or(true)
640 {
641 return Err(SearchAttributeUpdateError::PayloadTooLarge);
642 }
643 Ok(())
644 }
645
646 fn into_wire_parts(self) -> (Value, BTreeMap<String, String>) {
647 let mut attributes = serde_json::Map::new();
648 let mut attribute_types = BTreeMap::new();
649 for (key, value) in self.attributes {
650 if let Some(type_name) = value.type_name() {
651 attribute_types.insert(key.clone(), type_name.to_string());
652 }
653 attributes.insert(key, value.into_json());
654 }
655 (Value::Object(attributes), attribute_types)
656 }
657
658 fn validate(&self) -> std::result::Result<(), SearchAttributeUpdateError> {
659 if self.attributes.is_empty() {
660 return Err(SearchAttributeUpdateError::Empty);
661 }
662 self.validate_size()
663 }
664}
665
666fn validate_search_attribute_key(key: &str) -> std::result::Result<(), SearchAttributeUpdateError> {
667 let valid = !key.is_empty()
668 && key.len() <= MAX_SEARCH_ATTRIBUTE_KEY_LENGTH
669 && key
670 .bytes()
671 .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-' | b':'));
672 if valid {
673 Ok(())
674 } else {
675 Err(SearchAttributeUpdateError::InvalidKey(key.to_string()))
676 }
677}
678
679#[derive(Clone, Copy, Debug, PartialEq, Eq)]
681pub enum HandlerKind {
682 Workflow,
683 Activity,
684}
685
686impl std::fmt::Display for HandlerKind {
687 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
688 formatter.write_str(match self {
689 Self::Workflow => "workflow",
690 Self::Activity => "activity",
691 })
692 }
693}
694
695#[derive(Clone, Copy, Debug, PartialEq, Eq)]
697pub enum HandlerValueKind {
698 Input,
699 Result,
700}
701
702impl std::fmt::Display for HandlerValueKind {
703 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
704 formatter.write_str(match self {
705 Self::Input => "input",
706 Self::Result => "result",
707 })
708 }
709}
710
711#[derive(Clone, Copy, Debug, PartialEq, Eq)]
713pub enum WorkflowCommandKind {
714 Cancel,
715 Terminate,
716}
717
718impl WorkflowCommandKind {
719 fn as_str(self) -> &'static str {
720 match self {
721 Self::Cancel => "cancel",
722 Self::Terminate => "terminate",
723 }
724 }
725}
726
727#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
729pub struct WorkflowCommandOptions {
730 #[serde(skip_serializing_if = "Option::is_none")]
731 pub reason: Option<String>,
732 #[serde(skip_serializing_if = "Option::is_none")]
733 pub request_id: Option<String>,
734}
735
736#[derive(Clone, Debug, PartialEq, Eq)]
743pub struct WorkflowStartOptions {
744 pub execution_timeout_seconds: u64,
745 pub run_timeout_seconds: u64,
746}
747
748impl Default for WorkflowStartOptions {
749 fn default() -> Self {
750 Self {
751 execution_timeout_seconds: 3600,
752 run_timeout_seconds: 600,
753 }
754 }
755}
756
757impl WorkflowStartOptions {
758 pub fn new() -> Self {
759 Self::default()
760 }
761
762 pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
763 self.execution_timeout_seconds = seconds;
764 self
765 }
766
767 pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
768 self.run_timeout_seconds = seconds;
769 self
770 }
771
772 fn validate(&self) -> Result<()> {
773 if self.execution_timeout_seconds == 0 {
774 return Err(Error::Codec(
775 "execution_timeout_seconds must be at least 1".to_string(),
776 ));
777 }
778 if self.run_timeout_seconds == 0 {
779 return Err(Error::Codec(
780 "run_timeout_seconds must be at least 1".to_string(),
781 ));
782 }
783 if self.run_timeout_seconds > self.execution_timeout_seconds {
784 return Err(Error::Codec(
785 "run_timeout_seconds cannot exceed execution_timeout_seconds".to_string(),
786 ));
787 }
788
789 Ok(())
790 }
791}
792
793#[derive(Clone, Debug, Default, PartialEq, Eq)]
798pub struct ContinueAsNewOptions {
799 pub workflow_type: Option<String>,
800 pub task_queue: Option<String>,
801}
802
803impl ContinueAsNewOptions {
804 pub fn new() -> Self {
805 Self::default()
806 }
807
808 pub fn workflow_type(mut self, workflow_type: impl Into<String>) -> Self {
809 self.workflow_type = Some(workflow_type.into());
810 self
811 }
812
813 pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
814 self.task_queue = Some(task_queue.into());
815 self
816 }
817
818 fn validate(&self) -> std::result::Result<(), ContinueAsNewOptionsError> {
819 for (field, value) in [
820 ("workflow_type", self.workflow_type.as_deref()),
821 ("task_queue", self.task_queue.as_deref()),
822 ] {
823 if value.is_some_and(|value| value.trim().is_empty()) {
824 return Err(ContinueAsNewOptionsError {
825 field,
826 message: format!("{field} must not be empty"),
827 });
828 }
829 }
830 Ok(())
831 }
832}
833
834#[derive(Clone, Debug, Error, PartialEq, Eq)]
836#[error("invalid continue-as-new option {field}: {message}")]
837pub struct ContinueAsNewOptionsError {
838 pub field: &'static str,
839 pub message: String,
840}
841
842#[derive(Clone, Debug, Default, PartialEq, Eq)]
844pub struct WorkflowHistoryBudget {
845 pub event_count: u64,
846 pub size_bytes: Option<u64>,
847 pub continue_as_new_recommended: bool,
848 pub pressure: Option<String>,
849}
850
851#[doc(hidden)]
852#[derive(Clone, Debug)]
853pub struct ContinueAsNewRequest {
854 arguments: AvroValue,
855 options: ContinueAsNewOptions,
856}
857
858impl WorkflowCommandOptions {
859 pub fn new() -> Self {
860 Self::default()
861 }
862
863 pub fn reason(mut self, reason: impl Into<String>) -> Self {
864 self.reason = Some(reason.into());
865 self
866 }
867
868 pub fn request_id(mut self, request_id: impl Into<String>) -> Self {
869 self.request_id = Some(request_id.into());
870 self
871 }
872}
873
874#[derive(Clone, Debug, PartialEq)]
876pub struct WorkflowCommandResult {
877 pub command: WorkflowCommandKind,
878 pub workflow_id: String,
879 pub run_id: Option<String>,
880 pub outcome: Option<String>,
881 pub reason: Option<String>,
882 pub command_status: Option<String>,
883 pub raw: Value,
884}
885
886#[derive(Clone, Debug, PartialEq)]
888pub struct WorkflowRedriveResult {
889 pub workflow_id: String,
890 pub source_run_id: String,
891 pub run_id: String,
892 pub resume_step_sequence: u64,
893 pub raw: Value,
894}
895
896#[derive(Clone, Debug, Error)]
898#[error("workflow {command:?} rejected ({reason}, HTTP {status}): {message}")]
899pub struct WorkflowCommandRejection {
900 pub command: WorkflowCommandKind,
901 pub status: u16,
902 pub reason: String,
903 pub message: String,
904 pub workflow_id: String,
905 pub run_id: Option<String>,
906 pub target_scope: Option<String>,
907 pub body: Value,
908}
909
910#[derive(Clone, Copy, Debug, PartialEq, Eq)]
912pub enum WorkflowTerminalKind {
913 Failed,
914 Cancelled,
915 Terminated,
916 TimedOut,
917}
918
919#[derive(Clone, Debug, Error)]
925#[error("workflow {workflow_id} run {run_id:?} ended as {kind:?} ({reason})")]
926pub struct WorkflowTerminalOutcome {
927 pub kind: WorkflowTerminalKind,
928 pub workflow_id: String,
929 pub run_id: Option<String>,
930 pub reason: String,
931 pub failure_category: Option<String>,
932 pub failure_id: Option<String>,
933 pub exception_type: Option<String>,
934 pub exception_class: Option<String>,
935 pub non_retryable: Option<bool>,
936 pub message: Option<String>,
937 pub exception: Option<Value>,
938 pub raw: Value,
939}
940
941#[derive(Clone, Debug, Error)]
943#[error("activity task {operation} rejected ({reason}, HTTP {status})")]
944pub struct ActivityTaskRejection {
945 pub operation: String,
946 pub status: u16,
947 pub reason: String,
948 pub task_id: String,
949 pub activity_attempt_id: String,
950 pub cancel_requested: bool,
951 pub can_continue: Option<bool>,
952 pub run_closed_reason: Option<String>,
953 pub body: Value,
954}
955
956#[derive(Clone, Copy, Debug, PartialEq, Eq)]
958pub enum ActivityOptionsErrorKind {
959 EmptyTaskQueue,
960 EmptyRetryPolicy,
961 InvalidMaxAttempts,
962 BackoffWithoutRetryBudget,
963 TooManyBackoffIntervals,
964 InvalidBackoffCoefficient,
965 BackoffGenerationTooLarge,
966 BackoffOverflow,
967 EmptyNonRetryableErrorType,
968 TimeoutNotPositive,
969 TimeoutOverflow,
970 TimeoutOrder,
971}
972
973#[derive(Clone, Debug, Error, PartialEq, Eq)]
975#[error("invalid activity options ({kind:?}, {field:?}): {message}")]
976pub struct ActivityOptionsError {
977 pub kind: ActivityOptionsErrorKind,
978 pub field: Option<&'static str>,
979 pub message: String,
980}
981
982impl ActivityOptionsError {
983 fn new(
984 kind: ActivityOptionsErrorKind,
985 field: Option<&'static str>,
986 message: impl Into<String>,
987 ) -> Self {
988 Self {
989 kind,
990 field,
991 message: message.into(),
992 }
993 }
994}
995
996#[derive(Clone, Copy, Debug, PartialEq, Eq)]
998pub enum ActivityFailureKind {
999 Failed,
1000 Cancelled,
1001 TimedOut,
1002}
1003
1004#[derive(Clone, Debug, Error, PartialEq)]
1009#[error("activity failed ({reason}): {message}")]
1010pub struct ActivityFailure {
1011 pub kind: ActivityFailureKind,
1012 pub reason: String,
1013 pub message: String,
1014 pub activity_execution_id: Option<String>,
1015 pub activity_attempt_id: Option<String>,
1016 pub activity_type: Option<String>,
1017 pub activity_class: Option<String>,
1018 pub attempt_number: Option<u64>,
1019 pub failure_id: Option<String>,
1020 pub failure_category: Option<String>,
1021 pub timeout_kind: Option<String>,
1022 pub non_retryable: bool,
1023 pub exception_type: Option<String>,
1024 pub exception_class: Option<String>,
1025 pub code: Option<Value>,
1026 pub exception: Option<Value>,
1027}
1028
1029#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1031pub enum ChildWorkflowFailureKind {
1032 Failed,
1033 Cancelled,
1034 Terminated,
1035}
1036
1037#[derive(Clone, Debug, Error)]
1043#[error("child workflow failed ({reason}): {message}")]
1044pub struct ChildWorkflowFailure {
1045 pub kind: ChildWorkflowFailureKind,
1046 pub reason: String,
1047 pub message: String,
1048 pub parent_workflow_id: Option<String>,
1049 pub parent_workflow_run_id: Option<String>,
1050 pub child_workflow_id: Option<String>,
1051 pub child_workflow_run_id: Option<String>,
1052 pub child_workflow_type: Option<String>,
1053 pub failure_id: Option<String>,
1054 pub failure_category: Option<String>,
1055 pub exception_type: Option<String>,
1056 pub exception_class: Option<String>,
1057 pub non_retryable: bool,
1058 pub code: Option<Value>,
1059 pub exception: Option<Value>,
1060}
1061
1062#[derive(Clone, Debug, PartialEq, Eq)]
1064pub struct WorkflowIdentity {
1065 pub workflow_id: Option<String>,
1066 pub run_id: Option<String>,
1067}
1068
1069#[derive(Clone, Debug, PartialEq)]
1071pub struct ChildWorkflowResult {
1072 pub parent: WorkflowIdentity,
1073 pub child: WorkflowIdentity,
1074 pub child_workflow_type: Option<String>,
1075 pub result: Value,
1076}
1077
1078#[derive(Clone, Debug, PartialEq)]
1080pub struct ChildWorkflowAvroResult {
1081 pub parent: WorkflowIdentity,
1082 pub child: WorkflowIdentity,
1083 pub child_workflow_type: Option<String>,
1084 pub result: AvroValue,
1085}
1086
1087#[derive(Clone, Debug, Deserialize, Hash, PartialEq, Eq, Serialize)]
1089#[serde(untagged)]
1090pub enum SelectionKey {
1091 Index(usize),
1092 Name(String),
1093}
1094
1095impl From<usize> for SelectionKey {
1096 fn from(value: usize) -> Self {
1097 Self::Index(value)
1098 }
1099}
1100
1101impl From<String> for SelectionKey {
1102 fn from(value: String) -> Self {
1103 Self::Name(value)
1104 }
1105}
1106
1107impl From<&str> for SelectionKey {
1108 fn from(value: &str) -> Self {
1109 Self::Name(value.to_string())
1110 }
1111}
1112
1113#[derive(Clone, Debug, Error, PartialEq, Eq)]
1115#[error("selected {operation_kind} operation {operation_identity} was explicitly cancelled")]
1116pub struct DurableOperationCancelled {
1117 pub selection_group_id: String,
1118 pub member_key: SelectionKey,
1119 pub member_index: usize,
1120 pub operation_kind: String,
1121 pub operation_identity: String,
1122}
1123
1124#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
1130pub struct ParallelGroupMetadata {
1131 pub parallel_group_id: String,
1132 pub parallel_group_kind: String,
1133 pub parallel_group_base_sequence: u64,
1134 pub parallel_group_size: usize,
1135 pub parallel_group_index: usize,
1136 #[serde(default, skip_serializing_if = "Option::is_none")]
1137 pub parallel_group_mode: Option<String>,
1138 #[serde(default, skip_serializing_if = "Option::is_none")]
1139 pub selection_member_key: Option<SelectionKey>,
1140 #[serde(default, skip_serializing_if = "Option::is_none")]
1141 pub selection_member_index: Option<usize>,
1142 #[serde(default, skip_serializing_if = "Option::is_none")]
1143 pub selection_member_base_sequence: Option<u64>,
1144 #[serde(default, skip_serializing_if = "Option::is_none")]
1145 pub selection_member_size: Option<usize>,
1146 #[serde(default, skip_serializing_if = "Option::is_none")]
1147 pub selection_member_kind: Option<String>,
1148}
1149
1150#[derive(Clone, Debug, PartialEq)]
1152pub enum ParallelResult {
1153 Activity(Value),
1154 ChildWorkflow(ChildWorkflowResult),
1155 Timer,
1156 Signal(Vec<Value>),
1157 Condition(ConditionWaitResult),
1158 Group(Vec<ParallelResult>),
1159}
1160
1161#[derive(Clone, Debug, PartialEq)]
1163pub enum ParallelAvroResult {
1164 Activity(AvroValue),
1165 ChildWorkflow(ChildWorkflowAvroResult),
1166 Timer,
1167 Signal(Vec<AvroValue>),
1168 Condition(ConditionWaitResult),
1169 Group(Vec<ParallelAvroResult>),
1170}
1171
1172impl ParallelAvroResult {
1173 fn into_json_result(self) -> Result<ParallelResult> {
1174 match self {
1175 Self::Activity(value) => Ok(ParallelResult::Activity(value.into_json()?)),
1176 Self::ChildWorkflow(result) => Ok(ParallelResult::ChildWorkflow(ChildWorkflowResult {
1177 parent: result.parent,
1178 child: result.child,
1179 child_workflow_type: result.child_workflow_type,
1180 result: result.result.into_json()?,
1181 })),
1182 Self::Timer => Ok(ParallelResult::Timer),
1183 Self::Signal(values) => Ok(ParallelResult::Signal(
1184 values
1185 .into_iter()
1186 .map(AvroValue::into_json)
1187 .collect::<Result<Vec<_>>>()?,
1188 )),
1189 Self::Condition(result) => Ok(ParallelResult::Condition(result)),
1190 Self::Group(results) => Ok(ParallelResult::Group(
1191 results
1192 .into_iter()
1193 .map(Self::into_json_result)
1194 .collect::<Result<Vec<_>>>()?,
1195 )),
1196 }
1197 }
1198}
1199
1200#[derive(Clone, Debug, PartialEq)]
1202pub struct ParallelCompletion {
1203 pub member_path: Vec<usize>,
1204 pub result: ParallelResult,
1205}
1206
1207#[derive(Debug, Error)]
1215#[error("parallel group {group_id} member {member_path:?} failed: {cause}")]
1216pub struct ParallelFailure {
1217 pub group_id: String,
1218 pub member_path: Vec<usize>,
1219 pub group_path: Vec<ParallelGroupMetadata>,
1220 pub completed: Vec<ParallelCompletion>,
1221 #[source]
1222 pub cause: Box<Error>,
1223}
1224
1225#[derive(Clone, Debug, Error, PartialEq, Eq)]
1227#[error("invalid deterministic parallel group ({reason}): {message}")]
1228pub struct ParallelGroupError {
1229 pub reason: &'static str,
1230 pub member_path: Vec<usize>,
1231 pub message: String,
1232}
1233
1234#[derive(Clone, Debug, Error, PartialEq, Eq)]
1236#[error("workflow cancellation was requested")]
1237pub struct WorkflowCancellationRequested;
1238
1239#[derive(Debug, Error)]
1241#[error(
1242 "saga forward execution failed; compensation activity {compensation_activity_type} (registration {compensation_registration_order}) also failed: {compensation_failure}"
1243)]
1244pub struct SagaCompensationFailure {
1245 pub initiating_failure: Box<Error>,
1246 pub compensation_failure: Box<Error>,
1247 pub compensation_activity_type: String,
1248 pub compensation_registration_order: usize,
1249}
1250
1251#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
1253pub enum ParentClosePolicy {
1254 #[default]
1255 Abandon,
1256 RequestCancel,
1257 Terminate,
1258}
1259
1260impl ParentClosePolicy {
1261 fn as_str(self) -> &'static str {
1262 match self {
1263 Self::Abandon => "abandon",
1264 Self::RequestCancel => "request_cancel",
1265 Self::Terminate => "terminate",
1266 }
1267 }
1268}
1269
1270#[derive(Clone, Debug, Default, PartialEq, Eq)]
1272pub struct ChildWorkflowRetryPolicy {
1273 pub max_attempts: Option<u32>,
1274 pub backoff_seconds: Vec<u64>,
1275 pub non_retryable_error_types: Vec<String>,
1276}
1277
1278#[derive(Clone, Debug, PartialEq, Eq)]
1282pub struct ChildWorkflowOptions {
1283 pub task_queue: String,
1284 pub parent_close_policy: ParentClosePolicy,
1285 pub retry_policy: Option<ChildWorkflowRetryPolicy>,
1286 pub execution_timeout_seconds: Option<u64>,
1287 pub run_timeout_seconds: Option<u64>,
1288}
1289
1290impl ChildWorkflowOptions {
1291 pub fn new(task_queue: impl Into<String>) -> Self {
1292 Self {
1293 task_queue: task_queue.into(),
1294 parent_close_policy: ParentClosePolicy::Abandon,
1295 retry_policy: None,
1296 execution_timeout_seconds: None,
1297 run_timeout_seconds: None,
1298 }
1299 }
1300
1301 pub fn parent_close_policy(mut self, policy: ParentClosePolicy) -> Self {
1302 self.parent_close_policy = policy;
1303 self
1304 }
1305
1306 pub fn retry_policy(mut self, policy: ChildWorkflowRetryPolicy) -> Self {
1307 self.retry_policy = Some(policy);
1308 self
1309 }
1310
1311 pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
1312 self.execution_timeout_seconds = Some(seconds);
1313 self
1314 }
1315
1316 pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
1317 self.run_timeout_seconds = Some(seconds);
1318 self
1319 }
1320}
1321
1322#[derive(Clone, Debug, PartialEq, Eq)]
1324pub enum ActivityBackoff {
1325 Explicit(Vec<Duration>),
1328 Exponential {
1330 initial_interval: Duration,
1331 coefficient: u32,
1332 maximum_interval: Option<Duration>,
1333 },
1334}
1335
1336#[derive(Clone, Debug, Default, PartialEq, Eq)]
1338pub struct ActivityRetryPolicy {
1339 pub max_attempts: Option<u32>,
1340 pub backoff: Option<ActivityBackoff>,
1341 pub non_retryable_error_types: Vec<String>,
1342}
1343
1344impl ActivityRetryPolicy {
1345 pub fn new(max_attempts: u32) -> Self {
1347 Self {
1348 max_attempts: Some(max_attempts),
1349 ..Self::default()
1350 }
1351 }
1352
1353 pub fn backoff_intervals(mut self, intervals: impl IntoIterator<Item = Duration>) -> Self {
1354 self.backoff = Some(ActivityBackoff::Explicit(intervals.into_iter().collect()));
1355 self
1356 }
1357
1358 pub fn exponential_backoff(
1359 mut self,
1360 initial_interval: Duration,
1361 coefficient: u32,
1362 maximum_interval: Option<Duration>,
1363 ) -> Self {
1364 self.backoff = Some(ActivityBackoff::Exponential {
1365 initial_interval,
1366 coefficient,
1367 maximum_interval,
1368 });
1369 self
1370 }
1371
1372 pub fn non_retryable_error_type(mut self, error_type: impl Into<String>) -> Self {
1373 self.non_retryable_error_types.push(error_type.into());
1374 self
1375 }
1376
1377 pub fn non_retryable_error_types(
1378 mut self,
1379 error_types: impl IntoIterator<Item = impl Into<String>>,
1380 ) -> Self {
1381 self.non_retryable_error_types
1382 .extend(error_types.into_iter().map(Into::into));
1383 self
1384 }
1385}
1386
1387#[derive(Clone, Debug, Default, PartialEq, Eq)]
1392pub struct ActivityOptions {
1393 pub task_queue: Option<String>,
1394 pub retry_policy: Option<ActivityRetryPolicy>,
1395 pub start_to_close_timeout: Option<Duration>,
1396 pub schedule_to_start_timeout: Option<Duration>,
1397 pub schedule_to_close_timeout: Option<Duration>,
1398 pub heartbeat_timeout: Option<Duration>,
1399}
1400
1401impl ActivityOptions {
1402 pub fn new() -> Self {
1403 Self::default()
1404 }
1405
1406 pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
1407 self.task_queue = Some(task_queue.into());
1408 self
1409 }
1410
1411 pub fn retry_policy(mut self, policy: ActivityRetryPolicy) -> Self {
1412 self.retry_policy = Some(policy);
1413 self
1414 }
1415
1416 pub fn start_to_close_timeout(mut self, timeout: Duration) -> Self {
1417 self.start_to_close_timeout = Some(timeout);
1418 self
1419 }
1420
1421 pub fn schedule_to_start_timeout(mut self, timeout: Duration) -> Self {
1422 self.schedule_to_start_timeout = Some(timeout);
1423 self
1424 }
1425
1426 pub fn schedule_to_close_timeout(mut self, timeout: Duration) -> Self {
1427 self.schedule_to_close_timeout = Some(timeout);
1428 self
1429 }
1430
1431 pub fn heartbeat_timeout(mut self, timeout: Duration) -> Self {
1432 self.heartbeat_timeout = Some(timeout);
1433 self
1434 }
1435
1436 fn validate(&self) -> std::result::Result<ValidatedActivityOptions, ActivityOptionsError> {
1437 if self
1438 .task_queue
1439 .as_deref()
1440 .is_some_and(|queue| queue.trim().is_empty())
1441 {
1442 return Err(ActivityOptionsError::new(
1443 ActivityOptionsErrorKind::EmptyTaskQueue,
1444 Some("task_queue"),
1445 "task_queue must not be empty",
1446 ));
1447 }
1448
1449 for (field, value) in [
1450 ("start_to_close_timeout", self.start_to_close_timeout),
1451 ("schedule_to_start_timeout", self.schedule_to_start_timeout),
1452 ("schedule_to_close_timeout", self.schedule_to_close_timeout),
1453 ("heartbeat_timeout", self.heartbeat_timeout),
1454 ] {
1455 if value.is_some_and(|value| value.is_zero()) {
1456 return Err(ActivityOptionsError::new(
1457 ActivityOptionsErrorKind::TimeoutNotPositive,
1458 Some(field),
1459 format!("{field} must be positive"),
1460 ));
1461 }
1462 }
1463
1464 validate_timeout_order(
1465 "heartbeat_timeout",
1466 self.heartbeat_timeout,
1467 "start_to_close_timeout",
1468 self.start_to_close_timeout,
1469 )?;
1470 validate_timeout_order(
1471 "start_to_close_timeout",
1472 self.start_to_close_timeout,
1473 "schedule_to_close_timeout",
1474 self.schedule_to_close_timeout,
1475 )?;
1476 validate_timeout_order(
1477 "schedule_to_start_timeout",
1478 self.schedule_to_start_timeout,
1479 "schedule_to_close_timeout",
1480 self.schedule_to_close_timeout,
1481 )?;
1482
1483 Ok(ValidatedActivityOptions {
1484 task_queue: self.task_queue.clone(),
1485 retry_policy: self
1486 .retry_policy
1487 .as_ref()
1488 .map(validate_activity_retry_policy)
1489 .transpose()?,
1490 start_to_close_timeout: timeout_seconds(
1491 "start_to_close_timeout",
1492 self.start_to_close_timeout,
1493 )?,
1494 schedule_to_start_timeout: timeout_seconds(
1495 "schedule_to_start_timeout",
1496 self.schedule_to_start_timeout,
1497 )?,
1498 schedule_to_close_timeout: timeout_seconds(
1499 "schedule_to_close_timeout",
1500 self.schedule_to_close_timeout,
1501 )?,
1502 heartbeat_timeout: timeout_seconds("heartbeat_timeout", self.heartbeat_timeout)?,
1503 })
1504 }
1505}
1506
1507pub enum ParallelOperation {
1513 Activity {
1514 activity_type: String,
1515 options: ActivityOptions,
1516 arguments: Result<AvroValue>,
1517 },
1518 ChildWorkflow {
1519 workflow_type: String,
1520 options: ChildWorkflowOptions,
1521 arguments: Result<AvroValue>,
1522 },
1523 Timer(Duration),
1524 Signal(String),
1525 Condition {
1526 options: ConditionWaitOptions,
1527 predicate: Box<dyn Fn() -> Result<bool> + Send + 'static>,
1528 },
1529 Group(Vec<ParallelOperation>),
1530}
1531
1532impl ParallelOperation {
1533 pub fn activity<T: Serialize>(activity_type: impl Into<String>, args: T) -> Self {
1534 Self::activity_with_options(activity_type, ActivityOptions::new(), args)
1535 }
1536
1537 pub fn activity_with_options<T: Serialize>(
1538 activity_type: impl Into<String>,
1539 options: ActivityOptions,
1540 args: T,
1541 ) -> Self {
1542 Self::Activity {
1543 activity_type: activity_type.into(),
1544 options,
1545 arguments: AvroValue::from_serialize(&args),
1546 }
1547 }
1548
1549 pub fn child_workflow<T: Serialize>(
1550 workflow_type: impl Into<String>,
1551 options: ChildWorkflowOptions,
1552 args: T,
1553 ) -> Self {
1554 Self::ChildWorkflow {
1555 workflow_type: workflow_type.into(),
1556 options,
1557 arguments: AvroValue::from_serialize(&args),
1558 }
1559 }
1560
1561 pub fn timer(duration: Duration) -> Self {
1562 Self::Timer(duration)
1563 }
1564
1565 pub fn signal(signal_name: impl Into<String>) -> Self {
1566 Self::Signal(signal_name.into())
1567 }
1568
1569 pub fn condition<F>(options: ConditionWaitOptions, predicate: F) -> Self
1570 where
1571 F: Fn() -> Result<bool> + Send + 'static,
1572 {
1573 Self::Condition {
1574 options,
1575 predicate: Box::new(predicate),
1576 }
1577 }
1578
1579 pub fn group(operations: Vec<ParallelOperation>) -> Self {
1580 Self::Group(operations)
1581 }
1582}
1583
1584#[derive(Clone, Debug)]
1585struct ValidatedActivityOptions {
1586 task_queue: Option<String>,
1587 retry_policy: Option<Value>,
1588 start_to_close_timeout: Option<u64>,
1589 schedule_to_start_timeout: Option<u64>,
1590 schedule_to_close_timeout: Option<u64>,
1591 heartbeat_timeout: Option<u64>,
1592}
1593
1594fn validate_timeout_order(
1595 smaller_name: &'static str,
1596 smaller: Option<Duration>,
1597 larger_name: &'static str,
1598 larger: Option<Duration>,
1599) -> std::result::Result<(), ActivityOptionsError> {
1600 if matches!((smaller, larger), (Some(smaller), Some(larger)) if smaller > larger) {
1601 return Err(ActivityOptionsError::new(
1602 ActivityOptionsErrorKind::TimeoutOrder,
1603 Some(smaller_name),
1604 format!("{smaller_name} must be <= {larger_name}"),
1605 ));
1606 }
1607 Ok(())
1608}
1609
1610fn timeout_seconds(
1611 field: &'static str,
1612 value: Option<Duration>,
1613) -> std::result::Result<Option<u64>, ActivityOptionsError> {
1614 value
1615 .map(|value| {
1616 activity_protocol_seconds(value).ok_or_else(|| {
1617 ActivityOptionsError::new(
1618 ActivityOptionsErrorKind::TimeoutOverflow,
1619 Some(field),
1620 format!("{field} is too large for the worker protocol"),
1621 )
1622 })
1623 })
1624 .transpose()
1625}
1626
1627fn duration_seconds_ceil(value: Duration) -> Option<u64> {
1628 value
1629 .as_secs()
1630 .checked_add(u64::from(value.subsec_nanos() > 0))
1631}
1632
1633fn activity_protocol_seconds(value: Duration) -> Option<u64> {
1634 duration_seconds_ceil(value).filter(|seconds| *seconds <= i64::MAX as u64)
1635}
1636
1637fn validate_activity_retry_policy(
1638 policy: &ActivityRetryPolicy,
1639) -> std::result::Result<Value, ActivityOptionsError> {
1640 if policy.max_attempts.is_none()
1641 && policy.backoff.is_none()
1642 && policy.non_retryable_error_types.is_empty()
1643 {
1644 return Err(ActivityOptionsError::new(
1645 ActivityOptionsErrorKind::EmptyRetryPolicy,
1646 Some("retry_policy"),
1647 "retry_policy must configure at least one field",
1648 ));
1649 }
1650 if policy.max_attempts == Some(0) {
1651 return Err(ActivityOptionsError::new(
1652 ActivityOptionsErrorKind::InvalidMaxAttempts,
1653 Some("retry_policy.max_attempts"),
1654 "max_attempts must be >= 1",
1655 ));
1656 }
1657 if policy
1658 .non_retryable_error_types
1659 .iter()
1660 .any(|error_type| error_type.trim().is_empty())
1661 {
1662 return Err(ActivityOptionsError::new(
1663 ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
1664 Some("retry_policy.non_retryable_error_types"),
1665 "non_retryable_error_types must not contain empty values",
1666 ));
1667 }
1668
1669 let backoff_seconds = match &policy.backoff {
1670 None => None,
1671 Some(backoff) => {
1672 let max_attempts = policy.max_attempts.ok_or_else(|| {
1673 ActivityOptionsError::new(
1674 ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
1675 Some("retry_policy.backoff"),
1676 "backoff requires max_attempts",
1677 )
1678 })?;
1679 let retry_count = max_attempts.saturating_sub(1) as usize;
1680 let intervals = match backoff {
1681 ActivityBackoff::Explicit(intervals) => {
1682 if intervals.len() > retry_count {
1683 return Err(ActivityOptionsError::new(
1684 ActivityOptionsErrorKind::TooManyBackoffIntervals,
1685 Some("retry_policy.backoff"),
1686 "backoff interval count must not exceed max_attempts - 1",
1687 ));
1688 }
1689 intervals.clone()
1690 }
1691 ActivityBackoff::Exponential {
1692 initial_interval,
1693 coefficient,
1694 maximum_interval,
1695 } => {
1696 if *coefficient < 1 {
1697 return Err(ActivityOptionsError::new(
1698 ActivityOptionsErrorKind::InvalidBackoffCoefficient,
1699 Some("retry_policy.backoff.coefficient"),
1700 "backoff coefficient must be >= 1",
1701 ));
1702 }
1703 if retry_count > 10_000 {
1704 return Err(ActivityOptionsError::new(
1705 ActivityOptionsErrorKind::BackoffGenerationTooLarge,
1706 Some("retry_policy.max_attempts"),
1707 "generated backoff supports at most 10000 retry intervals",
1708 ));
1709 }
1710 let mut current = *initial_interval;
1711 let mut intervals = Vec::with_capacity(retry_count);
1712 for _ in 0..retry_count {
1713 let interval = maximum_interval
1714 .map(|maximum| current.min(maximum))
1715 .unwrap_or(current);
1716 intervals.push(interval);
1717 if maximum_interval.is_some_and(|maximum| interval == maximum) {
1718 break;
1719 }
1720 current = current.checked_mul(*coefficient).ok_or_else(|| {
1721 ActivityOptionsError::new(
1722 ActivityOptionsErrorKind::BackoffOverflow,
1723 Some("retry_policy.backoff"),
1724 "generated backoff interval overflowed",
1725 )
1726 })?;
1727 }
1728 intervals
1729 }
1730 };
1731 Some(
1732 intervals
1733 .into_iter()
1734 .map(|interval| {
1735 activity_protocol_seconds(interval).ok_or_else(|| {
1736 ActivityOptionsError::new(
1737 ActivityOptionsErrorKind::BackoffOverflow,
1738 Some("retry_policy.backoff"),
1739 "backoff interval is too large for the worker protocol",
1740 )
1741 })
1742 })
1743 .collect::<std::result::Result<Vec<_>, _>>()?,
1744 )
1745 }
1746 };
1747
1748 let mut encoded = serde_json::Map::new();
1749 if let Some(max_attempts) = policy.max_attempts {
1750 encoded.insert("max_attempts".to_string(), json!(max_attempts));
1751 }
1752 if let Some(backoff_seconds) = backoff_seconds {
1753 encoded.insert("backoff_seconds".to_string(), json!(backoff_seconds));
1754 }
1755 if !policy.non_retryable_error_types.is_empty() {
1756 let mut canonical_error_types = Vec::new();
1757 for error_type in policy
1758 .non_retryable_error_types
1759 .iter()
1760 .map(|error_type| error_type.trim())
1761 {
1762 if !canonical_error_types.contains(&error_type) {
1763 canonical_error_types.push(error_type);
1764 }
1765 }
1766 encoded.insert(
1767 "non_retryable_error_types".to_string(),
1768 json!(canonical_error_types),
1769 );
1770 }
1771 Ok(Value::Object(encoded))
1772}
1773
1774#[derive(Clone, Debug, Error)]
1777#[error("non-deterministic workflow replay ({reason}) at sequence {sequence:?}: {message}")]
1778pub struct ReplayFailure {
1779 pub reason: String,
1780 pub sequence: Option<u64>,
1781 pub expected: Option<String>,
1782 pub actual: Option<String>,
1783 pub message: String,
1784}
1785
1786impl ReplayFailure {
1787 fn new(
1788 reason: impl Into<String>,
1789 sequence: Option<u64>,
1790 expected: Option<String>,
1791 actual: Option<String>,
1792 message: impl Into<String>,
1793 ) -> Self {
1794 Self {
1795 reason: reason.into(),
1796 sequence,
1797 expected,
1798 actual,
1799 message: message.into(),
1800 }
1801 }
1802}
1803
1804#[derive(Clone, Debug, Error)]
1806#[error("query failed ({reason}, HTTP {status}): {message}")]
1807pub struct QueryFailure {
1808 pub status: u16,
1809 pub reason: String,
1810 pub message: String,
1811 pub body: Value,
1812}
1813
1814#[derive(Clone, Debug, Error)]
1816#[error("protocol rejected ({reason}, HTTP {status}): {message}")]
1817pub struct ProtocolFailure {
1818 pub status: u16,
1819 pub reason: String,
1820 pub message: String,
1821 pub supported_version: Option<String>,
1822 pub requested_version: Option<String>,
1823 pub body: Value,
1824}
1825
1826#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
1827pub struct PayloadEnvelope {
1828 pub codec: String,
1829 pub blob: String,
1830}
1831
1832impl PayloadEnvelope {
1833 pub fn avro<T: Serialize>(value: &T) -> Result<Self> {
1834 encode_payload(value, DEFAULT_CODEC)
1835 }
1836
1837 pub fn avro_value(value: &AvroValue) -> Result<Self> {
1840 encode_avro_value(value)
1841 }
1842}
1843
1844#[derive(Clone, Debug)]
1846pub enum AvroValue {
1847 Null,
1848 Boolean(bool),
1849 Long(i64),
1850 Double(f64),
1851 Bytes(Vec<u8>),
1852 String(String),
1853 Array(Vec<AvroValue>),
1854 Map(BTreeMap<String, AvroValue>),
1855}
1856
1857impl PartialEq for AvroValue {
1858 fn eq(&self, other: &Self) -> bool {
1859 match (self, other) {
1860 (Self::Null, Self::Null) => true,
1861 (Self::Boolean(left), Self::Boolean(right)) => left == right,
1862 (Self::Long(left), Self::Long(right)) => left == right,
1863 (Self::Double(left), Self::Double(right)) => left.to_bits() == right.to_bits(),
1864 (Self::Bytes(left), Self::Bytes(right)) => left == right,
1865 (Self::String(left), Self::String(right)) => left == right,
1866 (Self::Array(left), Self::Array(right)) => left == right,
1867 (Self::Map(left), Self::Map(right)) => left == right,
1868 _ => false,
1869 }
1870 }
1871}
1872
1873impl AvroValue {
1874 fn from_serialize<T: Serialize>(value: &T) -> Result<Self> {
1875 Self::from_serde_value(
1876 serde_value::to_value(value).map_err(|error| {
1877 Error::Codec(format!("could not adapt value for Avro: {error}"))
1878 })?,
1879 )
1880 }
1881
1882 fn from_serde_value(value: serde_value::Value) -> Result<Self> {
1883 use serde_value::Value as SerdeValue;
1884
1885 match value {
1886 SerdeValue::Unit => Ok(Self::Null),
1887 SerdeValue::Bool(value) => Ok(Self::Boolean(value)),
1888 SerdeValue::I8(value) => Ok(Self::Long(i64::from(value))),
1889 SerdeValue::I16(value) => Ok(Self::Long(i64::from(value))),
1890 SerdeValue::I32(value) => Ok(Self::Long(i64::from(value))),
1891 SerdeValue::I64(value) => Ok(Self::Long(value)),
1892 SerdeValue::U8(value) => Ok(Self::Long(i64::from(value))),
1893 SerdeValue::U16(value) => Ok(Self::Long(i64::from(value))),
1894 SerdeValue::U32(value) => Ok(Self::Long(i64::from(value))),
1895 SerdeValue::U64(value) => i64::try_from(value).map(Self::Long).map_err(|_| {
1896 Error::Codec(
1897 "integer_overflow: Avro Value long must be within signed 64-bit range"
1898 .to_string(),
1899 )
1900 }),
1901 SerdeValue::F32(value) => Self::finite_double(f64::from(value)),
1902 SerdeValue::F64(value) => Self::finite_double(value),
1903 SerdeValue::Char(value) => Ok(Self::String(value.to_string())),
1904 SerdeValue::String(value) => Ok(Self::String(value)),
1905 SerdeValue::Bytes(value) => Ok(Self::Bytes(value)),
1906 SerdeValue::Option(None) => Ok(Self::Null),
1907 SerdeValue::Option(Some(value)) | SerdeValue::Newtype(value) => {
1908 Self::from_serde_value(*value)
1909 }
1910 SerdeValue::Seq(values) => values
1911 .into_iter()
1912 .map(Self::from_serde_value)
1913 .collect::<Result<Vec<_>>>()
1914 .map(Self::Array),
1915 SerdeValue::Map(values) => values
1916 .into_iter()
1917 .map(|(key, value)| {
1918 let SerdeValue::String(key) = key else {
1919 return Err(Error::Codec(
1920 "invalid_map_key: Avro Value map keys must be strings".to_string(),
1921 ));
1922 };
1923
1924 Ok((key, Self::from_serde_value(value)?))
1925 })
1926 .collect::<Result<BTreeMap<_, _>>>()
1927 .map(Self::Map),
1928 }
1929 }
1930
1931 fn finite_double(value: f64) -> Result<Self> {
1932 if !value.is_finite() {
1933 return Err(Error::Codec(
1934 "non_finite_float: Avro Value doubles must be finite".to_string(),
1935 ));
1936 }
1937
1938 Ok(Self::Double(value))
1939 }
1940
1941 fn into_json(self) -> Result<Value> {
1942 match self {
1943 Self::Null => Ok(Value::Null),
1944 Self::Boolean(value) => Ok(Value::Bool(value)),
1945 Self::Long(value) => Ok(Value::Number(value.into())),
1946 Self::Double(value) => serde_json::Number::from_f64(value)
1947 .map(Value::Number)
1948 .ok_or_else(|| {
1949 Error::Codec(
1950 "non_finite_float: decoded Avro Value double is not finite".to_string(),
1951 )
1952 }),
1953 Self::Bytes(value) => Ok(json!({
1954 "$type": "bytes",
1955 "base64": BASE64.encode(value),
1956 })),
1957 Self::String(value) => Ok(Value::String(value)),
1958 Self::Array(values) => values
1959 .into_iter()
1960 .map(Self::into_json)
1961 .collect::<Result<Vec<_>>>()
1962 .map(Value::Array),
1963 Self::Map(values) => values
1964 .into_iter()
1965 .map(|(key, value)| Ok((key, value.into_json()?)))
1966 .collect::<Result<serde_json::Map<_, _>>>()
1967 .map(Value::Object),
1968 }
1969 }
1970
1971 fn into_serde_value(self) -> serde_value::Value {
1972 use serde_value::Value as SerdeValue;
1973
1974 match self {
1975 Self::Null => SerdeValue::Unit,
1976 Self::Boolean(value) => SerdeValue::Bool(value),
1977 Self::Long(value) => SerdeValue::I64(value),
1978 Self::Double(value) => SerdeValue::F64(value),
1979 Self::Bytes(value) => SerdeValue::Bytes(value),
1980 Self::String(value) => SerdeValue::String(value),
1981 Self::Array(values) => {
1982 SerdeValue::Seq(values.into_iter().map(Self::into_serde_value).collect())
1983 }
1984 Self::Map(values) => SerdeValue::Map(
1985 values
1986 .into_iter()
1987 .map(|(key, value)| (SerdeValue::String(key), value.into_serde_value()))
1988 .collect(),
1989 ),
1990 }
1991 }
1992
1993 pub fn deserialize<T: DeserializeOwned>(self) -> Result<T> {
1994 self.into_serde_value().deserialize_into().map_err(|error| {
1995 Error::Codec(format!(
1996 "avro_value_type_mismatch: could not adapt decoded value: {error}"
1997 ))
1998 })
1999 }
2000}
2001
2002impl Serialize for AvroValue {
2003 fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
2004 where
2005 S: Serializer,
2006 {
2007 match self {
2008 Self::Null => serializer.serialize_unit(),
2009 Self::Boolean(value) => serializer.serialize_bool(*value),
2010 Self::Long(value) => serializer.serialize_i64(*value),
2011 Self::Double(value) => serializer.serialize_f64(*value),
2012 Self::Bytes(value) => serializer.serialize_bytes(value),
2013 Self::String(value) => serializer.serialize_str(value),
2014 Self::Array(values) => {
2015 let mut sequence = serializer.serialize_seq(Some(values.len()))?;
2016 for value in values {
2017 sequence.serialize_element(value)?;
2018 }
2019 sequence.end()
2020 }
2021 Self::Map(values) => {
2022 let mut map = serializer.serialize_map(Some(values.len()))?;
2023 for (key, value) in values {
2024 map.serialize_entry(key, value)?;
2025 }
2026 map.end()
2027 }
2028 }
2029 }
2030}
2031
2032pub fn encode_avro_value(value: &AvroValue) -> Result<PayloadEnvelope> {
2033 let datum = avro_value_to_datum(value)?;
2034 let datum = to_avro_datum(avro_value_ordered_map_encoding_schema()?, datum)
2035 .map_err(|err| Error::Codec(format!("avro_value_encode_failed: {err}")))?;
2036 let mut bytes = Vec::with_capacity(datum.len() + 10);
2037 bytes.extend_from_slice(&AVRO_SINGLE_OBJECT_MAGIC);
2038 bytes.extend_from_slice(&AVRO_VALUE_SCHEMA_FINGERPRINT);
2039 bytes.extend_from_slice(&datum);
2040 Ok(PayloadEnvelope {
2041 codec: DEFAULT_CODEC.to_string(),
2042 blob: BASE64.encode(bytes),
2043 })
2044}
2045
2046pub fn decode_avro_value(envelope: &PayloadEnvelope) -> Result<AvroValue> {
2047 if envelope.codec != DEFAULT_CODEC {
2048 return Err(unsupported_payload_codec(&envelope.codec));
2049 }
2050 decode_avro_value_blob(&envelope.blob)
2051}
2052
2053pub fn encode_payload<T: Serialize>(value: &T, codec: &str) -> Result<PayloadEnvelope> {
2054 let blob = match codec {
2055 DEFAULT_CODEC => encode_avro_value(&AvroValue::from_serialize(value)?)?.blob,
2056 other => return Err(unsupported_payload_codec(other)),
2057 };
2058
2059 Ok(PayloadEnvelope {
2060 codec: codec.to_string(),
2061 blob,
2062 })
2063}
2064
2065pub fn decode_payload<T: DeserializeOwned>(envelope: &PayloadEnvelope) -> Result<T> {
2066 match envelope.codec.as_str() {
2067 DEFAULT_CODEC => decode_avro_value(envelope)?.deserialize(),
2068 other => Err(unsupported_payload_codec(other)),
2069 }
2070}
2071
2072fn handler_type_error<T>(
2073 handler_kind: HandlerKind,
2074 handler_name: &str,
2075 value_kind: HandlerValueKind,
2076 message: impl Into<String>,
2077) -> Error {
2078 Error::HandlerType {
2079 handler_kind,
2080 handler_name: handler_name.to_string(),
2081 value_kind,
2082 rust_type: type_name::<T>(),
2083 message: message.into(),
2084 }
2085}
2086
2087fn decode_handler_input<T: DeserializeOwned>(
2088 arguments: AvroValue,
2089 handler_kind: HandlerKind,
2090 handler_name: &str,
2091) -> Result<T> {
2092 let argument = match arguments {
2093 AvroValue::Array(mut arguments) if arguments.len() == 1 => {
2094 arguments.pop().expect("one typed handler argument")
2095 }
2096 AvroValue::Array(arguments) if arguments.is_empty() => AvroValue::Null,
2097 AvroValue::Array(arguments) => {
2098 return Err(handler_type_error::<T>(
2099 handler_kind,
2100 handler_name,
2101 HandlerValueKind::Input,
2102 format!(
2103 "typed handlers accept one request value, but the task carried {} arguments",
2104 arguments.len()
2105 ),
2106 ));
2107 }
2108 argument => argument,
2109 };
2110
2111 argument.deserialize().map_err(|error| {
2112 handler_type_error::<T>(
2113 handler_kind,
2114 handler_name,
2115 HandlerValueKind::Input,
2116 error.to_string(),
2117 )
2118 })
2119}
2120
2121fn encode_handler_result<T: Serialize>(
2122 result: &T,
2123 handler_kind: HandlerKind,
2124 handler_name: &str,
2125) -> Result<AvroValue> {
2126 AvroValue::from_serialize(result).map_err(|error| {
2127 handler_type_error::<T>(
2128 handler_kind,
2129 handler_name,
2130 HandlerValueKind::Result,
2131 error.to_string(),
2132 )
2133 })
2134}
2135
2136fn decode_handler_result<T: DeserializeOwned>(
2137 result: AvroValue,
2138 handler_kind: HandlerKind,
2139 handler_name: &str,
2140) -> Result<T> {
2141 result.deserialize().map_err(|error| {
2142 handler_type_error::<T>(
2143 handler_kind,
2144 handler_name,
2145 HandlerValueKind::Result,
2146 error.to_string(),
2147 )
2148 })
2149}
2150
2151#[cfg(test)]
2152fn encode_value_envelope(value: &Value, codec: &str) -> Result<Value> {
2153 Ok(serde_json::to_value(encode_payload(value, codec)?)?)
2154}
2155
2156fn decode_wire_value(value: &Value, fallback_codec: &str) -> Result<Value> {
2157 validate_payload_codec(fallback_codec)?;
2158
2159 if value.is_null() {
2160 return Ok(Value::Null);
2161 }
2162
2163 if let Some((codec, blob)) = payload_envelope_parts(value)? {
2164 return decode_blob(blob, codec);
2165 }
2166
2167 if let Some(blob) = value.as_str() {
2168 return decode_blob(blob, fallback_codec);
2169 }
2170
2171 Err(untagged_payload_value())
2172}
2173
2174fn encode_typed_envelope(value: &AvroValue, codec: &str) -> Result<Value> {
2175 let envelope = match codec {
2176 DEFAULT_CODEC => encode_avro_value(value)?,
2177 other => return Err(unsupported_payload_codec(other)),
2178 };
2179 Ok(serde_json::to_value(envelope)?)
2180}
2181
2182fn decode_wire_avro_value(value: &Value, fallback_codec: &str) -> Result<AvroValue> {
2183 validate_payload_codec(fallback_codec)?;
2184
2185 if value.is_null() {
2186 return Ok(AvroValue::Null);
2187 }
2188
2189 if let Some((codec, blob)) = payload_envelope_parts(value)? {
2190 validate_payload_codec(codec)?;
2191 return decode_avro_value_blob(blob);
2192 }
2193
2194 if let Some(blob) = value.as_str() {
2195 return match fallback_codec {
2196 DEFAULT_CODEC => decode_avro_value_blob(blob),
2197 other => Err(unsupported_payload_codec(other)),
2198 };
2199 }
2200
2201 Err(untagged_payload_value())
2202}
2203
2204fn normalize_avro_arguments(value: AvroValue) -> AvroValue {
2205 match value {
2206 AvroValue::Null => AvroValue::Array(Vec::new()),
2207 AvroValue::Array(_) => value,
2208 other => AvroValue::Array(vec![other]),
2209 }
2210}
2211
2212fn decode_blob(blob: &str, codec: &str) -> Result<Value> {
2213 match codec {
2214 DEFAULT_CODEC => decode_avro_value_blob(blob)?.into_json(),
2215 other => Err(unsupported_payload_codec(other)),
2216 }
2217}
2218
2219fn validate_payload_codec(codec: &str) -> Result<()> {
2220 match codec {
2221 DEFAULT_CODEC => Ok(()),
2222 MISSING_TASK_PAYLOAD_CODEC => {
2223 Err(invalid_task_payload_codec("task payload_codec is missing"))
2224 }
2225 NULL_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec("task payload_codec is null")),
2226 NON_STRING_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec(
2227 "task payload_codec must be a string",
2228 )),
2229 other => Err(unsupported_payload_codec(other)),
2230 }
2231}
2232
2233fn invalid_task_payload_codec(reason: &str) -> Error {
2234 Error::Codec(format!(
2235 "unsupported_payload_codec: {reason}; Durable Workflow 2.0 requires an explicit string payload_codec=\"avro\" before worker task execution"
2236 ))
2237}
2238
2239fn payload_envelope_parts(value: &Value) -> Result<Option<(&str, &str)>> {
2240 let Some(object) = value.as_object() else {
2241 return Ok(None);
2242 };
2243 if !object.contains_key("codec") && !object.contains_key("blob") {
2244 return Ok(None);
2245 }
2246
2247 let codec = object
2248 .get("codec")
2249 .and_then(Value::as_str)
2250 .ok_or_else(invalid_payload_envelope)?;
2251 validate_payload_codec(codec)?;
2252 let blob = object
2253 .get("blob")
2254 .and_then(Value::as_str)
2255 .ok_or_else(invalid_payload_envelope)?;
2256 Ok(Some((codec, blob)))
2257}
2258
2259fn invalid_payload_envelope() -> Error {
2260 Error::Codec(
2261 "invalid_payload_envelope: durable payloads must use an object with string codec=\"avro\" and blob fields"
2262 .to_string(),
2263 )
2264}
2265
2266fn validate_workflow_task_commands(commands: &[Value]) -> Result<()> {
2267 for command in commands {
2268 let Some(command) = command.as_object() else {
2269 continue;
2270 };
2271 let Some(command_type) = command.get("type").and_then(Value::as_str) else {
2272 continue;
2273 };
2274 let Some(payload_field) = workflow_command_payload_field(command_type) else {
2275 continue;
2276 };
2277
2278 if let Some(codec) = command.get("payload_codec") {
2279 let codec = codec.as_str().ok_or_else(invalid_payload_envelope)?;
2280 validate_payload_codec(codec)?;
2281 }
2282
2283 let payload = command
2284 .get(payload_field)
2285 .ok_or_else(invalid_payload_envelope)?;
2286 if runtime_payloads::Reference::parse(payload)?.is_none() {
2287 validate_outbound_payload_envelope(payload)?;
2288 }
2289 }
2290 Ok(())
2291}
2292
2293fn workflow_completion_protocol_version(commands: &[Value]) -> &'static str {
2294 if commands.iter().any(|command| {
2295 command.get("type").and_then(Value::as_str) == Some("open_condition_wait")
2296 && command
2297 .get("condition_wait_occurrence_id")
2298 .and_then(Value::as_str)
2299 .is_some_and(|occurrence_id| !occurrence_id.is_empty())
2300 }) {
2301 CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
2302 } else if commands.iter().any(|command| {
2303 command.get("type").and_then(Value::as_str) == Some("upsert_search_attributes")
2304 && command.get("attribute_types").is_some()
2305 }) {
2306 TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION
2307 } else if commands
2308 .iter()
2309 .any(|command| command.get("type").and_then(Value::as_str) == Some("upsert_memo"))
2310 {
2311 MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION
2312 } else if commands
2313 .iter()
2314 .any(|command| command.get("type").and_then(Value::as_str) == Some("open_condition_wait"))
2315 {
2316 CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION
2317 } else if commands.iter().any(|command| {
2318 command.get("type").and_then(Value::as_str) == Some("upsert_search_attributes")
2319 }) {
2320 SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION
2321 } else {
2322 WORKER_PROTOCOL_VERSION
2323 }
2324}
2325
2326fn workflow_completion_protocol_version_with_message_streams(
2327 commands: &[Value],
2328 has_message_stream_metadata: bool,
2329) -> &'static str {
2330 let command_protocol = workflow_completion_protocol_version(commands);
2331 if has_message_stream_metadata && !worker_protocol_supports_message_streams(command_protocol) {
2332 MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION
2333 } else {
2334 command_protocol
2335 }
2336}
2337
2338fn workflow_command_payload_field(command_type: &str) -> Option<&'static str> {
2339 match command_type {
2340 "complete_workflow" | "complete_update" | "record_side_effect" => Some("result"),
2341 "schedule_activity" | "start_child_workflow" | "continue_as_new" => Some("arguments"),
2342 "start_service_operation" => Some("request_payload"),
2343 "upsert_memo" => Some("entries"),
2344 _ => None,
2345 }
2346}
2347
2348fn validate_outbound_payload_envelope(value: &Value) -> Result<()> {
2349 let Some((codec, blob)) = payload_envelope_parts(value)? else {
2350 return Err(untagged_payload_value());
2351 };
2352 validate_payload_codec(codec)?;
2353 decode_avro_value_blob(blob)?;
2354 Ok(())
2355}
2356
2357fn unsupported_payload_codec(codec: &str) -> Error {
2358 Error::Codec(format!(
2359 "unsupported_payload_codec: workflow payload codec {codec:?} is not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
2360 ))
2361}
2362
2363fn untagged_payload_value() -> Error {
2364 Error::Codec(
2365 "unsupported_payload_codec: untagged durable payload values are not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
2366 .to_string(),
2367 )
2368}
2369
2370fn decode_avro_value_blob(blob: &str) -> Result<AvroValue> {
2371 let bytes = BASE64.decode(blob).map_err(|err| {
2372 Error::Codec(format!(
2373 "invalid_payload_framing: expected strict base64 Avro single-object bytes: {err}"
2374 ))
2375 })?;
2376
2377 if serde_json::from_slice::<Value>(&bytes).is_ok() {
2378 return Err(unsupported_payload_codec("json"));
2379 }
2380
2381 if bytes.len() < 10 || bytes[..2] != AVRO_SINGLE_OBJECT_MAGIC {
2382 return Err(Error::Codec(
2383 "invalid_payload_framing: expected Avro single-object magic c301".to_string(),
2384 ));
2385 }
2386
2387 let fingerprint: [u8; 8] = bytes[2..10]
2388 .try_into()
2389 .map_err(|_| Error::Codec("invalid Avro fingerprint length".to_string()))?;
2390 if fingerprint != AVRO_VALUE_SCHEMA_FINGERPRINT {
2391 return Err(Error::Codec(format!(
2392 "unsupported_payload_schema: unknown CRC-64-AVRO fingerprint {}",
2393 fingerprint
2394 .iter()
2395 .map(|byte| format!("{byte:02x}"))
2396 .collect::<String>()
2397 )));
2398 }
2399
2400 let mut datum_reader = StrictAvroDatumReader::new(&bytes[10..]);
2401 let datum = from_avro_datum(avro_value_schema()?, &mut datum_reader, None);
2405 if datum_reader.truncated {
2406 return Err(Error::Codec(
2407 "invalid_payload_framing: truncated Avro Value datum".to_string(),
2408 ));
2409 }
2410 let datum = datum.map_err(|err| {
2411 Error::Codec(format!(
2412 "invalid_payload_framing: malformed Avro Value datum: {err}"
2413 ))
2414 })?;
2415 if datum_reader.remaining() != 0 {
2416 return Err(Error::Codec(format!(
2417 "invalid_payload_framing: {} trailing bytes after Avro Value datum",
2418 datum_reader.remaining()
2419 )));
2420 }
2421 avro_value_from_datum(datum)
2422}
2423
2424struct StrictAvroDatumReader<'a> {
2425 bytes: &'a [u8],
2426 offset: usize,
2427 truncated: bool,
2428}
2429
2430impl<'a> StrictAvroDatumReader<'a> {
2431 fn new(bytes: &'a [u8]) -> Self {
2432 Self {
2433 bytes,
2434 offset: 0,
2435 truncated: false,
2436 }
2437 }
2438
2439 fn remaining(&self) -> usize {
2440 self.bytes.len() - self.offset
2441 }
2442}
2443
2444impl Read for StrictAvroDatumReader<'_> {
2445 fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
2446 let count = buffer.len().min(self.remaining());
2447 buffer[..count].copy_from_slice(&self.bytes[self.offset..self.offset + count]);
2448 self.offset += count;
2449 if count < buffer.len() {
2450 self.truncated = true;
2451 }
2452
2453 Ok(count)
2454 }
2455}
2456
2457fn avro_value_to_datum(value: &AvroValue) -> Result<AvroDatum> {
2458 let branch = match value {
2459 AvroValue::Null => AvroDatum::Union(0, Box::new(AvroDatum::Null)),
2460 AvroValue::Boolean(value) => AvroDatum::Union(
2461 1,
2462 Box::new(AvroDatum::Record(vec![(
2463 "boolean".to_string(),
2464 AvroDatum::Boolean(*value),
2465 )])),
2466 ),
2467 AvroValue::Long(value) => AvroDatum::Union(
2468 2,
2469 Box::new(AvroDatum::Record(vec![(
2470 "long".to_string(),
2471 AvroDatum::Long(*value),
2472 )])),
2473 ),
2474 AvroValue::Double(value) => {
2475 if !value.is_finite() {
2476 return Err(Error::Codec(
2477 "non_finite_float: Avro Value doubles must be finite".to_string(),
2478 ));
2479 }
2480 AvroDatum::Union(
2481 3,
2482 Box::new(AvroDatum::Record(vec![(
2483 "double".to_string(),
2484 AvroDatum::Double(*value),
2485 )])),
2486 )
2487 }
2488 AvroValue::Bytes(value) => AvroDatum::Union(
2489 4,
2490 Box::new(AvroDatum::Record(vec![(
2491 "bytes".to_string(),
2492 AvroDatum::Bytes(value.clone()),
2493 )])),
2494 ),
2495 AvroValue::String(value) => AvroDatum::Union(
2496 5,
2497 Box::new(AvroDatum::Record(vec![(
2498 "string".to_string(),
2499 AvroDatum::String(value.clone()),
2500 )])),
2501 ),
2502 AvroValue::Array(values) => AvroDatum::Union(
2503 6,
2504 Box::new(AvroDatum::Record(vec![(
2505 "items".to_string(),
2506 AvroDatum::Array(
2507 values
2508 .iter()
2509 .map(avro_value_to_datum)
2510 .collect::<Result<Vec<_>>>()?,
2511 ),
2512 )])),
2513 ),
2514 AvroValue::Map(values) => AvroDatum::Union(
2515 7,
2516 Box::new(AvroDatum::Record(vec![(
2517 "entries".to_string(),
2518 AvroDatum::Array(
2519 values
2520 .iter()
2521 .map(|(key, value)| {
2522 Ok(AvroDatum::Record(vec![
2523 ("key".to_string(), AvroDatum::String(key.clone())),
2524 ("value".to_string(), avro_value_to_datum(value)?),
2525 ]))
2526 })
2527 .collect::<Result<Vec<_>>>()?,
2528 ),
2529 )])),
2530 ),
2531 };
2532 Ok(AvroDatum::Record(vec![("value".to_string(), branch)]))
2533}
2534
2535fn avro_value_from_datum(datum: AvroDatum) -> Result<AvroValue> {
2536 let AvroDatum::Record(mut outer) = datum else {
2537 return Err(Error::Codec(
2538 "invalid_payload_framing: datum is not a Value record".to_string(),
2539 ));
2540 };
2541 let (_, branch) = outer
2542 .pop()
2543 .filter(|(name, _)| name == "value")
2544 .ok_or_else(|| Error::Codec("invalid_payload_framing: Value field missing".to_string()))?;
2545 let AvroDatum::Union(_, branch) = branch else {
2546 return Err(Error::Codec(
2547 "invalid_payload_framing: invalid Value union".to_string(),
2548 ));
2549 };
2550 match *branch {
2551 AvroDatum::Null => Ok(AvroValue::Null),
2552 AvroDatum::Record(mut fields) => {
2553 let (name, value) = fields.pop().ok_or_else(|| {
2554 Error::Codec("invalid_payload_framing: empty Value branch".to_string())
2555 })?;
2556 match (name.as_str(), value) {
2557 ("boolean", AvroDatum::Boolean(value)) => Ok(AvroValue::Boolean(value)),
2558 ("long", AvroDatum::Long(value)) => Ok(AvroValue::Long(value)),
2559 ("double", AvroDatum::Double(value)) if value.is_finite() => {
2560 Ok(AvroValue::Double(value))
2561 }
2562 ("bytes", AvroDatum::Bytes(value)) => Ok(AvroValue::Bytes(value)),
2563 ("string", AvroDatum::String(value)) => Ok(AvroValue::String(value)),
2564 ("items", AvroDatum::Array(values)) => values
2565 .into_iter()
2566 .map(avro_value_from_datum)
2567 .collect::<Result<Vec<_>>>()
2568 .map(AvroValue::Array),
2569 ("entries", AvroDatum::Map(values)) => values
2570 .into_iter()
2571 .map(|(key, value)| Ok((key, avro_value_from_datum(value)?)))
2572 .collect::<Result<BTreeMap<_, _>>>()
2573 .map(AvroValue::Map),
2574 _ => Err(Error::Codec(
2575 "invalid_payload_framing: unknown Value branch".to_string(),
2576 )),
2577 }
2578 }
2579 _ => Err(Error::Codec(
2580 "invalid_payload_framing: invalid Value branch".to_string(),
2581 )),
2582 }
2583}
2584
2585fn avro_value_schema() -> Result<&'static Schema> {
2586 match AVRO_VALUE_SCHEMA.get_or_init(|| {
2587 Schema::parse_str(AVRO_VALUE_SCHEMA_JSON)
2588 .map_err(|err| format!("could not parse Avro Value schema: {err}"))
2589 }) {
2590 Ok(schema) => Ok(schema),
2591 Err(message) => Err(Error::Codec(message.clone())),
2592 }
2593}
2594
2595fn avro_value_ordered_map_encoding_schema() -> Result<&'static Schema> {
2596 match AVRO_VALUE_ORDERED_MAP_ENCODING_SCHEMA.get_or_init(|| {
2597 let mut schema: Value = serde_json::from_str(AVRO_VALUE_SCHEMA_JSON)
2602 .map_err(|err| format!("could not read packaged Avro Value schema: {err}"))?;
2603 let entries_schema = schema
2604 .pointer_mut("/fields/0/type/7/fields/0/type")
2605 .ok_or_else(|| "packaged Avro Value map schema is missing".to_string())?;
2606 if *entries_schema != json!({"type": "map", "values": "Value"}) {
2607 return Err("packaged Avro Value map schema changed unexpectedly".to_string());
2608 }
2609 *entries_schema = json!({
2610 "type": "array",
2611 "items": {
2612 "type": "record",
2613 "name": "MapEntry",
2614 "fields": [
2615 {"name": "key", "type": "string"},
2616 {"name": "value", "type": "Value"}
2617 ]
2618 }
2619 });
2620 Schema::parse_str(&schema.to_string())
2621 .map_err(|err| format!("could not parse ordered-map Avro Value schema: {err}"))
2622 }) {
2623 Ok(schema) => Ok(schema),
2624 Err(message) => Err(Error::Codec(message.clone())),
2625 }
2626}
2627
2628#[derive(Clone, Debug)]
2629pub struct Client {
2630 http: reqwest::Client,
2631 base_url: String,
2632 token: Option<String>,
2633 control_token: Option<String>,
2634 worker_token: Option<String>,
2635 namespace: String,
2636 max_external_payload_bytes: usize,
2637 worker_storage_admission: Option<WorkerStorageAdmission>,
2638 runtime_upload_policy: Arc<Mutex<runtime_uploads::PolicyCache>>,
2639}
2640
2641impl Client {
2642 pub fn new(base_url: impl Into<String>) -> Result<Self> {
2643 Self::builder(base_url).build()
2644 }
2645
2646 pub fn builder(base_url: impl Into<String>) -> ClientBuilder {
2647 ClientBuilder {
2648 base_url: base_url.into(),
2649 token: None,
2650 control_token: None,
2651 worker_token: None,
2652 namespace: "default".to_string(),
2653 timeout: Duration::from_secs(60),
2654 max_external_payload_bytes: 64 * 1024 * 1024,
2655 }
2656 }
2657
2658 pub async fn health(&self) -> Result<Value> {
2659 self.request_json(
2660 reqwest::Method::GET,
2661 "/health",
2662 RequestProtocol::ControlPlane,
2663 Option::<&Value>::None,
2664 )
2665 .await
2666 }
2667
2668 pub async fn cluster_info(&self) -> Result<Value> {
2669 self.request_json(
2670 reqwest::Method::GET,
2671 "/cluster/info",
2672 RequestProtocol::ControlPlane,
2673 Option::<&Value>::None,
2674 )
2675 .await
2676 }
2677
2678 pub async fn start_workflow<T: Serialize>(
2679 &self,
2680 workflow_type: &str,
2681 task_queue: &str,
2682 workflow_id: &str,
2683 input: T,
2684 ) -> Result<WorkflowHandle> {
2685 self.start_workflow_with_options(
2686 workflow_type,
2687 task_queue,
2688 workflow_id,
2689 WorkflowStartOptions::default(),
2690 input,
2691 )
2692 .await
2693 }
2694
2695 pub async fn start_workflow_with_options<T: Serialize>(
2698 &self,
2699 workflow_type: &str,
2700 task_queue: &str,
2701 workflow_id: &str,
2702 options: WorkflowStartOptions,
2703 input: T,
2704 ) -> Result<WorkflowHandle> {
2705 options.validate()?;
2706 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2707 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2708 let body = json!({
2709 "workflow_id": workflow_id,
2710 "workflow_type": workflow_type,
2711 "task_queue": task_queue,
2712 "input": input_envelope,
2713 "execution_timeout_seconds": options.execution_timeout_seconds,
2714 "run_timeout_seconds": options.run_timeout_seconds
2715 });
2716
2717 let data: Value = self
2718 .request_json(
2719 reqwest::Method::POST,
2720 "/workflows",
2721 RequestProtocol::ControlPlane,
2722 Some(&body),
2723 )
2724 .await?;
2725
2726 Ok(WorkflowHandle {
2727 client: self.clone(),
2728 workflow_id: data
2729 .get("workflow_id")
2730 .and_then(Value::as_str)
2731 .unwrap_or(workflow_id)
2732 .to_string(),
2733 run_id: data
2734 .get("run_id")
2735 .and_then(Value::as_str)
2736 .map(str::to_string),
2737 workflow_type: data
2738 .get("workflow_type")
2739 .and_then(Value::as_str)
2740 .unwrap_or(workflow_type)
2741 .to_string(),
2742 })
2743 }
2744
2745 pub async fn signal_workflow<T: Serialize>(
2746 &self,
2747 workflow_id: &str,
2748 signal_name: &str,
2749 input: T,
2750 ) -> Result<Value> {
2751 self.signal_workflow_target(workflow_id, None, signal_name, input)
2752 .await
2753 }
2754
2755 pub async fn append_message_stream<T: Serialize>(
2757 &self,
2758 workflow_id: &str,
2759 stream_name: &str,
2760 message_id: &str,
2761 input: T,
2762 ) -> Result<Value> {
2763 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2764 let body = json!({
2765 "message_id": message_id,
2766 "input": encode_typed_envelope(&input, DEFAULT_CODEC)?
2767 });
2768 self.request_json(
2769 reqwest::Method::POST,
2770 &format!("/workflows/{workflow_id}/message-streams/{stream_name}/messages"),
2771 RequestProtocol::ControlPlane,
2772 Some(&body),
2773 )
2774 .await
2775 }
2776
2777 pub async fn signal_workflow_run<T: Serialize>(
2779 &self,
2780 workflow_id: &str,
2781 run_id: &str,
2782 signal_name: &str,
2783 input: T,
2784 ) -> Result<Value> {
2785 self.signal_workflow_target(workflow_id, Some(run_id), signal_name, input)
2786 .await
2787 }
2788
2789 async fn signal_workflow_target<T: Serialize>(
2790 &self,
2791 workflow_id: &str,
2792 run_id: Option<&str>,
2793 signal_name: &str,
2794 input: T,
2795 ) -> Result<Value> {
2796 validate_user_signal_name(signal_name)?;
2797 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2798 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2799 let body = json!({
2800 "input": input_envelope
2801 });
2802 let path = match run_id {
2803 Some(run_id) => {
2804 format!("/workflows/{workflow_id}/runs/{run_id}/signal/{signal_name}")
2805 }
2806 None => format!("/workflows/{workflow_id}/signal/{signal_name}"),
2807 };
2808 self.request_json(
2809 reqwest::Method::POST,
2810 &path,
2811 RequestProtocol::ControlPlane,
2812 Some(&body),
2813 )
2814 .await
2815 }
2816
2817 pub async fn cancel_workflow(
2823 &self,
2824 workflow_id: &str,
2825 options: WorkflowCommandOptions,
2826 ) -> Result<WorkflowCommandResult> {
2827 self.workflow_command(workflow_id, None, WorkflowCommandKind::Cancel, options)
2828 .await
2829 }
2830
2831 pub async fn cancel_workflow_run(
2833 &self,
2834 workflow_id: &str,
2835 run_id: &str,
2836 options: WorkflowCommandOptions,
2837 ) -> Result<WorkflowCommandResult> {
2838 self.workflow_command(
2839 workflow_id,
2840 Some(run_id),
2841 WorkflowCommandKind::Cancel,
2842 options,
2843 )
2844 .await
2845 }
2846
2847 pub async fn terminate_workflow(
2849 &self,
2850 workflow_id: &str,
2851 options: WorkflowCommandOptions,
2852 ) -> Result<WorkflowCommandResult> {
2853 self.workflow_command(workflow_id, None, WorkflowCommandKind::Terminate, options)
2854 .await
2855 }
2856
2857 pub async fn terminate_workflow_run(
2859 &self,
2860 workflow_id: &str,
2861 run_id: &str,
2862 options: WorkflowCommandOptions,
2863 ) -> Result<WorkflowCommandResult> {
2864 self.workflow_command(
2865 workflow_id,
2866 Some(run_id),
2867 WorkflowCommandKind::Terminate,
2868 options,
2869 )
2870 .await
2871 }
2872
2873 pub async fn redrive_workflow_run(
2875 &self,
2876 workflow_id: &str,
2877 failed_run_id: &str,
2878 request_id: Option<&str>,
2879 ) -> Result<WorkflowRedriveResult> {
2880 let body = request_id
2881 .map(|id| json!({"request_id": id}))
2882 .unwrap_or_else(|| json!({}));
2883 let data: Value = self
2884 .request_json(
2885 reqwest::Method::POST,
2886 &format!("/workflows/{workflow_id}/runs/{failed_run_id}/redrive"),
2887 RequestProtocol::ControlPlane,
2888 Some(&body),
2889 )
2890 .await?;
2891 if data.get("command_status").and_then(Value::as_str) != Some("accepted")
2892 || data.get("outcome").and_then(Value::as_str) != Some("redriven")
2893 {
2894 return Err(Error::Codec(
2895 "redrive response was not accepted".to_string(),
2896 ));
2897 }
2898 if data.get("workflow_id").and_then(Value::as_str) != Some(workflow_id)
2899 || data.get("continued_from_run_id").and_then(Value::as_str) != Some(failed_run_id)
2900 {
2901 return Err(Error::Codec(
2902 "redrive response does not match the requested source run".to_string(),
2903 ));
2904 }
2905 let run_id = data
2906 .get("run_id")
2907 .and_then(Value::as_str)
2908 .filter(|id| !id.is_empty() && *id != failed_run_id)
2909 .ok_or_else(|| {
2910 Error::Codec("redrive response is missing a distinct successor run_id".to_string())
2911 })?;
2912 let resume_step_sequence = data
2913 .get("resume_step_sequence")
2914 .and_then(Value::as_u64)
2915 .filter(|sequence| *sequence > 0)
2916 .ok_or_else(|| {
2917 Error::Codec("redrive response is missing resume_step_sequence".to_string())
2918 })?;
2919 Ok(WorkflowRedriveResult {
2920 workflow_id: workflow_id.to_string(),
2921 source_run_id: failed_run_id.to_string(),
2922 run_id: run_id.to_string(),
2923 resume_step_sequence,
2924 raw: data,
2925 })
2926 }
2927
2928 async fn workflow_command(
2929 &self,
2930 workflow_id: &str,
2931 run_id: Option<&str>,
2932 command: WorkflowCommandKind,
2933 options: WorkflowCommandOptions,
2934 ) -> Result<WorkflowCommandResult> {
2935 let path = match run_id {
2936 Some(run_id) => format!(
2937 "/workflows/{workflow_id}/runs/{run_id}/{}",
2938 command.as_str()
2939 ),
2940 None => format!("/workflows/{workflow_id}/{}", command.as_str()),
2941 };
2942 let data = match self
2943 .request_json(
2944 reqwest::Method::POST,
2945 &path,
2946 RequestProtocol::ControlPlane,
2947 Some(&options),
2948 )
2949 .await
2950 {
2951 Ok(data) => data,
2952 Err(Error::Http { status, body }) => {
2953 return Err(Error::WorkflowCommandRejected(workflow_command_rejection(
2954 command,
2955 status,
2956 body,
2957 workflow_id,
2958 run_id,
2959 )));
2960 }
2961 Err(error) => return Err(error),
2962 };
2963
2964 Ok(workflow_command_result(command, data, workflow_id, run_id))
2965 }
2966
2967 pub async fn query_workflow<T: Serialize>(
2973 &self,
2974 workflow_id: &str,
2975 query_name: &str,
2976 input: T,
2977 ) -> Result<Value> {
2978 self.query_workflow_target(workflow_id, None, query_name, input)
2979 .await
2980 }
2981
2982 pub async fn query_workflow_run<T: Serialize>(
2984 &self,
2985 workflow_id: &str,
2986 run_id: &str,
2987 query_name: &str,
2988 input: T,
2989 ) -> Result<Value> {
2990 self.query_workflow_target(workflow_id, Some(run_id), query_name, input)
2991 .await
2992 }
2993
2994 pub async fn query_workflow_avro_value<T: Serialize>(
2996 &self,
2997 workflow_id: &str,
2998 query_name: &str,
2999 input: T,
3000 ) -> Result<AvroValue> {
3001 self.query_workflow_avro_value_target(workflow_id, None, query_name, input)
3002 .await
3003 }
3004
3005 pub async fn query_workflow_run_avro_value<T: Serialize>(
3007 &self,
3008 workflow_id: &str,
3009 run_id: &str,
3010 query_name: &str,
3011 input: T,
3012 ) -> Result<AvroValue> {
3013 self.query_workflow_avro_value_target(workflow_id, Some(run_id), query_name, input)
3014 .await
3015 }
3016
3017 async fn query_workflow_avro_value_target<T: Serialize>(
3018 &self,
3019 workflow_id: &str,
3020 run_id: Option<&str>,
3021 query_name: &str,
3022 input: T,
3023 ) -> Result<AvroValue> {
3024 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
3025 let body = json!({"input": encode_typed_envelope(&input, DEFAULT_CODEC)?});
3026 let path = match run_id {
3027 Some(run_id) => {
3028 format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
3029 }
3030 None => format!("/workflows/{workflow_id}/query/{query_name}"),
3031 };
3032 let response: Value = match self
3033 .request_json(
3034 reqwest::Method::POST,
3035 &path,
3036 RequestProtocol::ControlPlane,
3037 Some(&body),
3038 )
3039 .await
3040 {
3041 Ok(response) => response,
3042 Err(Error::Http { status, body }) => {
3043 return Err(Error::QueryFailed(query_failure(status, body)));
3044 }
3045 Err(error) => return Err(error),
3046 };
3047
3048 let envelope = response
3049 .get("result_envelope")
3050 .filter(|envelope| !envelope.is_null())
3051 .ok_or_else(|| {
3052 Error::Codec(
3053 "missing_payload_envelope: typed query result requires result_envelope"
3054 .to_string(),
3055 )
3056 })?;
3057 decode_wire_avro_value(envelope, DEFAULT_CODEC)
3058 }
3059
3060 async fn query_workflow_target<T: Serialize>(
3061 &self,
3062 workflow_id: &str,
3063 run_id: Option<&str>,
3064 query_name: &str,
3065 input: T,
3066 ) -> Result<Value> {
3067 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
3068 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
3069 let body = json!({
3070 "input": input_envelope
3071 });
3072 let path = match run_id {
3073 Some(run_id) => {
3074 format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
3075 }
3076 None => format!("/workflows/{workflow_id}/query/{query_name}"),
3077 };
3078 let response: Value = match self
3079 .request_json(
3080 reqwest::Method::POST,
3081 &path,
3082 RequestProtocol::ControlPlane,
3083 Some(&body),
3084 )
3085 .await
3086 {
3087 Ok(response) => response,
3088 Err(Error::Http { status, body }) => {
3089 return Err(Error::QueryFailed(query_failure(status, body)));
3090 }
3091 Err(error) => return Err(error),
3092 };
3093
3094 if let Some(envelope) = response
3095 .get("result_envelope")
3096 .filter(|envelope| !envelope.is_null())
3097 {
3098 return decode_wire_value(envelope, DEFAULT_CODEC);
3099 }
3100
3101 Ok(response.get("result").cloned().unwrap_or(Value::Null))
3102 }
3103
3104 pub async fn update_workflow<T: Serialize>(
3106 &self,
3107 workflow_id: &str,
3108 update_name: &str,
3109 input: T,
3110 request_id: Option<&str>,
3111 ) -> Result<Value> {
3112 let response = self
3113 .update_workflow_response(workflow_id, update_name, input, request_id)
3114 .await?;
3115 if let Some(envelope) = response
3116 .get("result_envelope")
3117 .filter(|envelope| !envelope.is_null())
3118 {
3119 return decode_wire_value(envelope, DEFAULT_CODEC);
3120 }
3121 Ok(response.get("result").cloned().unwrap_or(response))
3122 }
3123
3124 pub async fn update_workflow_avro_value<T: Serialize>(
3126 &self,
3127 workflow_id: &str,
3128 update_name: &str,
3129 input: T,
3130 request_id: Option<&str>,
3131 ) -> Result<AvroValue> {
3132 let response = self
3133 .update_workflow_response(workflow_id, update_name, input, request_id)
3134 .await?;
3135 let envelope = response
3136 .get("result_envelope")
3137 .filter(|envelope| !envelope.is_null())
3138 .ok_or_else(|| {
3139 Error::Codec(
3140 "missing_payload_envelope: typed update result requires result_envelope"
3141 .to_string(),
3142 )
3143 })?;
3144 decode_wire_avro_value(envelope, DEFAULT_CODEC)
3145 }
3146
3147 async fn update_workflow_response<T: Serialize>(
3148 &self,
3149 workflow_id: &str,
3150 update_name: &str,
3151 input: T,
3152 request_id: Option<&str>,
3153 ) -> Result<Value> {
3154 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
3155 let mut body = json!({
3156 "input": encode_typed_envelope(&input, DEFAULT_CODEC)?,
3157 "wait_for": "completed",
3158 });
3159 if let Some(request_id) = request_id {
3160 body["request_id"] = json!(request_id);
3161 }
3162 self.request_json(
3163 reqwest::Method::POST,
3164 &format!("/workflows/{workflow_id}/update/{update_name}"),
3165 RequestProtocol::ControlPlane,
3166 Some(&body),
3167 )
3168 .await
3169 }
3170
3171 pub async fn describe_workflow(&self, workflow_id: &str) -> Result<WorkflowDescription> {
3172 let path = format!("/workflows/{workflow_id}");
3173 let mut data: WorkflowDescription = self
3174 .request_json(
3175 reqwest::Method::GET,
3176 &path,
3177 RequestProtocol::ControlPlane,
3178 Option::<&Value>::None,
3179 )
3180 .await?;
3181 data.decode_payloads()?;
3182 Ok(data)
3183 }
3184
3185 pub async fn describe_workflow_run(
3187 &self,
3188 workflow_id: &str,
3189 run_id: &str,
3190 ) -> Result<WorkflowDescription> {
3191 let path = format!("/workflows/{workflow_id}/runs/{run_id}");
3192 let mut data: WorkflowDescription = self
3193 .request_json(
3194 reqwest::Method::GET,
3195 &path,
3196 RequestProtocol::ControlPlane,
3197 Option::<&Value>::None,
3198 )
3199 .await?;
3200 data.decode_payloads()?;
3201 Ok(data)
3202 }
3203
3204 fn workflow_stream_path(workflow_id: &str, run_id: &str, stream_name: Option<&str>) -> String {
3205 let mut path = format!(
3206 "/workflows/{}/runs/{}/streams",
3207 percent_encode_path_segment(workflow_id),
3208 percent_encode_path_segment(run_id),
3209 );
3210 if let Some(stream_name) = stream_name {
3211 path.push('/');
3212 path.push_str(&percent_encode_path_segment(stream_name));
3213 }
3214 path
3215 }
3216
3217 pub async fn list_workflow_streams(
3219 &self,
3220 workflow_id: &str,
3221 run_id: &str,
3222 ) -> Result<Vec<WorkflowStreamDescription>> {
3223 let response: WorkflowStreamListResponse = self
3224 .request_json(
3225 reqwest::Method::GET,
3226 &Self::workflow_stream_path(workflow_id, run_id, None),
3227 RequestProtocol::ControlPlane,
3228 Option::<&Value>::None,
3229 )
3230 .await?;
3231 Ok(response.streams)
3232 }
3233
3234 pub async fn describe_workflow_stream(
3236 &self,
3237 workflow_id: &str,
3238 run_id: &str,
3239 stream_name: &str,
3240 ) -> Result<WorkflowStreamDescription> {
3241 let response: WorkflowStreamDescriptionResponse = self
3242 .request_json(
3243 reqwest::Method::GET,
3244 &Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3245 RequestProtocol::ControlPlane,
3246 Option::<&Value>::None,
3247 )
3248 .await?;
3249 Ok(response.stream)
3250 }
3251
3252 pub async fn subscribe_workflow_stream(
3259 &self,
3260 workflow_id: &str,
3261 run_id: &str,
3262 stream_name: &str,
3263 from_offset: u64,
3264 max_items: usize,
3265 wait: Duration,
3266 ) -> Result<WorkflowStreamPage> {
3267 let max_items = max_items.clamp(1, 500);
3268 let wait_seconds = wait.as_secs().min(MAX_LONG_POLL_TIMEOUT_SECONDS);
3269 let path = format!(
3270 "{}/items?from={from_offset}&max_items={max_items}&wait_seconds={wait_seconds}",
3271 Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3272 );
3273 let response: WorkflowStreamPageResponse = self
3274 .request_json_with_timeout(
3275 reqwest::Method::GET,
3276 &path,
3277 RequestProtocol::ControlPlane,
3278 Option::<&Value>::None,
3279 Duration::from_secs(wait_seconds.saturating_add(5).max(5)),
3280 )
3281 .await?;
3282
3283 let items = response
3284 .items
3285 .into_iter()
3286 .map(|raw| {
3287 let offset = raw.get("offset").and_then(Value::as_u64).unwrap_or(0);
3288 let envelope = raw.get("payload").cloned();
3289 let payload = envelope
3290 .as_ref()
3291 .filter(|value| value.get("blob").is_some())
3292 .map(|value| decode_wire_avro_value(value, DEFAULT_CODEC))
3293 .transpose()?
3294 .map(AvroValue::into_json)
3295 .transpose()?;
3296 Ok(WorkflowStreamItem {
3297 offset,
3298 payload,
3299 payload_envelope: envelope,
3300 payload_reference: raw
3301 .get("payload_reference")
3302 .and_then(Value::as_str)
3303 .map(str::to_string),
3304 payload_codec: raw
3305 .get("payload_codec")
3306 .and_then(Value::as_str)
3307 .map(str::to_string),
3308 idempotency_key: raw
3309 .get("idempotency_key")
3310 .and_then(Value::as_str)
3311 .map(str::to_string),
3312 item_type: raw
3313 .get("item_type")
3314 .and_then(Value::as_str)
3315 .map(str::to_string),
3316 content_type: raw
3317 .get("content_type")
3318 .and_then(Value::as_str)
3319 .map(str::to_string),
3320 origin: raw
3321 .get("origin")
3322 .and_then(Value::as_str)
3323 .map(str::to_string),
3324 origin_reference: raw
3325 .get("origin_reference")
3326 .and_then(Value::as_str)
3327 .map(str::to_string),
3328 emitted_at: raw
3329 .get("emitted_at")
3330 .and_then(Value::as_str)
3331 .map(str::to_string),
3332 raw,
3333 })
3334 })
3335 .collect::<Result<Vec<_>>>()?;
3336 Ok(WorkflowStreamPage {
3337 stream: response.stream,
3338 items,
3339 next_offset: response.next_offset,
3340 terminal: response.terminal,
3341 })
3342 }
3343
3344 pub async fn append_workflow_stream(
3346 &self,
3347 workflow_id: &str,
3348 run_id: &str,
3349 stream_name: &str,
3350 items: &[WorkflowStreamAppendItem],
3351 max_pending_items: Option<u64>,
3352 ) -> Result<WorkflowStreamAppendResult> {
3353 if items.is_empty() {
3354 return Err(Error::Codec(
3355 "workflow_stream_items_empty: append requires at least one item".to_string(),
3356 ));
3357 }
3358 let mut body = json!({
3359 "items": items
3360 .iter()
3361 .map(|item| item.wire_value(None))
3362 .collect::<Vec<_>>(),
3363 });
3364 if let Some(max_pending_items) = max_pending_items {
3365 if max_pending_items == 0 {
3366 return Err(Error::Codec(
3367 "workflow_stream_pending_limit_invalid: max_pending_items must be positive"
3368 .to_string(),
3369 ));
3370 }
3371 body["max_pending_items"] = json!(max_pending_items);
3372 }
3373 let response: WorkflowStreamAppendResponse = self
3374 .request_json(
3375 reqwest::Method::POST,
3376 &format!(
3377 "{}/items",
3378 Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3379 ),
3380 RequestProtocol::ControlPlane,
3381 Some(&body),
3382 )
3383 .await?;
3384 Ok(WorkflowStreamAppendResult {
3385 stream: response.stream,
3386 accepted_offsets: response.accepted_offsets,
3387 accepted: response.accepted,
3388 deduped: response.deduped,
3389 })
3390 }
3391
3392 pub async fn close_workflow_stream(
3394 &self,
3395 workflow_id: &str,
3396 run_id: &str,
3397 stream_name: &str,
3398 error_reason: Option<&str>,
3399 retention_seconds: Option<u64>,
3400 ) -> Result<WorkflowStreamDescription> {
3401 let mut body = json!({});
3402 if let Some(error_reason) = error_reason {
3403 body["error_reason"] = json!(error_reason);
3404 }
3405 if let Some(retention_seconds) = retention_seconds {
3406 if retention_seconds == 0 {
3407 return Err(Error::Codec(
3408 "workflow_stream_retention_invalid: retention_seconds must be positive"
3409 .to_string(),
3410 ));
3411 }
3412 body["retention_seconds"] = json!(retention_seconds);
3413 }
3414 let response: WorkflowStreamDescriptionResponse = self
3415 .request_json(
3416 reqwest::Method::POST,
3417 &format!(
3418 "{}/close",
3419 Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3420 ),
3421 RequestProtocol::ControlPlane,
3422 Some(&body),
3423 )
3424 .await?;
3425 Ok(response.stream)
3426 }
3427
3428 pub async fn register_worker(
3429 &self,
3430 worker_id: &str,
3431 task_queue: &str,
3432 supported_workflow_types: Vec<String>,
3433 supported_activity_types: Vec<String>,
3434 max_concurrent_workflow_tasks: usize,
3435 max_concurrent_activity_tasks: usize,
3436 ) -> Result<RegisterWorkerResponse> {
3437 self.register_worker_with_capabilities(
3438 worker_id,
3439 task_queue,
3440 supported_workflow_types,
3441 supported_activity_types,
3442 max_concurrent_workflow_tasks,
3443 max_concurrent_activity_tasks,
3444 Vec::new(),
3445 )
3446 .await
3447 }
3448
3449 pub async fn register_worker_with_capabilities(
3451 &self,
3452 worker_id: &str,
3453 task_queue: &str,
3454 supported_workflow_types: Vec<String>,
3455 supported_activity_types: Vec<String>,
3456 max_concurrent_workflow_tasks: usize,
3457 max_concurrent_activity_tasks: usize,
3458 capabilities: Vec<String>,
3459 ) -> Result<RegisterWorkerResponse> {
3460 self.register_worker_with_command_contracts(
3461 worker_id,
3462 task_queue,
3463 supported_workflow_types,
3464 supported_activity_types,
3465 max_concurrent_workflow_tasks,
3466 max_concurrent_activity_tasks,
3467 capabilities,
3468 Value::Object(serde_json::Map::new()),
3469 )
3470 .await
3471 }
3472
3473 #[allow(clippy::too_many_arguments)]
3480 pub async fn register_worker_with_command_contracts(
3481 &self,
3482 worker_id: &str,
3483 task_queue: &str,
3484 supported_workflow_types: Vec<String>,
3485 supported_activity_types: Vec<String>,
3486 max_concurrent_workflow_tasks: usize,
3487 max_concurrent_activity_tasks: usize,
3488 capabilities: Vec<String>,
3489 workflow_command_contracts: Value,
3490 ) -> Result<RegisterWorkerResponse> {
3491 self.register_worker_with_definition_fingerprints(
3492 worker_id,
3493 task_queue,
3494 supported_workflow_types,
3495 supported_activity_types,
3496 max_concurrent_workflow_tasks,
3497 max_concurrent_activity_tasks,
3498 capabilities,
3499 workflow_command_contracts,
3500 None,
3501 )
3502 .await
3503 }
3504
3505 #[allow(clippy::too_many_arguments)]
3506 async fn register_worker_with_definition_fingerprints(
3507 &self,
3508 worker_id: &str,
3509 task_queue: &str,
3510 supported_workflow_types: Vec<String>,
3511 supported_activity_types: Vec<String>,
3512 max_concurrent_workflow_tasks: usize,
3513 max_concurrent_activity_tasks: usize,
3514 capabilities: Vec<String>,
3515 workflow_command_contracts: Value,
3516 workflow_definition_fingerprints: Option<BTreeMap<String, String>>,
3517 ) -> Result<RegisterWorkerResponse> {
3518 if let Some(contracts) = workflow_command_contracts.as_object() {
3519 for (workflow_type, contract) in contracts {
3520 let Some(update_validators) = contract.get("update_validators") else {
3521 continue;
3522 };
3523 if !update_validators
3524 .as_array()
3525 .is_some_and(|validators| validators.is_empty())
3526 {
3527 return Err(Error::UnsupportedUpdateValidators {
3528 workflow_type: workflow_type.clone(),
3529 });
3530 }
3531 }
3532 }
3533
3534 let mut body = json!({
3535 "worker_id": worker_id,
3536 "task_queue": task_queue,
3537 "runtime": "rust",
3538 "sdk_version": SDK_VERSION,
3539 "supported_workflow_types": supported_workflow_types,
3540 "supported_activity_types": supported_activity_types,
3541 "capabilities": capabilities,
3542 "capability_manifest": portable_worker_affinity_capability_manifest(),
3543 "max_concurrent_workflow_tasks": max_concurrent_workflow_tasks,
3544 "max_concurrent_activity_tasks": max_concurrent_activity_tasks
3545 });
3546 if workflow_command_contracts
3547 .as_object()
3548 .is_some_and(|contracts| !contracts.is_empty())
3549 {
3550 body["workflow_command_contracts"] = workflow_command_contracts;
3551 }
3552 if let Some(fingerprints) = workflow_definition_fingerprints {
3553 body["workflow_definition_fingerprints"] = json!(fingerprints);
3554 }
3555
3556 self.request_json(
3557 reqwest::Method::POST,
3558 "/worker/register",
3559 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3560 Some(&body),
3561 )
3562 .await
3563 }
3564
3565 pub async fn deregister_worker_registration(
3571 &self,
3572 worker_id: &str,
3573 ) -> Result<WorkerDeregistrationEnvelope> {
3574 let path = format!(
3575 "/worker/registrations/{}",
3576 percent_encode_path_segment(worker_id)
3577 );
3578 self.request_json(
3579 reqwest::Method::DELETE,
3580 &path,
3581 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3582 Option::<&Value>::None,
3583 )
3584 .await
3585 }
3586
3587 pub async fn poll_query_task(
3589 &self,
3590 worker_id: &str,
3591 task_queue: &str,
3592 timeout: Duration,
3593 ) -> Result<Option<QueryTask>> {
3594 Ok(self
3595 .poll_query_task_response(worker_id, task_queue, timeout)
3596 .await?
3597 .task)
3598 }
3599
3600 pub async fn poll_query_task_response(
3602 &self,
3603 worker_id: &str,
3604 task_queue: &str,
3605 timeout: Duration,
3606 ) -> Result<PollQueryTaskResponse> {
3607 let poll_request_id = unique_request_id("rust-query-poll");
3608 self.poll_query_task_response_with_request_id(
3609 worker_id,
3610 task_queue,
3611 timeout,
3612 &poll_request_id,
3613 1,
3614 )
3615 .await
3616 }
3617
3618 async fn poll_query_task_response_with_request_id(
3619 &self,
3620 worker_id: &str,
3621 task_queue: &str,
3622 timeout: Duration,
3623 poll_request_id: &str,
3624 transport_retries: usize,
3625 ) -> Result<PollQueryTaskResponse> {
3626 let timeout_seconds = long_poll_timeout_seconds(timeout);
3627 let body = json!({
3628 "worker_id": worker_id,
3629 "task_queue": task_queue,
3630 "poll_request_id": poll_request_id,
3631 "timeout_seconds": timeout_seconds,
3632 });
3633 self.poll_request_json(
3634 "/worker/query-tasks/poll",
3635 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3636 &body,
3637 timeout + Duration::from_secs(5),
3638 transport_retries,
3639 )
3640 .await
3641 }
3642
3643 pub async fn complete_query_task<T: Serialize>(
3645 &self,
3646 query_task_id: &str,
3647 lease_owner: &str,
3648 query_task_attempt: u64,
3649 result: T,
3650 codec: &str,
3651 ) -> Result<Value> {
3652 let typed_result = AvroValue::from_serialize(&result)?;
3653 let result_envelope = encode_typed_envelope(&typed_result, codec)?;
3654 self.complete_query_task_with_envelope(
3655 query_task_id,
3656 lease_owner,
3657 query_task_attempt,
3658 typed_result.into_json()?,
3659 result_envelope,
3660 )
3661 .await
3662 }
3663
3664 async fn complete_query_task_with_envelope(
3665 &self,
3666 query_task_id: &str,
3667 lease_owner: &str,
3668 query_task_attempt: u64,
3669 result: Value,
3670 result_envelope: Value,
3671 ) -> Result<Value> {
3672 let body = json!({
3673 "lease_owner": lease_owner,
3674 "query_task_attempt": query_task_attempt,
3675 "result": result,
3676 "result_envelope": result_envelope,
3677 });
3678 let path = format!("/worker/query-tasks/{query_task_id}/complete");
3679 let response = self
3680 .request_json(
3681 reqwest::Method::POST,
3682 &path,
3683 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3684 Some(&body),
3685 )
3686 .await;
3687 query_task_response(response)
3688 }
3689
3690 pub async fn fail_query_task(
3692 &self,
3693 query_task_id: &str,
3694 lease_owner: &str,
3695 query_task_attempt: u64,
3696 message: impl Into<String>,
3697 reason: impl Into<String>,
3698 failure_type: impl Into<String>,
3699 ) -> Result<Value> {
3700 let body = json!({
3701 "lease_owner": lease_owner,
3702 "query_task_attempt": query_task_attempt,
3703 "failure": {
3704 "message": message.into(),
3705 "reason": reason.into(),
3706 "type": failure_type.into(),
3707 }
3708 });
3709 let path = format!("/worker/query-tasks/{query_task_id}/fail");
3710 let response = self
3711 .request_json(
3712 reqwest::Method::POST,
3713 &path,
3714 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3715 Some(&body),
3716 )
3717 .await;
3718 query_task_response(response)
3719 }
3720
3721 pub async fn heartbeat_worker(
3722 &self,
3723 worker_id: &str,
3724 workflow_available: usize,
3725 activity_available: usize,
3726 ) -> Result<Value> {
3727 let body = json!({
3728 "worker_id": worker_id,
3729 "task_slots": {
3730 "workflow_available": workflow_available,
3731 "activity_available": activity_available
3732 },
3733 "process_metrics": {
3734 "process_id": std::process::id(),
3735 "process_uptime_seconds": 0
3736 }
3737 });
3738
3739 self.request_json(
3740 reqwest::Method::POST,
3741 "/worker/heartbeat",
3742 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3743 Some(&body),
3744 )
3745 .await
3746 }
3747
3748 pub async fn poll_workflow_task(
3749 &self,
3750 worker_id: &str,
3751 task_queue: &str,
3752 timeout: Duration,
3753 ) -> Result<Option<WorkflowTask>> {
3754 Ok(self
3755 .poll_workflow_task_response(worker_id, task_queue, timeout)
3756 .await?
3757 .task)
3758 }
3759
3760 pub async fn poll_workflow_task_response(
3761 &self,
3762 worker_id: &str,
3763 task_queue: &str,
3764 timeout: Duration,
3765 ) -> Result<PollWorkflowTaskResponse> {
3766 let poll_request_id = unique_request_id("rust-workflow-poll");
3767 self.poll_workflow_task_response_with_request_id(
3768 worker_id,
3769 task_queue,
3770 timeout,
3771 &poll_request_id,
3772 1,
3773 )
3774 .await
3775 }
3776
3777 async fn poll_workflow_task_response_with_request_id(
3778 &self,
3779 worker_id: &str,
3780 task_queue: &str,
3781 timeout: Duration,
3782 poll_request_id: &str,
3783 transport_retries: usize,
3784 ) -> Result<PollWorkflowTaskResponse> {
3785 let body = json!({
3786 "worker_id": worker_id,
3787 "task_queue": task_queue,
3788 "poll_request_id": poll_request_id,
3789 "timeout_seconds": long_poll_timeout_seconds(timeout),
3790 });
3791 let mut data: PollWorkflowTaskResponse = self
3792 .poll_request_json(
3793 "/worker/workflow-tasks/poll",
3794 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3795 &body,
3796 timeout + Duration::from_secs(5),
3797 transport_retries,
3798 )
3799 .await?;
3800
3801 if let Some(task) = data.task.as_mut() {
3802 self.fetch_remaining_workflow_history(worker_id, task)
3803 .await?;
3804 }
3805
3806 Ok(data)
3807 }
3808
3809 async fn fetch_remaining_workflow_history(
3810 &self,
3811 worker_id: &str,
3812 task: &mut WorkflowTask,
3813 ) -> Result<()> {
3814 let mut next_token = task.next_history_page_token.clone();
3815
3816 while let Some(token) = next_token.take().filter(|token| !token.is_empty()) {
3817 let lease_owner = task
3818 .lease_owner
3819 .clone()
3820 .unwrap_or_else(|| worker_id.to_string());
3821 let page = self
3822 .workflow_task_history_page(
3823 &task.task_id,
3824 &lease_owner,
3825 task.workflow_task_attempt,
3826 &token,
3827 )
3828 .await?;
3829
3830 task.append_history_page(page);
3831
3832 if task.next_history_page_token.as_deref() == Some(token.as_str()) {
3833 return Err(Error::Codec(
3834 "workflow history pagination returned the same page token".to_string(),
3835 ));
3836 }
3837
3838 next_token = task.next_history_page_token.clone();
3839 }
3840
3841 Ok(())
3842 }
3843
3844 async fn workflow_task_history_page(
3845 &self,
3846 task_id: &str,
3847 lease_owner: &str,
3848 workflow_task_attempt: u64,
3849 next_history_page_token: &str,
3850 ) -> Result<WorkflowTaskHistoryPage> {
3851 let body = json!({
3852 "lease_owner": lease_owner,
3853 "workflow_task_attempt": workflow_task_attempt,
3854 "next_history_page_token": next_history_page_token
3855 });
3856 let path = format!("/worker/workflow-tasks/{task_id}/history");
3857
3858 self.request_json(
3859 reqwest::Method::POST,
3860 &path,
3861 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3862 Some(&body),
3863 )
3864 .await
3865 }
3866
3867 pub async fn complete_workflow_task(
3868 &self,
3869 task_id: &str,
3870 lease_owner: &str,
3871 workflow_task_attempt: u64,
3872 commands: Vec<Value>,
3873 ) -> Result<Value> {
3874 self.complete_workflow_task_with_message_streams(
3875 task_id,
3876 lease_owner,
3877 workflow_task_attempt,
3878 commands,
3879 Vec::new(),
3880 Vec::new(),
3881 )
3882 .await
3883 }
3884
3885 async fn complete_workflow_task_with_message_streams(
3886 &self,
3887 task_id: &str,
3888 lease_owner: &str,
3889 workflow_task_attempt: u64,
3890 commands: Vec<Value>,
3891 message_stream_cursors: Vec<Value>,
3892 message_stream_waits: Vec<Value>,
3893 ) -> Result<Value> {
3894 validate_workflow_task_commands(&commands)?;
3895 let has_message_stream_metadata =
3896 !message_stream_cursors.is_empty() || !message_stream_waits.is_empty();
3897 if has_message_stream_metadata
3898 && !worker_protocol_supports_message_streams(WORKER_PROTOCOL_VERSION)
3899 {
3900 return Err(Error::Codec(
3901 "message_streams_unavailable: message stream completion metadata requires worker protocol 1.15 or newer"
3902 .to_string(),
3903 ));
3904 }
3905 let protocol_version = workflow_completion_protocol_version_with_message_streams(
3906 &commands,
3907 has_message_stream_metadata,
3908 );
3909 let mut body = json!({
3910 "lease_owner": lease_owner,
3911 "workflow_task_attempt": workflow_task_attempt,
3912 "commands": commands
3913 });
3914 if !message_stream_cursors.is_empty() {
3915 body["message_stream_cursors"] = Value::Array(message_stream_cursors);
3916 }
3917 if !message_stream_waits.is_empty() {
3918 body["message_stream_waits"] = Value::Array(message_stream_waits);
3919 }
3920 let path = format!("/worker/workflow-tasks/{task_id}/complete");
3921 self.request_json(
3922 reqwest::Method::POST,
3923 &path,
3924 RequestProtocol::Worker(protocol_version),
3925 Some(&body),
3926 )
3927 .await
3928 }
3929
3930 pub async fn fail_workflow_task(
3931 &self,
3932 task_id: &str,
3933 lease_owner: &str,
3934 workflow_task_attempt: u64,
3935 message: impl Into<String>,
3936 ) -> Result<Value> {
3937 self.fail_workflow_task_with_type(
3938 task_id,
3939 lease_owner,
3940 workflow_task_attempt,
3941 message,
3942 "RustWorkflowTaskFailure",
3943 )
3944 .await
3945 }
3946
3947 async fn fail_workflow_task_with_type(
3948 &self,
3949 task_id: &str,
3950 lease_owner: &str,
3951 workflow_task_attempt: u64,
3952 message: impl Into<String>,
3953 failure_type: &str,
3954 ) -> Result<Value> {
3955 let body = json!({
3956 "lease_owner": lease_owner,
3957 "workflow_task_attempt": workflow_task_attempt,
3958 "failure": {
3959 "message": message.into(),
3960 "type": failure_type
3961 }
3962 });
3963 let path = format!("/worker/workflow-tasks/{task_id}/fail");
3964 self.request_json(
3965 reqwest::Method::POST,
3966 &path,
3967 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3968 Some(&body),
3969 )
3970 .await
3971 }
3972
3973 pub async fn poll_activity_task(
3974 &self,
3975 worker_id: &str,
3976 task_queue: &str,
3977 timeout: Duration,
3978 ) -> Result<Option<ActivityTask>> {
3979 Ok(self
3980 .poll_activity_task_response(worker_id, task_queue, timeout)
3981 .await?
3982 .task)
3983 }
3984
3985 pub async fn poll_activity_task_response(
3987 &self,
3988 worker_id: &str,
3989 task_queue: &str,
3990 timeout: Duration,
3991 ) -> Result<PollActivityTaskResponse> {
3992 let poll_request_id = unique_request_id("rust-activity-poll");
3993 self.poll_activity_task_response_with_request_id(
3994 worker_id,
3995 task_queue,
3996 timeout,
3997 &poll_request_id,
3998 1,
3999 )
4000 .await
4001 }
4002
4003 async fn poll_activity_task_response_with_request_id(
4004 &self,
4005 worker_id: &str,
4006 task_queue: &str,
4007 timeout: Duration,
4008 poll_request_id: &str,
4009 transport_retries: usize,
4010 ) -> Result<PollActivityTaskResponse> {
4011 let body = json!({
4012 "worker_id": worker_id,
4013 "task_queue": task_queue,
4014 "poll_request_id": poll_request_id,
4015 "timeout_seconds": long_poll_timeout_seconds(timeout),
4016 });
4017 let data: PollActivityTaskResponse = self
4018 .poll_request_json(
4019 "/worker/activity-tasks/poll",
4020 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
4021 &body,
4022 timeout + Duration::from_secs(5),
4023 transport_retries,
4024 )
4025 .await?;
4026 Ok(data)
4027 }
4028
4029 pub async fn complete_activity_task<T: Serialize>(
4030 &self,
4031 task_id: &str,
4032 activity_attempt_id: &str,
4033 lease_owner: &str,
4034 result: T,
4035 codec: &str,
4036 ) -> Result<Value> {
4037 let result = encode_typed_envelope(&AvroValue::from_serialize(&result)?, codec)?;
4038 let body = json!({
4039 "activity_attempt_id": activity_attempt_id,
4040 "lease_owner": lease_owner,
4041 "result": result
4042 });
4043 let path = format!("/worker/activity-tasks/{task_id}/complete");
4044 activity_task_response(
4045 self.request_json(
4046 reqwest::Method::POST,
4047 &path,
4048 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
4049 Some(&body),
4050 )
4051 .await,
4052 "complete",
4053 task_id,
4054 activity_attempt_id,
4055 )
4056 }
4057
4058 pub async fn fail_activity_task(
4059 &self,
4060 task_id: &str,
4061 activity_attempt_id: &str,
4062 lease_owner: &str,
4063 message: impl Into<String>,
4064 non_retryable: bool,
4065 ) -> Result<Value> {
4066 let body = json!({
4067 "activity_attempt_id": activity_attempt_id,
4068 "lease_owner": lease_owner,
4069 "failure": {
4070 "message": message.into(),
4071 "type": "RustActivityFailure",
4072 "non_retryable": non_retryable
4073 }
4074 });
4075 let path = format!("/worker/activity-tasks/{task_id}/fail");
4076 activity_task_response(
4077 self.request_json(
4078 reqwest::Method::POST,
4079 &path,
4080 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
4081 Some(&body),
4082 )
4083 .await,
4084 "fail",
4085 task_id,
4086 activity_attempt_id,
4087 )
4088 }
4089
4090 pub async fn heartbeat_activity_task<T: Serialize>(
4091 &self,
4092 task_id: &str,
4093 activity_attempt_id: &str,
4094 lease_owner: &str,
4095 details: T,
4096 ) -> Result<ActivityHeartbeatResponse> {
4097 let details = encode_typed_envelope(&AvroValue::from_serialize(&details)?, DEFAULT_CODEC)?;
4098 let body = json!({
4099 "activity_attempt_id": activity_attempt_id,
4100 "lease_owner": lease_owner,
4101 "details": details
4102 });
4103 let path = format!("/worker/activity-tasks/{task_id}/heartbeat");
4104 activity_task_response(
4105 self.request_json(
4106 reqwest::Method::POST,
4107 &path,
4108 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
4109 Some(&body),
4110 )
4111 .await,
4112 "heartbeat",
4113 task_id,
4114 activity_attempt_id,
4115 )
4116 }
4117
4118 async fn request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
4119 &self,
4120 method: reqwest::Method,
4121 path: &str,
4122 protocol: RequestProtocol,
4123 body: Option<&B>,
4124 ) -> Result<T> {
4125 self.request_json_with_timeout(method, path, protocol, body, Duration::from_secs(60))
4126 .await
4127 }
4128
4129 async fn request_json_with_timeout<T: DeserializeOwned, B: Serialize + ?Sized>(
4130 &self,
4131 method: reqwest::Method,
4132 path: &str,
4133 protocol: RequestProtocol,
4134 body: Option<&B>,
4135 timeout: Duration,
4136 ) -> Result<T> {
4137 let auth_token = self.auth_token(protocol)?;
4138 let mut request = self
4139 .http
4140 .request(method.clone(), format!("{}/api{}", self.base_url, path))
4141 .timeout(timeout)
4142 .header(reqwest::header::ACCEPT, "application/json")
4143 .header(reqwest::header::CONTENT_TYPE, "application/json")
4144 .header("X-Namespace", &self.namespace);
4145
4146 match protocol {
4147 RequestProtocol::Worker(version) => {
4148 request = request.header("X-Durable-Workflow-Protocol-Version", version);
4149 }
4150 RequestProtocol::ControlPlane => {
4151 request = request.header(
4152 "X-Durable-Workflow-Control-Plane-Version",
4153 CONTROL_PLANE_VERSION,
4154 );
4155 }
4156 }
4157
4158 if let Some(token) = auth_token {
4159 request = request.bearer_auth(token);
4160 }
4161
4162 if let Some(body) = body {
4163 let mut body = serde_json::to_value(body)?;
4164 if matches!(
4165 method,
4166 reqwest::Method::POST | reqwest::Method::PUT | reqwest::Method::PATCH
4167 ) {
4168 self.externalize_runtime_payloads(&mut body, path, protocol)
4169 .await?;
4170 }
4171 request = request.json(&body);
4172 }
4173
4174 let request = request.build()?;
4175 let request_body = request
4176 .body()
4177 .and_then(reqwest::Body::as_bytes)
4178 .and_then(|body| serde_json::from_slice::<Value>(body).ok());
4179 let poll_request_id = path.ends_with("/poll").then(|| {
4180 request_body
4181 .as_ref()
4182 .and_then(|body| body.get("poll_request_id")?.as_str().map(str::to_owned))
4183 .unwrap_or_default()
4184 });
4185 let mut storage_retries = 0_usize;
4186
4187 loop {
4188 let response = self
4190 .http
4191 .execute(request.try_clone().ok_or_else(|| {
4192 Error::WorkerLoop("worker request body cannot be retried".to_string())
4193 })?)
4194 .await?;
4195 let status = response.status();
4196 let bytes = response.bytes().await?;
4197
4198 if !status.is_success() {
4199 let body = String::from_utf8_lossy(&bytes).to_string();
4200 if let Some(protocol) = protocol_failure(status, &body) {
4201 return Err(Error::Protocol(protocol));
4202 }
4203 let error = Error::Http { status, body };
4204 if self
4205 .wait_for_storage_admission(
4206 &error,
4207 protocol,
4208 Some(path),
4209 poll_request_id.as_deref(),
4210 request_body.as_ref(),
4211 &mut storage_retries,
4212 )
4213 .await
4214 {
4215 continue;
4216 }
4217 return Err(error);
4218 }
4219
4220 if bytes.is_empty() {
4221 return Ok(serde_json::from_value(Value::Null)?);
4222 }
4223
4224 let mut value: Value = serde_json::from_slice(&bytes)?;
4225 self.resolve_runtime_payloads(&mut value, path, protocol)
4226 .await?;
4227 return Ok(serde_json::from_value(value)?);
4228 }
4229 }
4230
4231 async fn wait_for_storage_admission(
4232 &self,
4233 error: &Error,
4234 protocol: RequestProtocol,
4235 path: Option<&str>,
4236 poll_request_id: Option<&str>,
4237 request_body: Option<&Value>,
4238 retries: &mut usize,
4239 ) -> bool {
4240 let Some(admission) = self
4241 .worker_storage_admission
4242 .as_ref()
4243 .filter(|_| matches!(protocol, RequestProtocol::Worker(_)))
4244 else {
4245 return false;
4246 };
4247 let Some(advertised_delay) = worker_storage_admission_retry_after(error, poll_request_id)
4248 .or_else(|| {
4249 path.and_then(|path| {
4250 worker_backend_unavailable_retry_after(error, path, poll_request_id)
4251 .or_else(|| worker_fenced_completion_retry_after(error, path, request_body))
4252 })
4253 })
4254 else {
4255 return false;
4256 };
4257 *retries = retries.saturating_add(1);
4258 let delay = worker_retry_delay(admission.policy, *retries)
4259 .max(advertised_delay)
4260 .min(admission.policy.max_backoff.max(Duration::from_millis(1)));
4261 let deadline = tokio::time::Instant::now() + delay;
4262 loop {
4263 if admission.stop.load(Ordering::SeqCst) {
4264 return false;
4265 }
4266 let remaining = deadline.saturating_duration_since(tokio::time::Instant::now());
4267 if remaining.is_zero() {
4268 return true;
4269 }
4270 tokio::time::sleep(remaining.min(Duration::from_millis(100))).await;
4271 }
4272 }
4273
4274 async fn poll_request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
4275 &self,
4276 path: &str,
4277 protocol: RequestProtocol,
4278 body: &B,
4279 timeout: Duration,
4280 max_retries: usize,
4281 ) -> Result<T> {
4282 let mut retries = 0;
4283
4284 loop {
4285 let response = self
4286 .request_json_with_timeout(
4287 reqwest::Method::POST,
4288 path,
4289 protocol,
4290 Some(body),
4291 timeout,
4292 )
4293 .await;
4294
4295 match response {
4296 Err(Error::Transport(_)) if retries < max_retries => retries += 1,
4297 response => return worker_poll_response(response),
4298 }
4299 }
4300 }
4301
4302 fn auth_token(&self, protocol: RequestProtocol) -> Result<Option<&str>> {
4303 match protocol {
4304 RequestProtocol::Worker(_) => {
4305 if let Some(token) = self.worker_token.as_deref().or(self.token.as_deref()) {
4306 return Ok(Some(token));
4307 }
4308 if self.control_token.is_some() {
4309 return Err(Error::MissingRoleCredentials {
4310 role: "worker",
4311 opposite_role: "control",
4312 });
4313 }
4314 Ok(None)
4315 }
4316 RequestProtocol::ControlPlane => {
4317 if let Some(token) = self.control_token.as_deref().or(self.token.as_deref()) {
4318 return Ok(Some(token));
4319 }
4320 if self.worker_token.is_some() {
4321 return Err(Error::MissingRoleCredentials {
4322 role: "control",
4323 opposite_role: "worker",
4324 });
4325 }
4326 Ok(None)
4327 }
4328 }
4329 }
4330}
4331
4332fn query_failure(status: reqwest::StatusCode, raw_body: String) -> QueryFailure {
4333 let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
4334 let reason = body
4335 .get("reason")
4336 .and_then(Value::as_str)
4337 .unwrap_or("query_rejected")
4338 .to_string();
4339 let message = body
4340 .get("message")
4341 .or_else(|| body.get("error"))
4342 .and_then(Value::as_str)
4343 .unwrap_or("workflow query was rejected")
4344 .to_string();
4345
4346 QueryFailure {
4347 status: status.as_u16(),
4348 reason,
4349 message,
4350 body,
4351 }
4352}
4353
4354fn workflow_command_result(
4355 command: WorkflowCommandKind,
4356 data: Value,
4357 workflow_id: &str,
4358 run_id: Option<&str>,
4359) -> WorkflowCommandResult {
4360 WorkflowCommandResult {
4361 command,
4362 workflow_id: data
4363 .get("workflow_id")
4364 .and_then(Value::as_str)
4365 .unwrap_or(workflow_id)
4366 .to_string(),
4367 run_id: data
4368 .get("run_id")
4369 .and_then(Value::as_str)
4370 .or(run_id)
4371 .map(str::to_string),
4372 outcome: data
4373 .get("outcome")
4374 .and_then(Value::as_str)
4375 .map(str::to_string),
4376 reason: data
4377 .get("reason")
4378 .and_then(Value::as_str)
4379 .map(str::to_string),
4380 command_status: data
4381 .get("command_status")
4382 .and_then(Value::as_str)
4383 .map(str::to_string),
4384 raw: data,
4385 }
4386}
4387
4388fn workflow_command_rejection(
4389 command: WorkflowCommandKind,
4390 status: reqwest::StatusCode,
4391 raw_body: String,
4392 workflow_id: &str,
4393 run_id: Option<&str>,
4394) -> WorkflowCommandRejection {
4395 let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
4396 WorkflowCommandRejection {
4397 command,
4398 status: status.as_u16(),
4399 reason: body
4400 .get("reason")
4401 .and_then(Value::as_str)
4402 .unwrap_or("workflow_command_rejected")
4403 .to_string(),
4404 message: body
4405 .get("message")
4406 .or_else(|| body.get("error"))
4407 .and_then(Value::as_str)
4408 .unwrap_or("workflow lifecycle command was rejected")
4409 .to_string(),
4410 workflow_id: body
4411 .get("workflow_id")
4412 .and_then(Value::as_str)
4413 .unwrap_or(workflow_id)
4414 .to_string(),
4415 run_id: body
4416 .get("run_id")
4417 .and_then(Value::as_str)
4418 .or(run_id)
4419 .map(str::to_string),
4420 target_scope: body
4421 .get("target_scope")
4422 .and_then(Value::as_str)
4423 .map(str::to_string),
4424 body,
4425 }
4426}
4427
4428fn query_task_response(response: Result<Value>) -> Result<Value> {
4429 match response {
4430 Err(Error::Http { status, body }) => Err(Error::QueryFailed(query_failure(status, body))),
4431 response => response,
4432 }
4433}
4434
4435fn worker_poll_response<T: DeserializeOwned>(response: Result<T>) -> Result<T> {
4436 match response {
4437 Err(Error::Http { status, body })
4438 if status == reqwest::StatusCode::CONFLICT && worker_poll_body_is_stop(&body) =>
4439 {
4440 Ok(serde_json::from_str(&body)?)
4441 }
4442 response => response,
4443 }
4444}
4445
4446fn worker_poll_body_is_stop(body: &str) -> bool {
4447 serde_json::from_str::<Value>(body)
4448 .ok()
4449 .is_some_and(|body| {
4450 worker_poll_is_stop(
4451 body.get("poll_status").and_then(Value::as_str),
4452 body.get("reason").and_then(Value::as_str),
4453 )
4454 })
4455}
4456
4457fn worker_poll_is_stop(poll_status: Option<&str>, reason: Option<&str>) -> bool {
4458 matches!(poll_status, Some("draining" | "stopped"))
4459 || matches!(reason, Some("worker_draining" | "worker_stopped"))
4460}
4461
4462fn query_task_rejection_is_final(error: &Error) -> bool {
4463 matches!(
4464 error,
4465 Error::QueryFailed(failure)
4466 if QUERY_TASK_FINAL_REJECTION_REASONS.contains(&failure.reason.as_str())
4467 )
4468}
4469
4470fn activity_task_response<T>(
4471 response: Result<T>,
4472 operation: &str,
4473 task_id: &str,
4474 activity_attempt_id: &str,
4475) -> Result<T> {
4476 match response {
4477 Err(Error::Http { status, body }) => {
4478 let body = serde_json::from_str(&body).unwrap_or_else(|_| json!({"message": body}));
4479 Err(Error::ActivityTaskRejected(ActivityTaskRejection {
4480 operation: operation.to_string(),
4481 status: status.as_u16(),
4482 reason: body
4483 .get("reason")
4484 .and_then(Value::as_str)
4485 .unwrap_or("activity_task_rejected")
4486 .to_string(),
4487 task_id: body
4488 .get("task_id")
4489 .and_then(Value::as_str)
4490 .unwrap_or(task_id)
4491 .to_string(),
4492 activity_attempt_id: body
4493 .get("activity_attempt_id")
4494 .and_then(Value::as_str)
4495 .unwrap_or(activity_attempt_id)
4496 .to_string(),
4497 cancel_requested: body
4498 .get("cancel_requested")
4499 .and_then(Value::as_bool)
4500 .unwrap_or(false),
4501 can_continue: body.get("can_continue").and_then(Value::as_bool),
4502 run_closed_reason: body
4503 .get("run_closed_reason")
4504 .and_then(Value::as_str)
4505 .map(str::to_string),
4506 body,
4507 }))
4508 }
4509 response => response,
4510 }
4511}
4512
4513fn activity_task_rejection_is_final(error: &Error) -> bool {
4514 matches!(
4515 error,
4516 Error::ActivityTaskRejected(rejection)
4517 if matches!(
4518 rejection.reason.as_str(),
4519 "run_cancelled"
4520 | "run_terminated"
4521 | "attempt_closed"
4522 | "stale_attempt"
4523 | "activity_cancelled"
4524 | "task_cancelled"
4525 | "run_closed"
4526 | "activity_not_running"
4527 | "attempt_not_found"
4528 )
4529 )
4530}
4531
4532fn workflow_task_completion_is_terminal_timeout(
4533 error: &Error,
4534 task_id: &str,
4535 workflow_task_attempt: u64,
4536 run_id: Option<&str>,
4537) -> bool {
4538 let Error::Http { status, body } = error else {
4539 return false;
4540 };
4541 if *status != reqwest::StatusCode::CONFLICT {
4542 return false;
4543 }
4544
4545 let Some(run_id) = run_id else {
4546 return false;
4547 };
4548 let Ok(body) = serde_json::from_str::<Value>(body) else {
4549 return false;
4550 };
4551
4552 body.get("recorded").and_then(Value::as_bool) == Some(false)
4553 && body.get("reason").and_then(Value::as_str) == Some("run_timed_out")
4554 && body.get("run_status").and_then(Value::as_str) == Some("failed")
4555 && body.get("run_id").and_then(Value::as_str) == Some(run_id)
4556 && body.get("task_id").and_then(Value::as_str) == Some(task_id)
4557 && body.get("workflow_task_attempt").and_then(Value::as_u64) == Some(workflow_task_attempt)
4558}
4559
4560fn protocol_failure(status: reqwest::StatusCode, raw_body: &str) -> Option<ProtocolFailure> {
4561 let body: Value = serde_json::from_str(raw_body).ok()?;
4562 let reason = body.get("reason")?.as_str()?;
4563 if !matches!(
4564 reason,
4565 "missing_protocol_version"
4566 | "unsupported_protocol_version"
4567 | "missing_control_plane_version"
4568 | "unsupported_control_plane_version"
4569 ) {
4570 return None;
4571 }
4572
4573 Some(ProtocolFailure {
4574 status: status.as_u16(),
4575 reason: reason.to_string(),
4576 message: body
4577 .get("message")
4578 .or_else(|| body.get("error"))
4579 .and_then(Value::as_str)
4580 .unwrap_or("protocol version rejected")
4581 .to_string(),
4582 supported_version: body
4583 .get("supported_version")
4584 .and_then(Value::as_str)
4585 .map(str::to_string),
4586 requested_version: body
4587 .get("requested_version")
4588 .and_then(Value::as_str)
4589 .map(str::to_string),
4590 body,
4591 })
4592}
4593
4594fn long_poll_timeout_seconds(timeout: Duration) -> u64 {
4595 timeout
4596 .as_secs()
4597 .saturating_add(u64::from(timeout.subsec_nanos() > 0))
4598 .min(MAX_LONG_POLL_TIMEOUT_SECONDS)
4599}
4600
4601fn worker_operation_is_retryable(error: &Error) -> bool {
4602 if worker_poll_capacity_retry_after(error).is_some()
4603 || worker_storage_admission_body(error).is_some()
4604 || worker_backend_unavailable_body(error).is_some()
4605 || worker_operation_is_explicitly_non_retryable(error)
4606 {
4607 return false;
4608 }
4609
4610 match error {
4611 Error::Transport(error) => {
4612 error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()
4613 }
4614 Error::Http { status, .. } => {
4615 matches!(
4616 *status,
4617 reqwest::StatusCode::REQUEST_TIMEOUT | reqwest::StatusCode::TOO_MANY_REQUESTS
4618 ) || status.is_server_error()
4619 }
4620 _ => false,
4621 }
4622}
4623
4624fn worker_storage_admission_body(error: &Error) -> Option<Value> {
4625 let body: Value = match error {
4626 Error::Http { body, .. } => serde_json::from_str(body).ok()?,
4627 Error::ActivityTaskRejected(rejection) => rejection.body.clone(),
4628 _ => return None,
4629 };
4630 matches!(
4631 body.get("reason").and_then(Value::as_str),
4632 Some("storage_pressure" | "storage_admission_unavailable")
4633 )
4634 .then_some(body)
4635}
4636
4637fn worker_storage_admission_retry_after(
4638 error: &Error,
4639 poll_request_id: Option<&str>,
4640) -> Option<Duration> {
4641 let Error::Http { status, .. } = error else {
4642 return None;
4643 };
4644 let body = worker_storage_admission_body(error)?;
4645 let delay = body.get("retry_after_seconds")?.as_u64()?;
4646 if *status != reqwest::StatusCode::SERVICE_UNAVAILABLE
4647 || delay == 0
4648 || body.get("retryable") != Some(&Value::Bool(true))
4649 || !matches!(body.get("storage_state")?.as_str()?, "draining" | "fenced")
4650 || (body["reason"] == "storage_admission_unavailable" && body["storage_state"] != "fenced")
4651 || body
4652 .get("request_admitted")
4653 .is_some_and(|admitted| admitted != &Value::Bool(false))
4654 {
4655 return None;
4656 }
4657 match poll_request_id {
4658 Some(id) => {
4659 if id.is_empty()
4660 || body.get("task") != Some(&Value::Null)
4661 || body.get("poll_request_id").and_then(Value::as_str) != Some(id)
4662 || body.get("poll_status") != body.get("reason")
4663 || body.get("retry_same_poll_request_id") != Some(&Value::Bool(true))
4664 || body.get("claim_admitted") != Some(&Value::Bool(false))
4665 {
4666 return None;
4667 }
4668 }
4669 None if body.get("request_admitted") != Some(&Value::Bool(false)) => return None,
4670 None => {}
4671 }
4672 Some(Duration::from_secs(delay))
4673}
4674
4675fn worker_backend_unavailable_retry_after(
4676 error: &Error,
4677 path: &str,
4678 poll_request_id: Option<&str>,
4679) -> Option<Duration> {
4680 let operation = match path {
4681 "/worker/register" => "register_worker",
4682 "/worker/heartbeat" => "heartbeat_worker",
4683 "/worker/workflow-tasks/poll" => "poll_workflow_task",
4684 "/worker/activity-tasks/poll" => "poll_activity_task",
4685 "/worker/query-tasks/poll" => "poll_query_task",
4686 "/worker/update-validation-tasks/poll" => "poll_update_validation_task",
4687 _ => return None,
4688 };
4689 let body = worker_backend_unavailable_body(error)?;
4690 let delay = body.get("retry_after_seconds")?.as_u64()?;
4691 if delay == 0
4692 || body.get("operation")?.as_str()? != operation
4693 || body.get("outcome")?.as_str()? != "unknown"
4694 || body.get("retryable") != Some(&Value::Bool(true))
4695 || body.get("worker_id")?.as_str()?.is_empty()
4696 {
4697 return None;
4698 }
4699 if let Some(id) = poll_request_id {
4700 if id.is_empty()
4701 || body.get("task") != Some(&Value::Null)
4702 || body.get("poll_status")?.as_str()? != "backend_unavailable"
4703 || body.get("poll_request_id")?.as_str()? != id
4704 || body.get("retry_same_poll_request_id") != Some(&Value::Bool(true))
4705 {
4706 return None;
4707 }
4708 } else if path.ends_with("/poll") {
4709 return None;
4710 }
4711 Some(Duration::from_secs(delay))
4712}
4713
4714fn worker_fenced_completion_retry_after(
4715 error: &Error,
4716 path: &str,
4717 request_body: Option<&Value>,
4718) -> Option<Duration> {
4719 let (operation, task_id, attempt_field) = if let Some(task_id) = path
4720 .strip_prefix("/worker/workflow-tasks/")
4721 .and_then(|tail| tail.strip_suffix("/complete"))
4722 {
4723 ("complete_workflow_task", task_id, "workflow_task_attempt")
4724 } else if let Some(task_id) = path
4725 .strip_prefix("/worker/activity-tasks/")
4726 .and_then(|tail| tail.strip_suffix("/complete"))
4727 {
4728 ("complete_activity_task", task_id, "activity_attempt_id")
4729 } else {
4730 return None;
4731 };
4732 if task_id.is_empty() || task_id.contains('/') {
4733 return None;
4734 }
4735
4736 let request = request_body?.as_object()?;
4737 let lease_owner = request.get("lease_owner")?.as_str()?;
4738 if lease_owner.is_empty() {
4739 return None;
4740 }
4741 let attempt = request.get(attempt_field)?;
4742 if (attempt_field == "workflow_task_attempt" && attempt.as_u64().filter(|id| *id > 0).is_none())
4743 || (attempt_field == "activity_attempt_id"
4744 && attempt.as_str().filter(|id| !id.is_empty()).is_none())
4745 {
4746 return None;
4747 }
4748
4749 let body = worker_backend_unavailable_body(error)?;
4750 let delay = body.get("retry_after_seconds")?.as_u64()?;
4751 if delay == 0
4752 || body.get("operation")?.as_str()? != operation
4753 || body.get("outcome")?.as_str()? != "unknown"
4754 || body.get("retryable") != Some(&Value::Bool(true))
4755 || body.get("worker_id")?.as_str()? != lease_owner
4756 || body.get("task_queue") != Some(&Value::Null)
4757 || body.get("task_id")?.as_str()? != task_id
4758 || body.get("lease_owner")?.as_str()? != lease_owner
4759 || body.get(attempt_field) != Some(attempt)
4760 {
4761 return None;
4762 }
4763
4764 Some(Duration::from_secs(delay))
4765}
4766
4767fn worker_backend_unavailable_body(error: &Error) -> Option<Value> {
4768 let Error::Http { status, body } = error else {
4769 return None;
4770 };
4771 if *status != reqwest::StatusCode::SERVICE_UNAVAILABLE {
4772 return None;
4773 }
4774 let body: Value = serde_json::from_str(body).ok()?;
4775 (body.get("reason")?.as_str()? == "backend_unavailable").then_some(body)
4776}
4777
4778fn worker_operation_is_explicitly_non_retryable(error: &Error) -> bool {
4779 let Error::Http { body, .. } = error else {
4780 return false;
4781 };
4782
4783 serde_json::from_str::<Value>(body)
4784 .ok()
4785 .and_then(|body| body.get("retryable").and_then(Value::as_bool))
4786 == Some(false)
4787}
4788
4789fn worker_poll_capacity_retry_after(error: &Error) -> Option<Duration> {
4790 let Error::Http { status, body } = error else {
4791 return None;
4792 };
4793 if *status != reqwest::StatusCode::TOO_MANY_REQUESTS {
4794 return None;
4795 }
4796
4797 let body = serde_json::from_str::<Value>(body).ok()?;
4798 let capacity_exhausted = body.get("poll_status").and_then(Value::as_str)
4799 == Some("long_poll_capacity_exhausted")
4800 || body.get("reason").and_then(Value::as_str) == Some("long_poll_capacity_exhausted");
4801 if !capacity_exhausted || body.get("retryable").and_then(Value::as_bool) != Some(true) {
4802 return None;
4803 }
4804
4805 Some(Duration::from_secs(
4806 body.get("retry_after_seconds")
4807 .and_then(Value::as_u64)
4808 .unwrap_or_default(),
4809 ))
4810}
4811
4812fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration {
4813 let exponent = retry.saturating_sub(1).min(31) as u32;
4814 policy
4815 .initial_backoff
4816 .saturating_mul(1_u32 << exponent)
4817 .min(policy.max_backoff)
4818}
4819
4820#[derive(Debug)]
4821pub struct ClientBuilder {
4822 base_url: String,
4823 token: Option<String>,
4824 control_token: Option<String>,
4825 worker_token: Option<String>,
4826 namespace: String,
4827 timeout: Duration,
4828 max_external_payload_bytes: usize,
4829}
4830
4831impl ClientBuilder {
4832 pub fn token(mut self, token: Option<String>) -> Self {
4833 self.token = token;
4834 self
4835 }
4836
4837 pub fn control_token(mut self, token: Option<String>) -> Self {
4838 self.control_token = token;
4839 self
4840 }
4841
4842 pub fn worker_token(mut self, token: Option<String>) -> Self {
4843 self.worker_token = token;
4844 self
4845 }
4846
4847 pub fn namespace(mut self, namespace: impl Into<String>) -> Self {
4848 self.namespace = namespace.into();
4849 self
4850 }
4851
4852 pub fn timeout(mut self, timeout: Duration) -> Self {
4853 self.timeout = timeout;
4854 self
4855 }
4856
4857 pub fn max_external_payload_bytes(mut self, bytes: usize) -> Self {
4860 self.max_external_payload_bytes = bytes;
4861 self
4862 }
4863
4864 pub fn build(self) -> Result<Client> {
4865 let base_url = self.base_url.trim_end_matches('/').to_string();
4866 let has_sdk_api_suffix = reqwest::Url::parse(&base_url)
4867 .map(|url| url.path().trim_end_matches('/').ends_with("/api"))
4868 .unwrap_or_else(|_| base_url.ends_with("/api"));
4869
4870 if has_sdk_api_suffix {
4871 return Err(Error::InvalidBaseUrl);
4872 }
4873
4874 Ok(Client {
4875 http: reqwest::Client::builder()
4876 .timeout(self.timeout)
4877 .redirect(reqwest::redirect::Policy::none())
4878 .build()?,
4879 base_url,
4880 token: self.token,
4881 control_token: self.control_token,
4882 worker_token: self.worker_token,
4883 namespace: self.namespace,
4884 max_external_payload_bytes: self.max_external_payload_bytes,
4885 worker_storage_admission: None,
4886 runtime_upload_policy: Arc::new(Mutex::new([None, None])),
4887 })
4888 }
4889}
4890
4891#[derive(Clone, Debug)]
4892pub struct WorkflowHandle {
4893 client: Client,
4894 pub workflow_id: String,
4895 pub run_id: Option<String>,
4896 pub workflow_type: String,
4897}
4898
4899impl WorkflowHandle {
4900 pub async fn describe(&self) -> Result<WorkflowDescription> {
4902 self.client.describe_workflow(&self.workflow_id).await
4903 }
4904
4905 pub async fn describe_selected_run(&self) -> Result<WorkflowDescription> {
4907 let run_id = self.run_id.as_deref().ok_or_else(|| {
4908 Error::Codec("run_id is required for selected-run description".to_string())
4909 })?;
4910 self.client
4911 .describe_workflow_run(&self.workflow_id, run_id)
4912 .await
4913 }
4914
4915 pub async fn signal<T: Serialize>(&self, signal_name: &str, input: T) -> Result<Value> {
4916 self.client
4917 .signal_workflow(&self.workflow_id, signal_name, input)
4918 .await
4919 }
4920
4921 pub async fn append_message<T: Serialize>(
4922 &self,
4923 stream_name: &str,
4924 message_id: &str,
4925 input: T,
4926 ) -> Result<Value> {
4927 self.client
4928 .append_message_stream(&self.workflow_id, stream_name, message_id, input)
4929 .await
4930 }
4931
4932 pub async fn signal_selected_run<T: Serialize>(
4934 &self,
4935 signal_name: &str,
4936 input: T,
4937 ) -> Result<Value> {
4938 let run_id = self.run_id.as_deref().ok_or_else(|| {
4939 Error::Codec("run_id is required for selected-run signaling".to_string())
4940 })?;
4941 self.client
4942 .signal_workflow_run(&self.workflow_id, run_id, signal_name, input)
4943 .await
4944 }
4945
4946 pub async fn cancel(&self, options: WorkflowCommandOptions) -> Result<WorkflowCommandResult> {
4948 self.client
4949 .cancel_workflow(&self.workflow_id, options)
4950 .await
4951 }
4952
4953 pub async fn cancel_selected_run(
4955 &self,
4956 options: WorkflowCommandOptions,
4957 ) -> Result<WorkflowCommandResult> {
4958 let run_id = self.run_id.as_deref().ok_or_else(|| {
4959 Error::Codec("run_id is required for selected-run cancellation".to_string())
4960 })?;
4961 self.client
4962 .cancel_workflow_run(&self.workflow_id, run_id, options)
4963 .await
4964 }
4965
4966 pub async fn terminate(
4968 &self,
4969 options: WorkflowCommandOptions,
4970 ) -> Result<WorkflowCommandResult> {
4971 self.client
4972 .terminate_workflow(&self.workflow_id, options)
4973 .await
4974 }
4975
4976 pub async fn terminate_selected_run(
4978 &self,
4979 options: WorkflowCommandOptions,
4980 ) -> Result<WorkflowCommandResult> {
4981 let run_id = self.run_id.as_deref().ok_or_else(|| {
4982 Error::Codec("run_id is required for selected-run termination".to_string())
4983 })?;
4984 self.client
4985 .terminate_workflow_run(&self.workflow_id, run_id, options)
4986 .await
4987 }
4988
4989 pub async fn redrive(&self, request_id: Option<&str>) -> Result<WorkflowRedriveResult> {
4991 let run_id = self
4992 .run_id
4993 .as_deref()
4994 .ok_or_else(|| Error::Codec("run_id is required for redrive".to_string()))?;
4995 self.client
4996 .redrive_workflow_run(&self.workflow_id, run_id, request_id)
4997 .await
4998 }
4999
5000 pub async fn query<T: Serialize>(&self, query_name: &str, input: T) -> Result<Value> {
5002 self.client
5003 .query_workflow(&self.workflow_id, query_name, input)
5004 .await
5005 }
5006
5007 pub async fn query_avro_value<T: Serialize>(
5008 &self,
5009 query_name: &str,
5010 input: T,
5011 ) -> Result<AvroValue> {
5012 self.client
5013 .query_workflow_avro_value(&self.workflow_id, query_name, input)
5014 .await
5015 }
5016
5017 pub async fn update<T: Serialize>(
5018 &self,
5019 update_name: &str,
5020 input: T,
5021 request_id: Option<&str>,
5022 ) -> Result<Value> {
5023 self.client
5024 .update_workflow(&self.workflow_id, update_name, input, request_id)
5025 .await
5026 }
5027
5028 pub async fn update_avro_value<T: Serialize>(
5029 &self,
5030 update_name: &str,
5031 input: T,
5032 request_id: Option<&str>,
5033 ) -> Result<AvroValue> {
5034 self.client
5035 .update_workflow_avro_value(&self.workflow_id, update_name, input, request_id)
5036 .await
5037 }
5038
5039 pub async fn query_selected_run<T: Serialize>(
5041 &self,
5042 query_name: &str,
5043 input: T,
5044 ) -> Result<Value> {
5045 let run_id = self
5046 .run_id
5047 .as_deref()
5048 .ok_or_else(|| Error::Codec("run_id is required for selected-run query".to_string()))?;
5049 self.client
5050 .query_workflow_run(&self.workflow_id, run_id, query_name, input)
5051 .await
5052 }
5053
5054 pub async fn result(&self, options: WorkflowResultOptions) -> Result<Value> {
5056 self.result_target(options, None).await
5057 }
5058
5059 pub async fn result_avro_value(&self, options: WorkflowResultOptions) -> Result<AvroValue> {
5061 self.result_avro_value_target(options, None).await
5062 }
5063
5064 pub async fn result_typed<T: DeserializeOwned>(
5066 &self,
5067 options: WorkflowResultOptions,
5068 ) -> Result<T> {
5069 let result = self.result_avro_value(options).await?;
5070 decode_handler_result(result, HandlerKind::Workflow, &self.workflow_type)
5071 }
5072
5073 pub async fn result_selected_run(&self, options: WorkflowResultOptions) -> Result<Value> {
5075 let run_id = self.run_id.as_deref().ok_or_else(|| {
5076 Error::Codec("run_id is required for selected-run result".to_string())
5077 })?;
5078 self.result_target(options, Some(run_id)).await
5079 }
5080
5081 pub async fn result_selected_run_avro_value(
5083 &self,
5084 options: WorkflowResultOptions,
5085 ) -> Result<AvroValue> {
5086 let run_id = self.run_id.as_deref().ok_or_else(|| {
5087 Error::Codec("run_id is required for selected-run result".to_string())
5088 })?;
5089 self.result_avro_value_target(options, Some(run_id)).await
5090 }
5091
5092 pub async fn result_selected_run_typed<T: DeserializeOwned>(
5094 &self,
5095 options: WorkflowResultOptions,
5096 ) -> Result<T> {
5097 let result = self.result_selected_run_avro_value(options).await?;
5098 decode_handler_result(result, HandlerKind::Workflow, &self.workflow_type)
5099 }
5100
5101 async fn result_avro_value_target(
5102 &self,
5103 options: WorkflowResultOptions,
5104 selected_run_id: Option<&str>,
5105 ) -> Result<AvroValue> {
5106 let started = Instant::now();
5107
5108 loop {
5109 let description = match selected_run_id {
5110 Some(run_id) => {
5111 self.client
5112 .describe_workflow_run(&self.workflow_id, run_id)
5113 .await?
5114 }
5115 None => self.describe().await?,
5116 };
5117 if description.is_completed() {
5118 return description.output_avro_value.ok_or_else(|| {
5119 Error::Codec(
5120 "missing_payload_envelope: typed workflow result requires output_envelope"
5121 .to_string(),
5122 )
5123 });
5124 }
5125 if description.is_terminal() {
5126 let outcome =
5127 workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
5128 return Err(match outcome.kind {
5129 WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
5130 WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
5131 WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
5132 WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
5133 });
5134 }
5135 if started.elapsed() >= options.timeout {
5136 return Err(Error::Timeout);
5137 }
5138 tokio::time::sleep(options.poll_interval).await;
5139 }
5140 }
5141
5142 async fn result_target(
5143 &self,
5144 options: WorkflowResultOptions,
5145 selected_run_id: Option<&str>,
5146 ) -> Result<Value> {
5147 let started = Instant::now();
5148
5149 loop {
5150 let description = match selected_run_id {
5151 Some(run_id) => {
5152 self.client
5153 .describe_workflow_run(&self.workflow_id, run_id)
5154 .await?
5155 }
5156 None => self.describe().await?,
5157 };
5158 if description.is_completed() {
5159 return Ok(description.output.unwrap_or(Value::Null));
5160 }
5161
5162 if description.is_terminal() {
5163 let outcome =
5164 workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
5165 return Err(match outcome.kind {
5166 WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
5167 WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
5168 WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
5169 WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
5170 });
5171 }
5172
5173 if started.elapsed() >= options.timeout {
5174 return Err(Error::WorkflowTimedOut(WorkflowTerminalOutcome {
5175 kind: WorkflowTerminalKind::TimedOut,
5176 workflow_id: description
5177 .workflow_id
5178 .clone()
5179 .unwrap_or_else(|| self.workflow_id.clone()),
5180 run_id: description
5181 .run_id
5182 .clone()
5183 .or_else(|| selected_run_id.map(str::to_string)),
5184 reason: "result_wait_timeout".to_string(),
5185 failure_category: Some("client_timeout".to_string()),
5186 failure_id: None,
5187 exception_type: None,
5188 exception_class: None,
5189 non_retryable: None,
5190 message: Some(format!(
5191 "workflow result was not terminal within {:?}",
5192 options.timeout
5193 )),
5194 exception: None,
5195 raw: description.raw_value(),
5196 }));
5197 }
5198
5199 tokio::time::sleep(options.poll_interval).await;
5200 }
5201 }
5202}
5203
5204#[derive(Clone, Copy, Debug)]
5205pub struct WorkflowResultOptions {
5206 pub poll_interval: Duration,
5207 pub timeout: Duration,
5208}
5209
5210impl Default for WorkflowResultOptions {
5211 fn default() -> Self {
5212 Self {
5213 poll_interval: Duration::from_millis(500),
5214 timeout: Duration::from_secs(30),
5215 }
5216 }
5217}
5218
5219#[derive(Clone, Debug, Deserialize)]
5220pub struct WorkflowDescription {
5221 pub workflow_id: Option<String>,
5222 pub run_id: Option<String>,
5223 pub workflow_type: Option<String>,
5224 pub status: Option<String>,
5225 #[serde(default)]
5226 pub closed_reason: Option<String>,
5227 #[serde(default)]
5228 pub error: Option<String>,
5229 #[serde(default)]
5230 pub failure: Option<Value>,
5231 #[serde(default)]
5232 pub exception: Option<Value>,
5233 #[serde(default)]
5234 pub failures: Vec<Value>,
5235 #[serde(default)]
5236 pub output: Option<Value>,
5237 #[serde(default)]
5238 pub output_envelope: Option<Value>,
5239 #[serde(skip)]
5240 pub output_avro_value: Option<AvroValue>,
5241 #[serde(flatten)]
5242 pub raw: HashMap<String, Value>,
5243}
5244
5245#[derive(Clone, Debug, Deserialize)]
5247pub struct WorkflowStreamDescription {
5248 pub stream_name: String,
5249 pub status: String,
5250 pub last_offset: i64,
5251 pub total_items: u64,
5252 pub pending_items: u64,
5253 #[serde(default)]
5254 pub opened_at: Option<String>,
5255 #[serde(default)]
5256 pub last_appended_at: Option<String>,
5257 #[serde(default)]
5258 pub closed_at: Option<String>,
5259 #[serde(default)]
5260 pub error_reason: Option<String>,
5261 #[serde(default)]
5262 pub retention_seconds: Option<u64>,
5263 #[serde(flatten)]
5264 pub raw: HashMap<String, Value>,
5265}
5266
5267impl WorkflowStreamDescription {
5268 pub fn is_terminal(&self) -> bool {
5269 matches!(self.status.as_str(), "closed" | "errored")
5270 }
5271}
5272
5273#[derive(Clone, Debug, Default)]
5275pub struct WorkflowStreamAppendItem {
5276 pub payload_envelope: Option<Value>,
5277 pub payload_reference: Option<String>,
5278 pub item_type: Option<String>,
5279 pub content_type: Option<String>,
5280 pub idempotency_key: Option<String>,
5281}
5282
5283impl WorkflowStreamAppendItem {
5284 pub fn new<T: Serialize>(payload: T) -> Result<Self> {
5286 let value = AvroValue::from_serialize(&payload)?;
5287 Ok(Self {
5288 payload_envelope: Some(encode_typed_envelope(&value, DEFAULT_CODEC)?),
5289 ..Self::default()
5290 })
5291 }
5292
5293 pub fn from_reference(reference: impl Into<String>) -> Self {
5295 Self {
5296 payload_reference: Some(reference.into()),
5297 ..Self::default()
5298 }
5299 }
5300
5301 pub fn item_type(mut self, item_type: impl Into<String>) -> Self {
5302 self.item_type = Some(item_type.into());
5303 self
5304 }
5305
5306 pub fn content_type(mut self, content_type: impl Into<String>) -> Self {
5307 self.content_type = Some(content_type.into());
5308 self
5309 }
5310
5311 pub fn idempotency_key(mut self, idempotency_key: impl Into<String>) -> Self {
5312 self.idempotency_key = Some(idempotency_key.into());
5313 self
5314 }
5315
5316 fn wire_value(&self, derived_idempotency_key: Option<String>) -> Value {
5317 let mut item = serde_json::Map::new();
5318 if let Some(payload) = &self.payload_envelope {
5319 item.insert("payload".to_string(), payload.clone());
5320 item.insert("payload_codec".to_string(), json!(DEFAULT_CODEC));
5321 }
5322 if let Some(reference) = &self.payload_reference {
5323 item.insert("payload_reference".to_string(), json!(reference));
5324 }
5325 if let Some(item_type) = &self.item_type {
5326 item.insert("item_type".to_string(), json!(item_type));
5327 }
5328 if let Some(content_type) = &self.content_type {
5329 item.insert("content_type".to_string(), json!(content_type));
5330 }
5331 if let Some(key) = derived_idempotency_key
5332 .as_ref()
5333 .or(self.idempotency_key.as_ref())
5334 {
5335 item.insert("idempotency_key".to_string(), json!(key));
5336 }
5337 Value::Object(item)
5338 }
5339}
5340
5341#[derive(Clone, Debug)]
5343pub struct WorkflowStreamItem {
5344 pub offset: u64,
5345 pub payload: Option<Value>,
5346 pub payload_envelope: Option<Value>,
5347 pub payload_reference: Option<String>,
5348 pub payload_codec: Option<String>,
5349 pub idempotency_key: Option<String>,
5350 pub item_type: Option<String>,
5351 pub content_type: Option<String>,
5352 pub origin: Option<String>,
5353 pub origin_reference: Option<String>,
5354 pub emitted_at: Option<String>,
5355 pub raw: Value,
5356}
5357
5358#[derive(Clone, Debug)]
5360pub struct WorkflowStreamPage {
5361 pub stream: WorkflowStreamDescription,
5362 pub items: Vec<WorkflowStreamItem>,
5363 pub next_offset: u64,
5364 pub terminal: bool,
5365}
5366
5367#[derive(Clone, Debug)]
5369pub struct WorkflowStreamAppendResult {
5370 pub stream: WorkflowStreamDescription,
5371 pub accepted_offsets: Vec<u64>,
5372 pub accepted: u64,
5373 pub deduped: u64,
5374}
5375
5376#[derive(Deserialize)]
5377struct WorkflowStreamListResponse {
5378 #[serde(default)]
5379 streams: Vec<WorkflowStreamDescription>,
5380}
5381
5382#[derive(Deserialize)]
5383struct WorkflowStreamDescriptionResponse {
5384 stream: WorkflowStreamDescription,
5385}
5386
5387#[derive(Deserialize)]
5388struct WorkflowStreamPageResponse {
5389 stream: WorkflowStreamDescription,
5390 #[serde(default)]
5391 items: Vec<Value>,
5392 next_offset: u64,
5393 terminal: bool,
5394}
5395
5396#[derive(Deserialize)]
5397struct WorkflowStreamAppendResponse {
5398 stream: WorkflowStreamDescription,
5399 #[serde(default)]
5400 accepted_offsets: Vec<u64>,
5401 accepted: u64,
5402 deduped: u64,
5403}
5404
5405impl WorkflowDescription {
5406 pub fn is_completed(&self) -> bool {
5407 matches!(self.status.as_deref(), Some("completed" | "Completed"))
5408 }
5409
5410 pub fn is_terminal(&self) -> bool {
5411 matches!(
5412 self.status.as_deref(),
5413 Some(
5414 "completed"
5415 | "Completed"
5416 | "failed"
5417 | "Failed"
5418 | "cancelled"
5419 | "Cancelled"
5420 | "terminated"
5421 | "Terminated"
5422 | "timed_out"
5423 | "TimedOut",
5424 )
5425 )
5426 }
5427
5428 fn decode_payloads(&mut self) -> Result<()> {
5429 if let Some(envelope) = &self.output_envelope {
5430 let value = decode_wire_avro_value(envelope, DEFAULT_CODEC)?;
5431 self.output = Some(value.clone().into_json()?);
5432 self.output_avro_value = Some(value);
5433 }
5434
5435 Ok(())
5436 }
5437
5438 fn raw_value(&self) -> Value {
5439 let mut data = self.raw.clone();
5440 data.insert(
5441 "workflow_id".to_string(),
5442 self.workflow_id
5443 .clone()
5444 .map(Value::String)
5445 .unwrap_or(Value::Null),
5446 );
5447 data.insert(
5448 "run_id".to_string(),
5449 self.run_id
5450 .clone()
5451 .map(Value::String)
5452 .unwrap_or(Value::Null),
5453 );
5454 data.insert(
5455 "workflow_type".to_string(),
5456 self.workflow_type
5457 .clone()
5458 .map(Value::String)
5459 .unwrap_or(Value::Null),
5460 );
5461 data.insert(
5462 "status".to_string(),
5463 self.status
5464 .clone()
5465 .map(Value::String)
5466 .unwrap_or(Value::Null),
5467 );
5468 data.insert(
5469 "closed_reason".to_string(),
5470 self.closed_reason
5471 .clone()
5472 .map(Value::String)
5473 .unwrap_or(Value::Null),
5474 );
5475 if let Some(failure) = &self.failure {
5476 data.insert("failure".to_string(), failure.clone());
5477 }
5478 if let Some(exception) = &self.exception {
5479 data.insert("exception".to_string(), exception.clone());
5480 }
5481 Value::Object(data.into_iter().collect())
5482 }
5483}
5484
5485fn workflow_terminal_outcome(
5486 description: &WorkflowDescription,
5487 workflow_id: &str,
5488 run_id: Option<&str>,
5489) -> WorkflowTerminalOutcome {
5490 let terminal_kind = description
5491 .closed_reason
5492 .as_deref()
5493 .or(description.status.as_deref())
5494 .unwrap_or("failed")
5495 .to_ascii_lowercase();
5496 let kind = match terminal_kind.as_str() {
5497 "cancelled" | "canceled" => WorkflowTerminalKind::Cancelled,
5498 "terminated" => WorkflowTerminalKind::Terminated,
5499 "timed_out" | "timedout" => WorkflowTerminalKind::TimedOut,
5500 _ => WorkflowTerminalKind::Failed,
5501 };
5502 let default_reason = match kind {
5503 WorkflowTerminalKind::Failed => "workflow_failed",
5504 WorkflowTerminalKind::Cancelled => "cancelled",
5505 WorkflowTerminalKind::Terminated => "terminated",
5506 WorkflowTerminalKind::TimedOut => "timed_out",
5507 };
5508 let failure = description
5509 .failure
5510 .as_ref()
5511 .filter(|value| value.is_object());
5512 let nested_failure = failure
5513 .and_then(|value| value.get("failures"))
5514 .and_then(Value::as_array)
5515 .and_then(|failures| failures.last())
5516 .or_else(|| description.failures.last());
5517 let exception = description
5518 .exception
5519 .clone()
5520 .or_else(|| failure.and_then(|value| value.get("exception")).cloned())
5521 .or_else(|| {
5522 nested_failure
5523 .and_then(|value| value.get("exception_payload"))
5524 .cloned()
5525 });
5526 let string_field = |name: &str| {
5527 failure
5528 .and_then(|value| value.get(name))
5529 .and_then(Value::as_str)
5530 .or_else(|| {
5531 nested_failure
5532 .and_then(|value| value.get(name))
5533 .and_then(Value::as_str)
5534 })
5535 .map(str::to_string)
5536 };
5537 let exception_field = |name: &str| {
5538 exception
5539 .as_ref()
5540 .and_then(|value| value.get(name))
5541 .and_then(Value::as_str)
5542 .map(str::to_string)
5543 };
5544 let message = description
5545 .error
5546 .clone()
5547 .or_else(|| string_field("message"))
5548 .or_else(|| exception_field("message"));
5549 let reason = description
5550 .raw
5551 .get("reason")
5552 .and_then(Value::as_str)
5553 .map(str::to_string)
5554 .or_else(|| {
5555 failure
5556 .and_then(|value| value.get("reason"))
5557 .and_then(Value::as_str)
5558 .map(str::to_string)
5559 })
5560 .or_else(|| description.closed_reason.clone())
5561 .unwrap_or_else(|| default_reason.to_string());
5562 let failure_id = string_field("failure_id").or_else(|| {
5563 nested_failure
5564 .and_then(|value| value.get("id"))
5565 .and_then(Value::as_str)
5566 .map(str::to_string)
5567 });
5568
5569 WorkflowTerminalOutcome {
5570 kind,
5571 workflow_id: description
5572 .workflow_id
5573 .clone()
5574 .unwrap_or_else(|| workflow_id.to_string()),
5575 run_id: description
5576 .run_id
5577 .clone()
5578 .or_else(|| run_id.map(str::to_string)),
5579 reason,
5580 failure_category: string_field("failure_category")
5581 .or_else(|| Some(default_reason.to_string())),
5582 failure_id,
5583 exception_type: string_field("exception_type").or_else(|| exception_field("type")),
5584 exception_class: string_field("exception_class").or_else(|| exception_field("class")),
5585 non_retryable: failure
5586 .and_then(|value| value.get("non_retryable"))
5587 .and_then(Value::as_bool)
5588 .or_else(|| {
5589 nested_failure
5590 .and_then(|value| value.get("non_retryable"))
5591 .and_then(Value::as_bool)
5592 }),
5593 message,
5594 exception,
5595 raw: description.raw_value(),
5596 }
5597}
5598
5599#[derive(Clone, Debug, Deserialize)]
5600pub struct RegisterWorkerResponse {
5601 pub worker_id: String,
5602 pub registered: bool,
5603 #[serde(default)]
5604 pub heartbeat_interval_seconds: Option<u64>,
5605 #[serde(default)]
5606 pub protocol_version: Option<String>,
5607 #[serde(default)]
5608 pub server_capabilities: Option<Value>,
5609}
5610
5611#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
5613pub struct WorkerDeregistrationEnvelope {
5614 pub worker_id: String,
5615 pub outcome: String,
5616 pub recovered_workflow_task_count: u64,
5617}
5618
5619#[derive(Clone, Debug, Deserialize)]
5620pub struct PollWorkflowTaskResponse {
5621 #[serde(default)]
5622 pub task: Option<WorkflowTask>,
5623 #[serde(default)]
5624 pub poll_status: Option<String>,
5625 #[serde(default)]
5626 pub reason: Option<String>,
5627 #[serde(default)]
5628 pub protocol_version: Option<String>,
5629 #[serde(default)]
5630 pub server_capabilities: Option<Value>,
5631}
5632
5633impl PollWorkflowTaskResponse {
5634 pub fn outcome(&self) -> WorkerPollOutcome {
5636 worker_poll_outcome(
5637 self.task.is_some(),
5638 self.poll_status.as_deref(),
5639 self.reason.as_deref(),
5640 )
5641 }
5642}
5643
5644fn runtime_supports_workflow_memo_updates(capabilities: Option<&Value>) -> bool {
5645 let Some(capabilities) = capabilities.and_then(Value::as_object) else {
5646 return false;
5647 };
5648 let supported = capabilities
5649 .get("workflow_memo_updates")
5650 .and_then(Value::as_object)
5651 .and_then(|memo| memo.get("supported"))
5652 .and_then(Value::as_bool)
5653 == Some(true);
5654 let command_advertised = capabilities
5655 .get("supported_workflow_task_commands")
5656 .and_then(Value::as_array)
5657 .is_some_and(|commands| {
5658 commands
5659 .iter()
5660 .any(|command| command.as_str() == Some("upsert_memo"))
5661 });
5662 supported && command_advertised
5663}
5664
5665fn commands_use_workflow_memo_updates(commands: &[Value]) -> bool {
5666 commands
5667 .iter()
5668 .any(|command| command.get("type").and_then(Value::as_str) == Some("upsert_memo"))
5669}
5670
5671#[derive(Clone, Debug, Deserialize)]
5672pub struct PollActivityTaskResponse {
5673 #[serde(default)]
5674 pub task: Option<ActivityTask>,
5675 #[serde(default)]
5676 pub poll_status: Option<String>,
5677 #[serde(default)]
5678 pub reason: Option<String>,
5679}
5680
5681impl PollActivityTaskResponse {
5682 pub fn outcome(&self) -> WorkerPollOutcome {
5684 worker_poll_outcome(
5685 self.task.is_some(),
5686 self.poll_status.as_deref(),
5687 self.reason.as_deref(),
5688 )
5689 }
5690}
5691
5692#[derive(Clone, Debug, Deserialize)]
5693pub struct PollQueryTaskResponse {
5694 #[serde(default)]
5695 pub task: Option<QueryTask>,
5696 #[serde(default)]
5697 pub poll_status: Option<String>,
5698 #[serde(default)]
5699 pub reason: Option<String>,
5700}
5701
5702impl PollQueryTaskResponse {
5703 pub fn outcome(&self) -> WorkerPollOutcome {
5705 worker_poll_outcome(
5706 self.task.is_some(),
5707 self.poll_status.as_deref(),
5708 self.reason.as_deref(),
5709 )
5710 }
5711}
5712
5713#[derive(Clone, Debug, PartialEq, Eq)]
5715pub enum WorkerPollOutcome {
5716 Task,
5718 Idle {
5720 poll_status: Option<String>,
5721 reason: Option<String>,
5722 },
5723 Stop {
5725 poll_status: Option<String>,
5726 reason: Option<String>,
5727 },
5728}
5729
5730impl WorkerPollOutcome {
5731 pub fn should_stop(&self) -> bool {
5732 matches!(self, Self::Stop { .. })
5733 }
5734}
5735
5736fn worker_poll_outcome(
5737 has_task: bool,
5738 poll_status: Option<&str>,
5739 reason: Option<&str>,
5740) -> WorkerPollOutcome {
5741 if worker_poll_is_stop(poll_status, reason) {
5742 return WorkerPollOutcome::Stop {
5743 poll_status: poll_status.map(str::to_string),
5744 reason: reason.map(str::to_string),
5745 };
5746 }
5747
5748 if has_task {
5749 WorkerPollOutcome::Task
5750 } else {
5751 WorkerPollOutcome::Idle {
5752 poll_status: poll_status.map(str::to_string),
5753 reason: reason.map(str::to_string),
5754 }
5755 }
5756}
5757
5758#[derive(Clone, Debug, Deserialize)]
5760pub struct QueryTask {
5761 pub query_task_id: String,
5762 #[serde(default = "default_workflow_task_attempt")]
5763 pub query_task_attempt: u64,
5764 #[serde(default)]
5765 pub lease_owner: Option<String>,
5766 #[serde(default)]
5767 pub workflow_id: Option<String>,
5768 #[serde(default)]
5769 pub run_id: Option<String>,
5770 pub workflow_type: String,
5771 pub query_name: String,
5772 #[serde(
5773 default = "missing_task_payload_codec",
5774 deserialize_with = "deserialize_task_payload_codec"
5775 )]
5776 pub payload_codec: String,
5777 #[serde(default)]
5778 pub workflow_arguments: Option<Value>,
5779 #[serde(default)]
5780 pub query_arguments: Option<Value>,
5781 #[serde(default)]
5782 pub history_events: Vec<HistoryEvent>,
5783 #[serde(default)]
5784 pub history_export: Option<Value>,
5785 #[serde(default)]
5786 pub run_status: Option<String>,
5787}
5788
5789#[derive(Clone, Debug, Deserialize)]
5790pub struct WorkflowTask {
5791 pub task_id: String,
5792 #[serde(default)]
5793 pub workflow_command_id: Option<String>,
5794 #[serde(default)]
5795 pub workflow_id: Option<String>,
5796 #[serde(default)]
5797 pub run_id: Option<String>,
5798 pub workflow_type: String,
5799 #[serde(default)]
5800 pub cancel_requested: bool,
5801 #[serde(
5802 default = "missing_task_payload_codec",
5803 deserialize_with = "deserialize_task_payload_codec"
5804 )]
5805 pub payload_codec: String,
5806 #[serde(default)]
5807 pub arguments: Option<Value>,
5808 #[serde(default)]
5809 pub history_events: Vec<HistoryEvent>,
5810 #[serde(default)]
5811 pub total_history_events: Option<u64>,
5812 #[serde(default)]
5813 pub history_size_bytes: Option<u64>,
5814 #[serde(default)]
5815 pub continue_as_new_recommended: Option<bool>,
5816 #[serde(default)]
5817 pub history_budget_pressure: Option<String>,
5818 #[serde(default)]
5819 pub next_history_page_token: Option<String>,
5820 #[serde(default = "default_workflow_task_attempt")]
5821 pub workflow_task_attempt: u64,
5822 #[serde(default)]
5823 pub workflow_signal_id: Option<String>,
5824 #[serde(default)]
5825 pub signal_name: Option<String>,
5826 #[serde(default)]
5827 pub signal_arguments: Option<Value>,
5828 #[serde(default)]
5829 pub workflow_update_id: Option<String>,
5830 #[serde(default)]
5831 pub update_name: Option<String>,
5832 #[serde(default)]
5833 pub lease_owner: Option<String>,
5834}
5835
5836impl WorkflowTask {
5837 fn append_history_page(&mut self, page: WorkflowTaskHistoryPage) {
5838 self.history_events.extend(page.history_events);
5839
5840 if page.total_history_events.is_some() {
5841 self.total_history_events = page.total_history_events;
5842 }
5843
5844 self.next_history_page_token = page
5845 .next_history_page_token
5846 .filter(|token| !token.is_empty());
5847 }
5848}
5849
5850#[derive(Clone, Debug, Deserialize)]
5851struct WorkflowTaskHistoryPage {
5852 #[serde(default)]
5853 history_events: Vec<HistoryEvent>,
5854 #[serde(default)]
5855 total_history_events: Option<u64>,
5856 #[serde(default)]
5857 next_history_page_token: Option<String>,
5858}
5859
5860#[derive(Clone, Debug, Deserialize)]
5861pub struct ActivityTask {
5862 pub task_id: String,
5863 #[serde(default)]
5864 pub activity_attempt_id: Option<String>,
5865 #[serde(default)]
5866 pub attempt_id: Option<String>,
5867 pub activity_type: String,
5868 #[serde(
5869 default = "missing_task_payload_codec",
5870 deserialize_with = "deserialize_task_payload_codec"
5871 )]
5872 pub payload_codec: String,
5873 #[serde(default)]
5874 pub arguments: Option<Value>,
5875 #[serde(default = "default_attempt_number")]
5876 pub attempt_number: u64,
5877 #[serde(default)]
5878 pub lease_owner: Option<String>,
5879}
5880
5881#[derive(Clone, Debug, Deserialize)]
5882pub struct HistoryEvent {
5883 #[serde(alias = "type")]
5884 pub event_type: String,
5885 #[serde(default)]
5886 pub payload: Value,
5887 #[serde(flatten)]
5888 pub raw: HashMap<String, Value>,
5889}
5890
5891#[derive(Clone, Debug, PartialEq)]
5893pub struct QuerySignal {
5894 pub id: Option<String>,
5895 pub name: String,
5896 pub arguments: Vec<Value>,
5897 avro_arguments: Vec<AvroValue>,
5898 pub workflow_sequence: Option<u64>,
5899}
5900
5901impl QuerySignal {
5902 pub fn arguments_avro_value(&self) -> &[AvroValue] {
5904 &self.avro_arguments
5905 }
5906}
5907
5908#[derive(Clone, Debug)]
5914pub struct QueryContext {
5915 pub workflow_id: Option<String>,
5916 pub run_id: Option<String>,
5917 pub workflow_type: String,
5918 pub run_status: Option<String>,
5919 workflow_input: Value,
5920 workflow_input_avro_value: AvroValue,
5921 history_events: Arc<Vec<HistoryEvent>>,
5922 signal_events: Arc<Vec<QuerySignal>>,
5923}
5924
5925impl QueryContext {
5926 pub fn workflow_input(&self) -> &Value {
5928 &self.workflow_input
5929 }
5930
5931 pub fn workflow_input_avro_value(&self) -> &AvroValue {
5933 &self.workflow_input_avro_value
5934 }
5935
5936 pub fn history_events(&self) -> &[HistoryEvent] {
5938 self.history_events.as_slice()
5939 }
5940
5941 pub fn signal_events(&self) -> &[QuerySignal] {
5943 self.signal_events.as_slice()
5944 }
5945
5946 pub fn signals(&self, signal_name: &str) -> Vec<Vec<Value>> {
5948 self.signal_events
5949 .iter()
5950 .filter(|signal| signal.name == signal_name)
5951 .map(|signal| signal.arguments.clone())
5952 .collect()
5953 }
5954
5955 pub fn signals_avro_value(&self, signal_name: &str) -> Vec<Vec<AvroValue>> {
5957 self.signal_events
5958 .iter()
5959 .filter(|signal| signal.name == signal_name)
5960 .map(|signal| signal.avro_arguments.clone())
5961 .collect()
5962 }
5963}
5964
5965#[derive(Clone, Debug, Deserialize)]
5966pub struct ActivityHeartbeatResponse {
5967 #[serde(default)]
5968 pub cancel_requested: bool,
5969 #[serde(default)]
5970 pub heartbeat_recorded: bool,
5971 #[serde(default)]
5972 pub can_continue: Option<bool>,
5973 #[serde(default)]
5974 pub reason: Option<String>,
5975 #[serde(default)]
5976 pub run_closed_reason: Option<String>,
5977 #[serde(default)]
5978 pub run_closed_at: Option<String>,
5979 #[serde(default)]
5980 pub lease_expires_at: Option<String>,
5981 #[serde(default)]
5982 pub last_heartbeat_at: Option<String>,
5983}
5984
5985impl ActivityHeartbeatResponse {
5986 pub fn should_stop(&self) -> bool {
5988 self.cancel_requested || self.can_continue == Some(false)
5989 }
5990}
5991
5992fn missing_task_payload_codec() -> String {
5993 MISSING_TASK_PAYLOAD_CODEC.to_string()
5994}
5995
5996fn deserialize_task_payload_codec<'de, D>(deserializer: D) -> std::result::Result<String, D::Error>
5997where
5998 D: Deserializer<'de>,
5999{
6000 Ok(match Value::deserialize(deserializer)? {
6001 Value::String(codec) => codec,
6002 Value::Null => NULL_TASK_PAYLOAD_CODEC.to_string(),
6003 _ => NON_STRING_TASK_PAYLOAD_CODEC.to_string(),
6004 })
6005}
6006
6007fn default_workflow_task_attempt() -> u64 {
6008 1
6009}
6010
6011fn default_attempt_number() -> u64 {
6012 1
6013}
6014
6015type WorkflowFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
6016type WorkflowHandler = Arc<dyn Fn(WorkflowContext, AvroValue) -> WorkflowFuture + Send + Sync>;
6017type ErasedWorkflowState = Arc<dyn Any + Send + Sync>;
6018type WorkflowStateSnapshot = Arc<dyn Fn() -> Result<ErasedWorkflowState> + Send + Sync>;
6019type ReplayedWorkflowHandler =
6020 Arc<dyn Fn(WorkflowContext, AvroValue) -> ReplayedWorkflowInvocation + Send + Sync>;
6021type ActivityFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
6022type ActivityHandler = Arc<dyn Fn(ActivityContext, AvroValue) -> ActivityFuture + Send + Sync>;
6023type QueryFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
6024type QueryHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
6025type UpdateHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
6026type ReplayedQueryHandler = Arc<
6027 dyn Fn(QueryContext, ErasedWorkflowState, AvroValue) -> std::result::Result<QueryFuture, String>
6028 + Send
6029 + Sync,
6030>;
6031type WorkerHeartbeatObserver = Arc<dyn Fn(&WorkerHeartbeatObservation) + Send + Sync>;
6032
6033struct ReplayedWorkflowInvocation {
6034 future: WorkflowFuture,
6035 snapshot: WorkflowStateSnapshot,
6036}
6037
6038#[derive(Clone)]
6039struct RegisteredWorkflow {
6040 execute: WorkflowHandler,
6041 replay: Option<ReplayedWorkflowHandler>,
6042 state_type: Option<TypeId>,
6043 definition_fingerprint: Option<String>,
6044}
6045
6046#[derive(Debug)]
6047struct WorkflowTaskDecision {
6048 commands: Vec<Value>,
6049 message_stream_cursors: Vec<Value>,
6050 message_stream_waits: Vec<Value>,
6051}
6052
6053impl WorkflowTaskDecision {
6054 fn without_message_streams(commands: Vec<Value>) -> Self {
6055 Self {
6056 commands,
6057 message_stream_cursors: Vec::new(),
6058 message_stream_waits: Vec::new(),
6059 }
6060 }
6061}
6062
6063#[derive(Clone)]
6064enum RegisteredQuery {
6065 Snapshot(QueryHandler),
6066 Replayed {
6067 state_type: TypeId,
6068 handler: ReplayedQueryHandler,
6069 },
6070}
6071
6072#[derive(Clone, Debug)]
6073pub struct WorkerHeartbeatObservation {
6074 pub worker_id: String,
6075 pub task_queue: String,
6076 pub acknowledged_at_unix_millis: u64,
6077 pub acknowledgement: Value,
6078}
6079
6080#[derive(Clone, Copy, Debug)]
6092pub struct WorkerRetryPolicy {
6093 pub max_retries: usize,
6095 pub initial_backoff: Duration,
6097 pub max_backoff: Duration,
6099}
6100
6101impl Default for WorkerRetryPolicy {
6102 fn default() -> Self {
6103 Self {
6104 max_retries: 5,
6105 initial_backoff: Duration::from_millis(100),
6106 max_backoff: Duration::from_secs(5),
6107 }
6108 }
6109}
6110
6111#[derive(Clone, Debug)]
6112struct WorkerStorageAdmission {
6113 policy: WorkerRetryPolicy,
6114 stop: Arc<AtomicBool>,
6115}
6116
6117struct StopWorkerOnDrop(Arc<AtomicBool>);
6118
6119impl Drop for StopWorkerOnDrop {
6120 fn drop(&mut self) {
6121 self.0.store(true, Ordering::SeqCst);
6122 }
6123}
6124
6125async fn wait_for_worker_stop(stop: &AtomicBool) {
6126 while !stop.load(Ordering::SeqCst) {
6127 tokio::time::sleep(Duration::from_millis(100)).await;
6128 }
6129}
6130
6131#[derive(Clone, Copy, Debug, PartialEq, Eq)]
6132enum ManagedPollOutcome {
6133 Idle,
6134 Handled,
6135 Stop,
6136}
6137
6138#[derive(Clone)]
6139pub struct Worker {
6140 client: Client,
6141 worker_id: String,
6142 task_queue: String,
6143 workflows: HashMap<String, RegisteredWorkflow>,
6144 activities: HashMap<String, ActivityHandler>,
6145 queries: HashMap<String, HashMap<String, RegisteredQuery>>,
6146 updates: HashMap<String, HashMap<String, UpdateHandler>>,
6147 max_concurrent_workflow_tasks: usize,
6148 max_concurrent_activity_tasks: usize,
6149 poll_timeout: Duration,
6150 heartbeat_interval: Duration,
6151 retry_policy: WorkerRetryPolicy,
6152 heartbeat_observer: Option<WorkerHeartbeatObserver>,
6153}
6154
6155impl Worker {
6156 pub fn new(client: Client, task_queue: impl Into<String>) -> Self {
6157 Self {
6158 client,
6159 worker_id: default_worker_id(),
6160 task_queue: task_queue.into(),
6161 workflows: HashMap::new(),
6162 activities: HashMap::new(),
6163 queries: HashMap::new(),
6164 updates: HashMap::new(),
6165 max_concurrent_workflow_tasks: 10,
6166 max_concurrent_activity_tasks: 10,
6167 poll_timeout: Duration::from_secs(30),
6168 heartbeat_interval: Duration::from_secs(60),
6169 retry_policy: WorkerRetryPolicy::default(),
6170 heartbeat_observer: None,
6171 }
6172 }
6173
6174 pub fn worker_id(mut self, worker_id: impl Into<String>) -> Self {
6175 self.worker_id = worker_id.into();
6176 self
6177 }
6178
6179 pub fn poll_timeout(mut self, timeout: Duration) -> Self {
6180 self.poll_timeout = timeout;
6181 self
6182 }
6183
6184 pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
6185 self.heartbeat_interval = interval;
6186 self
6187 }
6188
6189 pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self {
6191 self.retry_policy = policy;
6192 self
6193 }
6194
6195 pub fn on_worker_heartbeat<F>(mut self, observer: F) -> Self
6196 where
6197 F: Fn(&WorkerHeartbeatObservation) + Send + Sync + 'static,
6198 {
6199 self.heartbeat_observer = Some(Arc::new(observer));
6200 self
6201 }
6202
6203 pub fn max_concurrent_workflow_tasks(mut self, count: usize) -> Self {
6204 self.max_concurrent_workflow_tasks = count.max(1);
6205 self
6206 }
6207
6208 pub fn max_concurrent_activity_tasks(mut self, count: usize) -> Self {
6209 self.max_concurrent_activity_tasks = count.max(1);
6210 self
6211 }
6212
6213 pub fn register_workflow<F, Fut>(&mut self, workflow_type: impl Into<String>, handler: F)
6220 where
6221 F: Fn(WorkflowContext, Value) -> Fut + Send + Sync + 'static,
6222 Fut: Future<Output = Result<Value>> + Send + 'static,
6223 {
6224 let handler = Arc::new(handler);
6225 self.workflows.insert(
6226 workflow_type.into(),
6227 RegisteredWorkflow {
6228 execute: Arc::new(move |ctx, input| {
6229 let handler = Arc::clone(&handler);
6230 Box::pin(async move {
6231 let result = handler(ctx, input.into_json()?).await?;
6232 AvroValue::from_serialize(&result)
6233 })
6234 }),
6235 replay: None,
6236 state_type: None,
6237 definition_fingerprint: None,
6238 },
6239 );
6240 }
6241
6242 pub fn register_typed_workflow<I, O, F, Fut>(
6253 &mut self,
6254 workflow_type: impl Into<String>,
6255 handler: F,
6256 ) where
6257 I: DeserializeOwned + Send + 'static,
6258 O: Serialize + Send + 'static,
6259 F: Fn(WorkflowContext, I) -> Fut + Send + Sync + 'static,
6260 Fut: Future<Output = Result<O>> + Send + 'static,
6261 {
6262 let workflow_type = workflow_type.into();
6263 let handler_name = workflow_type.clone();
6264 let handler = Arc::new(handler);
6265 self.workflows.insert(
6266 workflow_type,
6267 RegisteredWorkflow {
6268 execute: Arc::new(move |ctx, input| {
6269 let handler = Arc::clone(&handler);
6270 let handler_name = handler_name.clone();
6271 Box::pin(async move {
6272 let input =
6273 decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
6274 let result = handler(ctx, input).await?;
6275 encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
6276 })
6277 }),
6278 replay: None,
6279 state_type: None,
6280 definition_fingerprint: None,
6281 },
6282 );
6283 }
6284
6285 pub fn register_workflow_avro_value<F, Fut>(
6287 &mut self,
6288 workflow_type: impl Into<String>,
6289 handler: F,
6290 ) where
6291 F: Fn(WorkflowContext, AvroValue) -> Fut + Send + Sync + 'static,
6292 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6293 {
6294 self.workflows.insert(
6295 workflow_type.into(),
6296 RegisteredWorkflow {
6297 execute: Arc::new(move |ctx, input| Box::pin(handler(ctx, input))),
6298 replay: None,
6299 state_type: None,
6300 definition_fingerprint: None,
6301 },
6302 );
6303 }
6304
6305 pub fn register_replayed_workflow<S, Factory, F, Fut>(
6313 &mut self,
6314 workflow_type: impl Into<String>,
6315 state_factory: Factory,
6316 handler: F,
6317 ) where
6318 S: Clone + Send + Sync + 'static,
6319 Factory: Fn() -> S + Send + Sync + 'static,
6320 F: Fn(WorkflowContext, Value, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
6321 Fut: Future<Output = Result<Value>> + Send + 'static,
6322 {
6323 let state_factory = Arc::new(state_factory);
6324 let handler = Arc::new(handler);
6325
6326 let execute_factory = Arc::clone(&state_factory);
6327 let execute_handler = Arc::clone(&handler);
6328 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6329 let state = WorkflowInstance::new(execute_factory());
6330 let handler = Arc::clone(&execute_handler);
6331 Box::pin(async move {
6332 let result = handler(ctx, input.into_json()?, state).await?;
6333 AvroValue::from_serialize(&result)
6334 }) as WorkflowFuture
6335 });
6336
6337 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6338 let state = WorkflowInstance::new(state_factory());
6339 let snapshot_state = state.clone();
6340 let snapshot: WorkflowStateSnapshot =
6341 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6342 let replay_handler = Arc::clone(&handler);
6343 let future = async move {
6344 let result = replay_handler(ctx, input.into_json()?, state).await?;
6345 AvroValue::from_serialize(&result)
6346 };
6347 ReplayedWorkflowInvocation {
6348 future: Box::pin(future),
6349 snapshot,
6350 }
6351 });
6352
6353 self.workflows.insert(
6354 workflow_type.into(),
6355 RegisteredWorkflow {
6356 execute,
6357 replay: Some(replay),
6358 state_type: Some(TypeId::of::<S>()),
6359 definition_fingerprint: None,
6360 },
6361 );
6362 }
6363
6364 pub fn register_typed_replayed_workflow<I, O, S, Factory, F, Fut>(
6370 &mut self,
6371 workflow_type: impl Into<String>,
6372 state_factory: Factory,
6373 handler: F,
6374 ) where
6375 I: DeserializeOwned + Send + 'static,
6376 O: Serialize + Send + 'static,
6377 S: Clone + Send + Sync + 'static,
6378 Factory: Fn() -> S + Send + Sync + 'static,
6379 F: Fn(WorkflowContext, I, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
6380 Fut: Future<Output = Result<O>> + Send + 'static,
6381 {
6382 let workflow_type = workflow_type.into();
6383 let state_factory = Arc::new(state_factory);
6384 let handler = Arc::new(handler);
6385
6386 let execute_name = workflow_type.clone();
6387 let execute_factory = Arc::clone(&state_factory);
6388 let execute_handler = Arc::clone(&handler);
6389 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6390 let state = WorkflowInstance::new(execute_factory());
6391 let handler = Arc::clone(&execute_handler);
6392 let handler_name = execute_name.clone();
6393 Box::pin(async move {
6394 let input = decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
6395 let result = handler(ctx, input, state).await?;
6396 encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
6397 }) as WorkflowFuture
6398 });
6399
6400 let replay_name = workflow_type.clone();
6401 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6402 let state = WorkflowInstance::new(state_factory());
6403 let snapshot_state = state.clone();
6404 let snapshot: WorkflowStateSnapshot =
6405 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6406 let handler = Arc::clone(&handler);
6407 let handler_name = replay_name.clone();
6408 let future = async move {
6409 let input = decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
6410 let result = handler(ctx, input, state).await?;
6411 encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
6412 };
6413 ReplayedWorkflowInvocation {
6414 future: Box::pin(future),
6415 snapshot,
6416 }
6417 });
6418
6419 self.workflows.insert(
6420 workflow_type,
6421 RegisteredWorkflow {
6422 execute,
6423 replay: Some(replay),
6424 state_type: Some(TypeId::of::<S>()),
6425 definition_fingerprint: None,
6426 },
6427 );
6428 }
6429
6430 pub fn set_workflow_definition_sources(
6440 &mut self,
6441 workflow_type: &str,
6442 sources: &[&str],
6443 ) -> Result<()> {
6444 if sources.is_empty() || sources.iter().any(|source| source.is_empty()) {
6445 return Err(Error::Codec(
6446 "workflow definition sources must be non-empty".to_string(),
6447 ));
6448 }
6449 let workflow = self.workflows.get_mut(workflow_type).ok_or_else(|| {
6450 Error::Codec(format!("workflow type {workflow_type:?} is not registered"))
6451 })?;
6452 let mut hasher = Sha256::new();
6453 hasher.update(b"durable-workflow-rust.workflow-definition.v1\0");
6454 hasher.update((workflow_type.len() as u64).to_be_bytes());
6455 hasher.update(workflow_type.as_bytes());
6456 for source in sources {
6457 hasher.update((source.len() as u64).to_be_bytes());
6458 hasher.update(source.as_bytes());
6459 }
6460 workflow.definition_fingerprint = Some(format!("sha256:{:x}", hasher.finalize()));
6461 Ok(())
6462 }
6463
6464 pub fn register_replayed_workflow_avro_value<S, Factory, F, Fut>(
6466 &mut self,
6467 workflow_type: impl Into<String>,
6468 state_factory: Factory,
6469 handler: F,
6470 ) where
6471 S: Clone + Send + Sync + 'static,
6472 Factory: Fn() -> S + Send + Sync + 'static,
6473 F: Fn(WorkflowContext, AvroValue, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
6474 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6475 {
6476 let state_factory = Arc::new(state_factory);
6477 let handler = Arc::new(handler);
6478
6479 let execute_factory = Arc::clone(&state_factory);
6480 let execute_handler = Arc::clone(&handler);
6481 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6482 let state = WorkflowInstance::new(execute_factory());
6483 Box::pin(execute_handler(ctx, input, state)) as WorkflowFuture
6484 });
6485
6486 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6487 let state = WorkflowInstance::new(state_factory());
6488 let snapshot_state = state.clone();
6489 let snapshot: WorkflowStateSnapshot =
6490 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6491 ReplayedWorkflowInvocation {
6492 future: Box::pin(handler(ctx, input, state)),
6493 snapshot,
6494 }
6495 });
6496
6497 self.workflows.insert(
6498 workflow_type.into(),
6499 RegisteredWorkflow {
6500 execute,
6501 replay: Some(replay),
6502 state_type: Some(TypeId::of::<S>()),
6503 definition_fingerprint: None,
6504 },
6505 );
6506 }
6507
6508 pub fn register_activity<F, Fut>(&mut self, activity_type: impl Into<String>, handler: F)
6509 where
6510 F: Fn(ActivityContext, Value) -> Fut + Send + Sync + 'static,
6511 Fut: Future<Output = Result<Value>> + Send + 'static,
6512 {
6513 let handler = Arc::new(handler);
6514 self.activities.insert(
6515 activity_type.into(),
6516 Arc::new(move |ctx, args| {
6517 let handler = Arc::clone(&handler);
6518 Box::pin(async move {
6519 let result = handler(ctx, args.into_json()?).await?;
6520 AvroValue::from_serialize(&result)
6521 })
6522 }),
6523 );
6524 }
6525
6526 pub fn register_typed_activity<I, O, F, Fut>(
6532 &mut self,
6533 activity_type: impl Into<String>,
6534 handler: F,
6535 ) where
6536 I: DeserializeOwned + Send + 'static,
6537 O: Serialize + Send + 'static,
6538 F: Fn(ActivityContext, I) -> Fut + Send + Sync + 'static,
6539 Fut: Future<Output = Result<O>> + Send + 'static,
6540 {
6541 let activity_type = activity_type.into();
6542 let handler_name = activity_type.clone();
6543 let handler = Arc::new(handler);
6544 self.activities.insert(
6545 activity_type,
6546 Arc::new(move |ctx, input| {
6547 let handler = Arc::clone(&handler);
6548 let handler_name = handler_name.clone();
6549 Box::pin(async move {
6550 let input =
6551 decode_handler_input::<I>(input, HandlerKind::Activity, &handler_name)?;
6552 let result = handler(ctx, input).await?;
6553 encode_handler_result(&result, HandlerKind::Activity, &handler_name)
6554 })
6555 }),
6556 );
6557 }
6558
6559 pub fn register_activity_avro_value<F, Fut>(
6561 &mut self,
6562 activity_type: impl Into<String>,
6563 handler: F,
6564 ) where
6565 F: Fn(ActivityContext, AvroValue) -> Fut + Send + Sync + 'static,
6566 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6567 {
6568 self.activities.insert(
6569 activity_type.into(),
6570 Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
6571 );
6572 }
6573
6574 pub fn register_query<F, Fut>(
6580 &mut self,
6581 workflow_type: impl Into<String>,
6582 query_name: impl Into<String>,
6583 handler: F,
6584 ) where
6585 F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
6586 Fut: Future<Output = Result<Value>> + Send + 'static,
6587 {
6588 let handler = Arc::new(handler);
6589 self.queries
6590 .entry(workflow_type.into())
6591 .or_default()
6592 .insert(
6593 query_name.into(),
6594 RegisteredQuery::Snapshot(Arc::new(move |ctx, args| {
6595 let handler = Arc::clone(&handler);
6596 Box::pin(async move {
6597 let result = handler(ctx, args.into_json()?).await?;
6598 AvroValue::from_serialize(&result)
6599 })
6600 })),
6601 );
6602 }
6603
6604 pub fn register_query_avro_value<F, Fut>(
6606 &mut self,
6607 workflow_type: impl Into<String>,
6608 query_name: impl Into<String>,
6609 handler: F,
6610 ) where
6611 F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
6612 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6613 {
6614 self.queries
6615 .entry(workflow_type.into())
6616 .or_default()
6617 .insert(
6618 query_name.into(),
6619 RegisteredQuery::Snapshot(Arc::new(move |ctx, args| Box::pin(handler(ctx, args)))),
6620 );
6621 }
6622
6623 pub fn register_replayed_query<S, F, Fut>(
6630 &mut self,
6631 workflow_type: impl Into<String>,
6632 query_name: impl Into<String>,
6633 handler: F,
6634 ) where
6635 S: Clone + Send + Sync + 'static,
6636 F: Fn(QueryContext, Arc<S>, Value) -> Fut + Send + Sync + 'static,
6637 Fut: Future<Output = Result<Value>> + Send + 'static,
6638 {
6639 let handler = Arc::new(handler);
6640 let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
6641 let state = state.downcast::<S>().map_err(|_| {
6642 "registered query state type does not match the replayed workflow state".to_string()
6643 })?;
6644 let handler = Arc::clone(&handler);
6645 Ok(Box::pin(async move {
6646 let result = handler(ctx, state, args.into_json()?).await?;
6647 AvroValue::from_serialize(&result)
6648 }))
6649 });
6650
6651 self.queries
6652 .entry(workflow_type.into())
6653 .or_default()
6654 .insert(
6655 query_name.into(),
6656 RegisteredQuery::Replayed {
6657 state_type: TypeId::of::<S>(),
6658 handler: erased_handler,
6659 },
6660 );
6661 }
6662
6663 pub fn register_replayed_query_avro_value<S, F, Fut>(
6665 &mut self,
6666 workflow_type: impl Into<String>,
6667 query_name: impl Into<String>,
6668 handler: F,
6669 ) where
6670 S: Clone + Send + Sync + 'static,
6671 F: Fn(QueryContext, Arc<S>, AvroValue) -> Fut + Send + Sync + 'static,
6672 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6673 {
6674 let handler = Arc::new(handler);
6675 let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
6676 let state = state.downcast::<S>().map_err(|_| {
6677 "registered query state type does not match the replayed workflow state".to_string()
6678 })?;
6679 Ok(Box::pin(handler(ctx, state, args)))
6680 });
6681
6682 self.queries
6683 .entry(workflow_type.into())
6684 .or_default()
6685 .insert(
6686 query_name.into(),
6687 RegisteredQuery::Replayed {
6688 state_type: TypeId::of::<S>(),
6689 handler: erased_handler,
6690 },
6691 );
6692 }
6693
6694 pub fn register_update<F, Fut>(
6696 &mut self,
6697 workflow_type: impl Into<String>,
6698 update_name: impl Into<String>,
6699 handler: F,
6700 ) where
6701 F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
6702 Fut: Future<Output = Result<Value>> + Send + 'static,
6703 {
6704 let handler = Arc::new(handler);
6705 self.updates
6706 .entry(workflow_type.into())
6707 .or_default()
6708 .insert(
6709 update_name.into(),
6710 Arc::new(move |ctx, args| {
6711 let handler = Arc::clone(&handler);
6712 Box::pin(async move {
6713 let result = handler(ctx, args.into_json()?).await?;
6714 AvroValue::from_serialize(&result)
6715 })
6716 }),
6717 );
6718 }
6719
6720 pub fn register_update_avro_value<F, Fut>(
6722 &mut self,
6723 workflow_type: impl Into<String>,
6724 update_name: impl Into<String>,
6725 handler: F,
6726 ) where
6727 F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
6728 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6729 {
6730 self.updates
6731 .entry(workflow_type.into())
6732 .or_default()
6733 .insert(
6734 update_name.into(),
6735 Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
6736 );
6737 }
6738
6739 pub async fn register(&self) -> Result<RegisterWorkerResponse> {
6740 let mut command_contracts = serde_json::Map::new();
6741 for workflow_type in self.workflows.keys() {
6742 let mut queries = self
6743 .queries
6744 .get(workflow_type)
6745 .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
6746 .unwrap_or_default();
6747 queries.sort();
6748 let mut updates = self
6749 .updates
6750 .get(workflow_type)
6751 .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
6752 .unwrap_or_default();
6753 updates.sort();
6754 command_contracts.insert(
6755 workflow_type.clone(),
6756 json!({
6757 "queries": queries,
6758 "query_contracts": [],
6759 "signals": [],
6760 "signal_contracts": [],
6761 "updates": updates,
6762 "update_contracts": [],
6763 "update_validators": [],
6764 }),
6765 );
6766 }
6767
6768 self.client
6769 .register_worker_with_definition_fingerprints(
6770 &self.worker_id,
6771 &self.task_queue,
6772 self.workflows.keys().cloned().collect(),
6773 self.activities.keys().cloned().collect(),
6774 self.max_concurrent_workflow_tasks,
6775 self.max_concurrent_activity_tasks,
6776 [
6777 Some(CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY.to_string()),
6778 Some(DURABLE_SELECTION_CAPABILITY.to_string()),
6779 Some(MEMO_UPSERTS_CAPABILITY.to_string()),
6780 Some(TYPED_SEARCH_ATTRIBUTES_CAPABILITY.to_string()),
6781 (!self.queries.is_empty()).then(|| QUERY_TASKS_CAPABILITY.to_string()),
6782 (!self.updates.is_empty()).then(|| WORKFLOW_UPDATES_CAPABILITY.to_string()),
6783 worker_protocol_supports_message_streams(WORKER_PROTOCOL_VERSION)
6784 .then(|| MESSAGE_STREAMS_CAPABILITY.to_string()),
6785 ]
6786 .into_iter()
6787 .flatten()
6788 .collect(),
6789 Value::Object(command_contracts),
6790 Some(
6791 self.workflows
6792 .iter()
6793 .filter_map(|(workflow_type, workflow)| {
6794 workflow
6795 .definition_fingerprint
6796 .as_ref()
6797 .map(|fingerprint| (workflow_type.clone(), fingerprint.clone()))
6798 })
6799 .collect(),
6800 ),
6801 )
6802 .await
6803 }
6804
6805 pub async fn run(&self) -> Result<()> {
6811 self.run_until(std::future::pending::<()>()).await
6812 }
6813
6814 pub async fn run_until<F>(&self, shutdown: F) -> Result<()>
6818 where
6819 F: Future<Output = ()>,
6820 {
6821 let stop = Arc::new(AtomicBool::new(false));
6822 let _stop_on_drop = StopWorkerOnDrop(Arc::clone(&stop));
6823 let worker = self.with_storage_admission(Arc::clone(&stop));
6824 let run = worker.run_with_storage_admission(Arc::clone(&stop));
6825 tokio::pin!(run);
6826 tokio::pin!(shutdown);
6827 tokio::select! {
6828 result = &mut run => result,
6829 _ = &mut shutdown => {
6830 stop.store(true, Ordering::SeqCst);
6831 run.await
6832 }
6833 }
6834 }
6835
6836 fn with_storage_admission(&self, stop: Arc<AtomicBool>) -> Self {
6837 let mut worker = self.clone();
6838 worker.client.worker_storage_admission = Some(WorkerStorageAdmission {
6839 policy: self.retry_policy,
6840 stop,
6841 });
6842 worker
6843 }
6844
6845 async fn run_with_storage_admission(&self, stop: Arc<AtomicBool>) -> Result<()> {
6846 let registration = self.register().await?;
6847 if !registration.registered {
6848 return Err(Error::WorkerLoop(format!(
6849 "worker registration for {:?} was not accepted",
6850 self.worker_id
6851 )));
6852 }
6853 let registered_worker_id = registration.worker_id.clone();
6854 let primary = self.run_registered_until(stop, registration).await;
6855 let deregistration = self
6856 .client
6857 .deregister_worker_registration(®istered_worker_id)
6858 .await;
6859
6860 match (primary, deregistration) {
6861 (Ok(()), Ok(_)) => Ok(()),
6862 (Ok(()), Err(deregistration)) => Err(deregistration),
6863 (Err(primary), Ok(_)) => Err(primary),
6864 (Err(primary), Err(deregistration)) => Err(Error::WorkerShutdown {
6865 primary: Box::new(primary),
6866 deregistration: Box::new(deregistration),
6867 }),
6868 }
6869 }
6870
6871 async fn run_registered_until(
6872 &self,
6873 stop: Arc<AtomicBool>,
6874 registration: RegisterWorkerResponse,
6875 ) -> Result<()> {
6876 let heartbeat_interval = Duration::from_secs(
6877 registration
6878 .heartbeat_interval_seconds
6879 .unwrap_or(self.heartbeat_interval.as_secs().max(1)),
6880 );
6881 let heartbeat = tokio::time::sleep(Duration::ZERO);
6887 tokio::pin!(heartbeat);
6888 let mut workflow_poller = (!self.workflows.is_empty()).then(|| {
6892 let worker = self.clone();
6893 let stop = Arc::clone(&stop);
6894 tokio::spawn(async move { worker.poll_workflows_until_stopped(stop).await })
6895 });
6896 let mut activity_poller = (!self.activities.is_empty()).then(|| {
6897 let worker = self.clone();
6898 let stop = Arc::clone(&stop);
6899 tokio::spawn(async move { worker.poll_activities_until_stopped(stop).await })
6900 });
6901 let mut query_poller = (!self.queries.is_empty()).then(|| {
6902 let worker = self.clone();
6903 let stop = Arc::clone(&stop);
6904 tokio::spawn(async move { worker.poll_queries_until_stopped(stop).await })
6905 });
6906
6907 loop {
6908 tokio::select! {
6909 _ = wait_for_worker_stop(&stop) => {
6910 stop.store(true, Ordering::SeqCst);
6911 break;
6912 }
6913 _ = &mut heartbeat => {
6914 let result = self.retry_worker_operation(|| {
6915 self.client.heartbeat_worker(
6916 &self.worker_id,
6917 self.max_concurrent_workflow_tasks,
6918 self.max_concurrent_activity_tasks,
6919 )
6920 }).await;
6921 heartbeat
6922 .as_mut()
6923 .reset(tokio::time::Instant::now() + heartbeat_interval);
6924 match result {
6925 Ok(acknowledgement) => {
6926 if let Some(observer) = &self.heartbeat_observer {
6927 observer(&WorkerHeartbeatObservation {
6928 worker_id: self.worker_id.clone(),
6929 task_queue: self.task_queue.clone(),
6930 acknowledged_at_unix_millis: SystemTime::now()
6931 .duration_since(UNIX_EPOCH)
6932 .unwrap_or_default()
6933 .as_millis()
6934 .min(u64::MAX as u128)
6935 as u64,
6936 acknowledgement,
6937 });
6938 }
6939 }
6940 Err(error) => {
6941 stop.store(true, Ordering::SeqCst);
6942 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await?;
6943 return Err(error);
6944 }
6945 }
6946 }
6947 result = OptionFuture::from(workflow_poller.as_mut()), if workflow_poller.is_some() => {
6948 workflow_poller = None;
6949 let stopped_by_server = stop.load(Ordering::SeqCst);
6950 stop.store(true, Ordering::SeqCst);
6951 let poller_result = optional_poller_result("workflow", result);
6952 let join_result =
6953 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6954 poller_result?;
6955 join_result?;
6956 if stopped_by_server {
6957 return Ok(());
6958 }
6959 return Err(Error::WorkerLoop(
6960 "workflow poller stopped unexpectedly".to_string(),
6961 ));
6962 }
6963 result = OptionFuture::from(activity_poller.as_mut()), if activity_poller.is_some() => {
6964 activity_poller = None;
6965 let stopped_by_server = stop.load(Ordering::SeqCst);
6966 stop.store(true, Ordering::SeqCst);
6967 let poller_result = optional_poller_result("activity", result);
6968 let join_result =
6969 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6970 poller_result?;
6971 join_result?;
6972 if stopped_by_server {
6973 return Ok(());
6974 }
6975 return Err(Error::WorkerLoop(
6976 "activity poller stopped unexpectedly".to_string(),
6977 ));
6978 }
6979 result = OptionFuture::from(query_poller.as_mut()), if query_poller.is_some() => {
6980 query_poller = None;
6981 let stopped_by_server = stop.load(Ordering::SeqCst);
6982 stop.store(true, Ordering::SeqCst);
6983 let poller_result = optional_poller_result("query", result);
6984 let join_result =
6985 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6986 poller_result?;
6987 join_result?;
6988 if stopped_by_server {
6989 return Ok(());
6990 }
6991 return Err(Error::WorkerLoop(
6992 "query poller stopped unexpectedly".to_string(),
6993 ));
6994 }
6995 }
6996 }
6997
6998 join_pollers(
6999 workflow_poller.take(),
7000 activity_poller.take(),
7001 query_poller.take(),
7002 )
7003 .await
7004 }
7005
7006 pub async fn run_once(&self) -> Result<usize> {
7020 let worker = self.with_storage_admission(Arc::new(AtomicBool::new(false)));
7021 let mut handled = 0;
7022 match worker.poll_workflow_once().await? {
7023 ManagedPollOutcome::Handled => handled += 1,
7024 ManagedPollOutcome::Stop => return Ok(handled),
7025 ManagedPollOutcome::Idle => {}
7026 }
7027 match worker.poll_activity_once().await? {
7028 ManagedPollOutcome::Handled => handled += 1,
7029 ManagedPollOutcome::Stop => return Ok(handled),
7030 ManagedPollOutcome::Idle => {}
7031 }
7032 if !self.queries.is_empty() {
7033 match worker.poll_query_once().await? {
7034 ManagedPollOutcome::Handled => handled += 1,
7035 ManagedPollOutcome::Stop => return Ok(handled),
7036 ManagedPollOutcome::Idle => {}
7037 }
7038 }
7039 Ok(handled)
7040 }
7041
7042 async fn poll_workflow_once(&self) -> Result<ManagedPollOutcome> {
7043 let poll_request_id = unique_request_id("rust-workflow-poll");
7044 let response = self
7045 .retry_worker_operation(|| {
7046 self.client.poll_workflow_task_response_with_request_id(
7047 &self.worker_id,
7048 &self.task_queue,
7049 self.poll_timeout,
7050 &poll_request_id,
7051 0,
7052 )
7053 })
7054 .await;
7055 let Some(response) = self.settle_worker_poll_response(response).await? else {
7056 return Ok(ManagedPollOutcome::Idle);
7057 };
7058 if response.outcome().should_stop() {
7059 return Ok(ManagedPollOutcome::Stop);
7060 }
7061 let memo_updates_supported =
7062 runtime_supports_workflow_memo_updates(response.server_capabilities.as_ref());
7063 let Some(task) = response.task else {
7064 return Ok(ManagedPollOutcome::Idle);
7065 };
7066
7067 let task_id = task.task_id.clone();
7068 let attempt = task.workflow_task_attempt;
7069 let run_id = task.run_id.clone();
7070 let lease_owner = task
7071 .lease_owner
7072 .clone()
7073 .unwrap_or_else(|| self.worker_id.clone());
7074
7075 match self.execute_workflow_task_decision(task) {
7076 Ok(decision)
7077 if commands_use_workflow_memo_updates(&decision.commands)
7078 && !memo_updates_supported =>
7079 {
7080 self.client
7081 .fail_workflow_task(
7082 &task_id,
7083 &lease_owner,
7084 attempt,
7085 Error::WorkflowMemoUpdatesUnavailable.to_string(),
7086 )
7087 .await?;
7088 }
7089 Ok(decision) if decision.commands.is_empty() => {
7090 self.client
7096 .fail_workflow_task_with_type(
7097 &task_id,
7098 &lease_owner,
7099 attempt,
7100 WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE,
7101 WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE,
7102 )
7103 .await?;
7104 }
7105 Ok(decision) => {
7106 let completion = self
7107 .client
7108 .complete_workflow_task_with_message_streams(
7109 &task_id,
7110 &lease_owner,
7111 attempt,
7112 decision.commands,
7113 decision.message_stream_cursors,
7114 decision.message_stream_waits,
7115 )
7116 .await;
7117 if let Err(error) = completion {
7118 if !workflow_task_completion_is_terminal_timeout(
7119 &error,
7120 &task_id,
7121 attempt,
7122 run_id.as_deref(),
7123 ) {
7124 return Err(error);
7125 }
7126 }
7127 }
7128 Err(error) => {
7129 self.client
7130 .fail_workflow_task(&task_id, &lease_owner, attempt, error.to_string())
7131 .await?;
7132 }
7133 }
7134
7135 Ok(ManagedPollOutcome::Handled)
7136 }
7137
7138 async fn poll_workflows_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
7139 while !stop.load(Ordering::SeqCst) {
7140 if self.poll_workflow_once().await? == ManagedPollOutcome::Stop {
7141 stop.store(true, Ordering::SeqCst);
7142 break;
7143 }
7144 }
7145
7146 Ok(())
7147 }
7148
7149 async fn poll_activity_once(&self) -> Result<ManagedPollOutcome> {
7150 let poll_request_id = unique_request_id("rust-activity-poll");
7151 let response = self
7152 .retry_worker_operation(|| {
7153 self.client.poll_activity_task_response_with_request_id(
7154 &self.worker_id,
7155 &self.task_queue,
7156 self.poll_timeout,
7157 &poll_request_id,
7158 0,
7159 )
7160 })
7161 .await;
7162 let Some(response) = self.settle_worker_poll_response(response).await? else {
7163 return Ok(ManagedPollOutcome::Idle);
7164 };
7165 if response.outcome().should_stop() {
7166 return Ok(ManagedPollOutcome::Stop);
7167 }
7168 let Some(task) = response.task else {
7169 return Ok(ManagedPollOutcome::Idle);
7170 };
7171
7172 let task_id = task.task_id.clone();
7173 let attempt_id = task
7174 .activity_attempt_id
7175 .clone()
7176 .or(task.attempt_id.clone())
7177 .unwrap_or_default();
7178 let lease_owner = task
7179 .lease_owner
7180 .clone()
7181 .unwrap_or_else(|| self.worker_id.clone());
7182 let codec = task.payload_codec.clone();
7183 let result = self.execute_activity_task(task).await;
7184 match result {
7185 Err(error) if worker_storage_admission_body(&error).is_some() => return Err(error),
7186 Ok(value) => {
7187 let completion = self
7188 .client
7189 .complete_activity_task(&task_id, &attempt_id, &lease_owner, value, &codec)
7190 .await;
7191 if let Err(error) = completion {
7192 if !activity_task_rejection_is_final(&error) {
7193 return Err(error);
7194 }
7195 }
7196 }
7197 Err(error) => {
7198 let failure = self
7199 .client
7200 .fail_activity_task(
7201 &task_id,
7202 &attempt_id,
7203 &lease_owner,
7204 error.to_string(),
7205 false,
7206 )
7207 .await;
7208 if let Err(error) = failure {
7209 if !activity_task_rejection_is_final(&error) {
7210 return Err(error);
7211 }
7212 }
7213 }
7214 }
7215
7216 Ok(ManagedPollOutcome::Handled)
7217 }
7218
7219 async fn poll_activities_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
7220 while !stop.load(Ordering::SeqCst) {
7221 if self.poll_activity_once().await? == ManagedPollOutcome::Stop {
7222 stop.store(true, Ordering::SeqCst);
7223 break;
7224 }
7225 }
7226
7227 Ok(())
7228 }
7229
7230 async fn poll_query_once(&self) -> Result<ManagedPollOutcome> {
7231 let poll_request_id = unique_request_id("rust-query-poll");
7232 let response = self
7233 .retry_worker_operation(|| {
7234 self.client.poll_query_task_response_with_request_id(
7235 &self.worker_id,
7236 &self.task_queue,
7237 self.poll_timeout,
7238 &poll_request_id,
7239 0,
7240 )
7241 })
7242 .await;
7243 let Some(response) = self.settle_worker_poll_response(response).await? else {
7244 return Ok(ManagedPollOutcome::Idle);
7245 };
7246 if response.outcome().should_stop() {
7247 return Ok(ManagedPollOutcome::Stop);
7248 }
7249 let Some(task) = response.task else {
7250 return Ok(ManagedPollOutcome::Idle);
7251 };
7252
7253 let query_task_id = task.query_task_id.clone();
7254 let attempt = task.query_task_attempt;
7255 let lease_owner = task
7256 .lease_owner
7257 .clone()
7258 .unwrap_or_else(|| self.worker_id.clone());
7259 let codec = task.payload_codec.clone();
7260
7261 match self.execute_query_task(task).await {
7262 Ok(value) => {
7263 let result_envelope = match encode_typed_envelope(&value, &codec) {
7264 Ok(result_envelope) => result_envelope,
7265 Err(error) => {
7266 let failure = self
7267 .client
7268 .fail_query_task(
7269 &query_task_id,
7270 &lease_owner,
7271 attempt,
7272 error.to_string(),
7273 "query_result_encode_failed",
7274 "QueryResultEncodeFailed",
7275 )
7276 .await;
7277 if let Err(error) = failure {
7278 if !query_task_rejection_is_final(&error) {
7279 return Err(error);
7280 }
7281 }
7282 return Ok(ManagedPollOutcome::Handled);
7283 }
7284 };
7285
7286 if let Err(error) = self
7287 .client
7288 .complete_query_task_with_envelope(
7289 &query_task_id,
7290 &lease_owner,
7291 attempt,
7292 value.clone().into_json()?,
7293 result_envelope,
7294 )
7295 .await
7296 {
7297 if !query_task_rejection_is_final(&error) {
7298 return Err(error);
7299 }
7300 }
7301 }
7302 Err(failure) => {
7303 let result = self
7304 .client
7305 .fail_query_task(
7306 &query_task_id,
7307 &lease_owner,
7308 attempt,
7309 failure.message,
7310 failure.reason,
7311 failure.failure_type,
7312 )
7313 .await;
7314 if let Err(error) = result {
7315 if !query_task_rejection_is_final(&error) {
7316 return Err(error);
7317 }
7318 }
7319 }
7320 }
7321
7322 Ok(ManagedPollOutcome::Handled)
7323 }
7324
7325 async fn poll_queries_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
7326 while !stop.load(Ordering::SeqCst) {
7327 if self.poll_query_once().await? == ManagedPollOutcome::Stop {
7328 stop.store(true, Ordering::SeqCst);
7329 break;
7330 }
7331 }
7332
7333 Ok(())
7334 }
7335
7336 async fn retry_worker_operation<T, F, Fut>(&self, mut operation: F) -> Result<T>
7337 where
7338 F: FnMut() -> Fut,
7339 Fut: Future<Output = Result<T>>,
7340 {
7341 let mut retries = 0;
7342
7343 loop {
7344 match operation().await {
7345 Err(error)
7346 if worker_operation_is_retryable(&error)
7347 && retries < self.retry_policy.max_retries =>
7348 {
7349 retries += 1;
7350 tokio::time::sleep(worker_retry_delay(self.retry_policy, retries)).await;
7351 }
7352 result => return result,
7353 }
7354 }
7355 }
7356
7357 async fn settle_worker_poll_response<T>(&self, response: Result<T>) -> Result<Option<T>> {
7358 match response {
7359 Ok(response) => Ok(Some(response)),
7360 Err(error) => {
7361 let Some(advertised_delay) = worker_poll_capacity_retry_after(&error) else {
7362 return Err(error);
7363 };
7364 let minimum_delay = self
7365 .retry_policy
7366 .initial_backoff
7367 .max(Duration::from_millis(1));
7368 let maximum_delay = self.retry_policy.max_backoff.max(minimum_delay);
7369 tokio::time::sleep(advertised_delay.max(minimum_delay).min(maximum_delay)).await;
7370 Ok(None)
7371 }
7372 }
7373 }
7374
7375 async fn execute_query_task(
7376 &self,
7377 mut task: QueryTask,
7378 ) -> std::result::Result<AvroValue, QueryTaskExecutionFailure> {
7379 validate_query_task_payloads(&task).map_err(|error| {
7380 QueryTaskExecutionFailure::new(
7381 "query_payload_decode_failed",
7382 error.to_string(),
7383 "QueryPayloadDecodeFailed",
7384 )
7385 })?;
7386
7387 if !self.workflows.contains_key(&task.workflow_type) {
7388 return Err(QueryTaskExecutionFailure::new(
7389 "query_workflow_type_not_registered",
7390 format!("no workflow registered for type {:?}", task.workflow_type),
7391 "WorkflowTypeNotRegistered",
7392 ));
7393 }
7394
7395 let Some(handlers) = self.queries.get(&task.workflow_type) else {
7396 return Err(QueryTaskExecutionFailure::new(
7397 "query_handler_unavailable",
7398 format!(
7399 "query handlers are unavailable for workflow type {:?}",
7400 task.workflow_type
7401 ),
7402 "QueryHandlerUnavailable",
7403 ));
7404 };
7405 let Some(query) = handlers.get(&task.query_name) else {
7406 return Err(QueryTaskExecutionFailure::new(
7407 "rejected_unknown_query",
7408 format!("unknown query {:?}", task.query_name),
7409 "QueryFailed",
7410 ));
7411 };
7412
7413 let args = decode_task_avro_arguments(task.query_arguments.as_ref(), &task.payload_codec)
7414 .map_err(|error| {
7415 QueryTaskExecutionFailure::new(
7416 "query_payload_decode_failed",
7417 format!("cannot decode query arguments: {error}"),
7418 "QueryPayloadDecodeFailed",
7419 )
7420 })?;
7421 let workflow_input_typed =
7422 decode_task_avro_arguments(task.workflow_arguments.as_ref(), &task.payload_codec)
7423 .map_err(|error| {
7424 QueryTaskExecutionFailure::new(
7425 "query_workflow_state_unavailable",
7426 format!("cannot decode workflow start input: {error}"),
7427 "QueryWorkflowStateUnavailable",
7428 )
7429 })?;
7430 let workflow_input = workflow_input_typed.clone().into_json().map_err(|error| {
7431 QueryTaskExecutionFailure::new(
7432 "query_workflow_state_unavailable",
7433 format!("cannot project workflow start input: {error}"),
7434 "QueryWorkflowStateUnavailable",
7435 )
7436 })?;
7437 hydrate_query_history_from_export(&mut task).map_err(|error| {
7438 QueryTaskExecutionFailure::new(
7439 "query_workflow_state_unavailable",
7440 format!("cannot restore query history snapshot: {error}"),
7441 "QueryWorkflowStateUnavailable",
7442 )
7443 })?;
7444 enrich_query_history_from_export(&mut task).map_err(|error| {
7445 QueryTaskExecutionFailure::new(
7446 "query_workflow_state_unavailable",
7447 format!("cannot restore compact query history payloads: {error}"),
7448 "QueryWorkflowStateUnavailable",
7449 )
7450 })?;
7451 let signal_events = query_signal_events(&task).map_err(|error| {
7452 QueryTaskExecutionFailure::new(
7453 "query_workflow_state_unavailable",
7454 format!("cannot decode committed workflow signals: {error}"),
7455 "QueryWorkflowStateUnavailable",
7456 )
7457 })?;
7458 let history_events = Arc::new(std::mem::take(&mut task.history_events));
7459 let context = QueryContext {
7460 workflow_id: task.workflow_id,
7461 run_id: task.run_id,
7462 workflow_type: task.workflow_type.clone(),
7463 run_status: task.run_status,
7464 workflow_input,
7465 workflow_input_avro_value: workflow_input_typed.clone(),
7466 history_events: Arc::clone(&history_events),
7467 signal_events: Arc::new(signal_events),
7468 };
7469
7470 let future = match query {
7471 RegisteredQuery::Snapshot(handler) => handler(context, args),
7472 RegisteredQuery::Replayed {
7473 state_type,
7474 handler,
7475 } => {
7476 let workflow = self
7477 .workflows
7478 .get(&task.workflow_type)
7479 .expect("workflow registration was checked above");
7480 if workflow.state_type != Some(*state_type) {
7481 return Err(QueryTaskExecutionFailure::new(
7482 "query_workflow_state_unavailable",
7483 "replayed query state type does not match its workflow registration",
7484 "QueryWorkflowStateUnavailable",
7485 ));
7486 }
7487 let replay = workflow.replay.as_ref().ok_or_else(|| {
7488 QueryTaskExecutionFailure::new(
7489 "query_workflow_state_unavailable",
7490 format!(
7491 "workflow type {:?} is not registered for instance-state replay",
7492 task.workflow_type
7493 ),
7494 "QueryWorkflowStateUnavailable",
7495 )
7496 })?;
7497 let workflow_state = Arc::new(Mutex::new(
7498 WorkflowState::new_with_identity(
7499 history_events.as_ref().clone(),
7500 context.workflow_id.clone(),
7501 context.run_id.clone(),
7502 self.task_queue.clone(),
7503 task.payload_codec,
7504 None,
7505 )
7506 .map_err(|error| {
7507 QueryTaskExecutionFailure::new(
7508 "query_workflow_state_unavailable",
7509 format!("workflow replay failed before query: {error}"),
7510 "QueryWorkflowStateUnavailable",
7511 )
7512 })?,
7513 ));
7514 let workflow_context = WorkflowContext {
7515 state: workflow_state,
7516 };
7517 let mut invocation = replay(workflow_context.clone(), workflow_input_typed.clone());
7518 let mut cx = TaskContext::from_waker(noop_waker_ref());
7519 match invocation.future.as_mut().poll(&mut cx) {
7520 Poll::Ready(Ok(_)) => {
7521 workflow_context
7522 .ensure_history_consumed()
7523 .map_err(|error| {
7524 QueryTaskExecutionFailure::new(
7525 "query_workflow_state_unavailable",
7526 format!("workflow replay failed before query: {error}"),
7527 "QueryWorkflowStateUnavailable",
7528 )
7529 })?;
7530 }
7531 Poll::Ready(Err(error)) => {
7532 return Err(QueryTaskExecutionFailure::new(
7533 "query_workflow_state_unavailable",
7534 format!("workflow replay failed before query: {error}"),
7535 "QueryWorkflowStateUnavailable",
7536 ));
7537 }
7538 Poll::Pending => {
7539 let commands = workflow_context.take_commands().map_err(|error| {
7540 QueryTaskExecutionFailure::new(
7541 "query_workflow_state_unavailable",
7542 format!("workflow replay failed before query: {error}"),
7543 "QueryWorkflowStateUnavailable",
7544 )
7545 })?;
7546 if commands.is_empty()
7547 && !workflow_context
7548 .matched_recorded_pending()
7549 .map_err(|error| {
7550 QueryTaskExecutionFailure::new(
7551 "query_workflow_state_unavailable",
7552 format!("workflow replay failed before query: {error}"),
7553 "QueryWorkflowStateUnavailable",
7554 )
7555 })?
7556 {
7557 return Err(QueryTaskExecutionFailure::new(
7558 "query_workflow_state_unavailable",
7559 "workflow replay yielded without a durable command",
7560 "QueryWorkflowStateUnavailable",
7561 ));
7562 }
7563 }
7564 }
7565 let state = (invocation.snapshot)().map_err(|error| {
7566 QueryTaskExecutionFailure::new(
7567 "query_workflow_state_unavailable",
7568 format!("cannot snapshot replayed workflow state: {error}"),
7569 "QueryWorkflowStateUnavailable",
7570 )
7571 })?;
7572 handler(context, state, args).map_err(|message| {
7573 QueryTaskExecutionFailure::new(
7574 "query_workflow_state_unavailable",
7575 message,
7576 "QueryWorkflowStateUnavailable",
7577 )
7578 })?
7579 }
7580 };
7581
7582 future.await.map_err(|error| {
7583 QueryTaskExecutionFailure::new("query_rejected", error.to_string(), "QueryFailed")
7584 })
7585 }
7586
7587 #[cfg(test)]
7588 fn execute_workflow_task(&self, task: WorkflowTask) -> Result<Vec<Value>> {
7589 Ok(self.execute_workflow_task_decision(task)?.commands)
7590 }
7591
7592 fn execute_workflow_task_decision(&self, task: WorkflowTask) -> Result<WorkflowTaskDecision> {
7593 validate_workflow_task_payloads(&task)?;
7594
7595 if let Some(update_id) = task
7596 .workflow_update_id
7597 .as_deref()
7598 .filter(|update_id| !update_id.is_empty())
7599 {
7600 return self
7601 .execute_update_task(&task, update_id)
7602 .map(WorkflowTaskDecision::without_message_streams);
7603 }
7604
7605 let workflow = self
7606 .workflows
7607 .get(&task.workflow_type)
7608 .ok_or_else(|| Error::WorkflowNotRegistered(task.workflow_type.clone()))?;
7609 let input = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
7610 let resume_signal = decode_resume_signal(&task)?;
7611 let history_budget = WorkflowHistoryBudget {
7612 event_count: task
7613 .total_history_events
7614 .unwrap_or_else(|| u64::try_from(task.history_events.len()).unwrap_or(u64::MAX)),
7615 size_bytes: task.history_size_bytes,
7616 continue_as_new_recommended: task.continue_as_new_recommended.unwrap_or(false),
7617 pressure: task.history_budget_pressure.clone(),
7618 };
7619 let workflow_command_identity = task
7620 .workflow_command_id
7621 .clone()
7622 .filter(|identity| !identity.is_empty())
7623 .unwrap_or_default();
7624 let mut workflow_state = WorkflowState::new_with_identity(
7625 task.history_events,
7626 task.workflow_id,
7627 task.run_id,
7628 self.task_queue.clone(),
7629 task.payload_codec.clone(),
7630 resume_signal,
7631 )?;
7632 workflow_state.history_budget = history_budget;
7633 workflow_state.workflow_command_identity = workflow_command_identity;
7634 workflow_state.cancel_requested = task.cancel_requested;
7635 let state = Arc::new(Mutex::new(workflow_state));
7636 let ctx = WorkflowContext { state };
7637 let mut future = (workflow.execute)(ctx.clone(), input);
7638 let mut cx = TaskContext::from_waker(noop_waker_ref());
7639
7640 match future.as_mut().poll(&mut cx) {
7641 Poll::Ready(Ok(result)) => {
7642 ctx.ensure_history_consumed()?;
7643 let result = encode_typed_envelope(&result, &task.payload_codec)?;
7644 let mut commands = ctx.take_commands()?;
7645 commands.push(json!({
7646 "type": "complete_workflow",
7647 "result": result
7648 }));
7649 self.message_stream_decision(&ctx, commands)
7650 }
7651 Poll::Ready(Err(error)) => {
7652 if let Error::ContinueAsNew(request) = error {
7653 let mut commands = ctx.take_commands()?;
7654 if let Some(command) = ctx.continue_as_new_command(request)? {
7655 commands.push(command);
7656 }
7657 ctx.ensure_history_consumed()?;
7658 return self.message_stream_decision(&ctx, commands);
7659 }
7660 if workflow_task_integrity_error(&error) {
7661 return Err(error);
7666 }
7667 ctx.ensure_history_consumed()?;
7670 let mut commands = ctx.take_commands()?;
7671 commands.push(workflow_failure_command(
7672 &error,
7673 recorded_activity_failure_boundary(&ctx, &error),
7674 ));
7675 self.message_stream_decision(&ctx, commands)
7676 }
7677 Poll::Pending => {
7678 let commands = ctx.take_commands()?;
7679 if commands.is_empty() && !ctx.matched_recorded_pending()? {
7680 Err(Error::WorkflowYieldedWithoutCommand)
7681 } else {
7682 self.message_stream_decision(&ctx, commands)
7683 }
7684 }
7685 }
7686 }
7687
7688 fn message_stream_decision(
7689 &self,
7690 ctx: &WorkflowContext,
7691 commands: Vec<Value>,
7692 ) -> Result<WorkflowTaskDecision> {
7693 let (message_stream_cursors, message_stream_waits) = ctx.message_stream_metadata()?;
7694 Ok(WorkflowTaskDecision {
7695 commands,
7696 message_stream_cursors,
7697 message_stream_waits,
7698 })
7699 }
7700
7701 fn execute_update_task(&self, task: &WorkflowTask, update_id: &str) -> Result<Vec<Value>> {
7702 if !self.workflows.contains_key(&task.workflow_type) {
7703 return Err(Error::WorkflowNotRegistered(task.workflow_type.clone()));
7704 }
7705
7706 let accepted = task.history_events.iter().rev().find_map(|event| {
7707 (event.event_type == "UpdateAccepted"
7708 && event.payload.get("update_id").and_then(Value::as_str) == Some(update_id))
7709 .then_some(&event.payload)
7710 });
7711 let update_name = accepted
7712 .and_then(|payload| payload.get("update_name"))
7713 .and_then(Value::as_str)
7714 .or(task.update_name.as_deref())
7715 .unwrap_or_default();
7716 let Some(handler) = self
7717 .updates
7718 .get(&task.workflow_type)
7719 .and_then(|handlers| handlers.get(update_name))
7720 else {
7721 return Ok(vec![json!({
7722 "type": "fail_update",
7723 "update_id": update_id,
7724 "message": format!(
7725 "no update handler is registered for {}.{update_name}",
7726 task.workflow_type
7727 ),
7728 "exception_type": "UnknownUpdate",
7729 "non_retryable": true,
7730 })]);
7731 };
7732 let arguments = accepted
7733 .and_then(|payload| payload.get("arguments"))
7734 .or(task.arguments.as_ref());
7735 let arguments = decode_task_avro_arguments(arguments, &task.payload_codec)?;
7736 let context = QueryContext {
7737 workflow_id: task.workflow_id.clone(),
7738 run_id: task.run_id.clone(),
7739 workflow_type: task.workflow_type.clone(),
7740 run_status: Some("running".to_string()),
7741 workflow_input: Value::Null,
7742 workflow_input_avro_value: AvroValue::Null,
7743 history_events: Arc::new(task.history_events.clone()),
7744 signal_events: Arc::new(Vec::new()),
7745 };
7746 let mut future = handler(context, arguments);
7747 let mut cx = TaskContext::from_waker(noop_waker_ref());
7748
7749 match future.as_mut().poll(&mut cx) {
7750 Poll::Ready(Ok(result)) => Ok(vec![json!({
7751 "type": "complete_update",
7752 "update_id": update_id,
7753 "result": encode_typed_envelope(&result, &task.payload_codec)?,
7754 })]),
7755 Poll::Ready(Err(error)) => Ok(vec![json!({
7756 "type": "fail_update",
7757 "update_id": update_id,
7758 "message": error.to_string(),
7759 "exception_type": "UpdateFailed",
7760 "non_retryable": true,
7761 })]),
7762 Poll::Pending => Err(Error::WorkflowYieldedWithoutCommand),
7763 }
7764 }
7765
7766 async fn execute_activity_task(&self, task: ActivityTask) -> Result<AvroValue> {
7767 validate_activity_task_payloads(&task)?;
7768
7769 let handler = self
7770 .activities
7771 .get(&task.activity_type)
7772 .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?;
7773 let args = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
7774 let attempt_id = task
7775 .activity_attempt_id
7776 .clone()
7777 .or(task.attempt_id.clone())
7778 .unwrap_or_default();
7779 let lease_owner = task
7780 .lease_owner
7781 .clone()
7782 .unwrap_or_else(|| self.worker_id.clone());
7783 let ctx = ActivityContext {
7784 client: self.client.clone(),
7785 task_id: task.task_id,
7786 activity_attempt_id: attempt_id,
7787 lease_owner,
7788 activity_type: task.activity_type,
7789 attempt_number: task.attempt_number,
7790 task_queue: self.task_queue.clone(),
7791 worker_id: self.worker_id.clone(),
7792 };
7793
7794 handler(ctx, args).await
7795 }
7796}
7797
7798fn poller_result(
7799 kind: &str,
7800 result: std::result::Result<Result<()>, tokio::task::JoinError>,
7801) -> Result<()> {
7802 match result {
7803 Ok(result) => result,
7804 Err(error) => Err(Error::WorkerLoop(format!(
7805 "{kind} poller join error: {error}"
7806 ))),
7807 }
7808}
7809
7810fn optional_poller_result(
7811 kind: &str,
7812 result: Option<std::result::Result<Result<()>, tokio::task::JoinError>>,
7813) -> Result<()> {
7814 match result {
7815 Some(result) => poller_result(kind, result),
7816 None => Ok(()),
7817 }
7818}
7819
7820async fn join_pollers(
7821 workflow_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7822 activity_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7823 query_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7824) -> Result<()> {
7825 let mut first_error = None;
7826
7827 if let Some(handle) = workflow_poller {
7828 if let Err(error) = poller_result("workflow", handle.await) {
7829 first_error.get_or_insert(error);
7830 }
7831 }
7832
7833 if let Some(handle) = activity_poller {
7834 if let Err(error) = poller_result("activity", handle.await) {
7835 first_error.get_or_insert(error);
7836 }
7837 }
7838
7839 if let Some(handle) = query_poller {
7840 if let Err(error) = poller_result("query", handle.await) {
7841 first_error.get_or_insert(error);
7842 }
7843 }
7844
7845 if let Some(error) = first_error {
7846 Err(error)
7847 } else {
7848 Ok(())
7849 }
7850}
7851
7852fn default_worker_id() -> String {
7853 let millis = SystemTime::now()
7854 .duration_since(UNIX_EPOCH)
7855 .unwrap_or_default()
7856 .as_millis();
7857 format!("rust-worker-{}-{millis}", std::process::id())
7858}
7859
7860fn percent_encode_path_segment(segment: &str) -> String {
7861 const HEX: &[u8; 16] = b"0123456789ABCDEF";
7862 let mut encoded = String::with_capacity(segment.len());
7863
7864 for byte in segment.bytes() {
7865 if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') {
7866 encoded.push(char::from(byte));
7867 } else {
7868 encoded.push('%');
7869 encoded.push(char::from(HEX[(byte >> 4) as usize]));
7870 encoded.push(char::from(HEX[(byte & 0x0f) as usize]));
7871 }
7872 }
7873
7874 encoded
7875}
7876
7877fn unique_request_id(prefix: &str) -> String {
7878 let nanos = SystemTime::now()
7879 .duration_since(UNIX_EPOCH)
7880 .unwrap_or_default()
7881 .as_nanos();
7882 format!("{prefix}-{}-{nanos}", std::process::id())
7883}
7884
7885#[derive(Debug)]
7886struct QueryTaskExecutionFailure {
7887 reason: String,
7888 message: String,
7889 failure_type: String,
7890}
7891
7892impl QueryTaskExecutionFailure {
7893 fn new(
7894 reason: impl Into<String>,
7895 message: impl Into<String>,
7896 failure_type: impl Into<String>,
7897 ) -> Self {
7898 Self {
7899 reason: reason.into(),
7900 message: message.into(),
7901 failure_type: failure_type.into(),
7902 }
7903 }
7904}
7905
7906#[derive(Clone, Debug)]
7912pub struct WorkflowInstance<S> {
7913 state: Arc<Mutex<S>>,
7914}
7915
7916impl<S> WorkflowInstance<S> {
7917 fn new(state: S) -> Self {
7918 Self {
7919 state: Arc::new(Mutex::new(state)),
7920 }
7921 }
7922
7923 pub fn read<R>(&self, reader: impl FnOnce(&S) -> R) -> Result<R> {
7925 let state = self
7926 .state
7927 .lock()
7928 .map_err(|_| Error::WorkflowStatePoisoned)?;
7929 Ok(reader(&state))
7930 }
7931
7932 pub fn update<R>(&self, transition: impl FnOnce(&mut S) -> R) -> Result<R> {
7934 let mut state = self
7935 .state
7936 .lock()
7937 .map_err(|_| Error::WorkflowStatePoisoned)?;
7938 Ok(transition(&mut state))
7939 }
7940}
7941
7942impl<S: Clone> WorkflowInstance<S> {
7943 fn snapshot(&self) -> Result<S> {
7944 self.read(Clone::clone)
7945 }
7946}
7947
7948#[derive(Clone, Debug, PartialEq)]
7949pub struct MessageStreamMessage {
7950 pub stream_name: String,
7951 pub message_id: String,
7952 pub position: u64,
7953 pub arguments: Vec<AvroValue>,
7954}
7955
7956#[derive(Clone, Debug)]
7957pub struct MessageStream {
7958 ctx: WorkflowContext,
7959 name: String,
7960}
7961
7962impl MessageStream {
7963 pub async fn receive(&self, max_items: usize) -> Result<Vec<MessageStreamMessage>> {
7965 if !(1..=MESSAGE_STREAM_MAX_BATCH).contains(&max_items) {
7966 return Err(Error::Codec(format!(
7967 "message stream max_items must be between 1 and {MESSAGE_STREAM_MAX_BATCH}"
7968 )));
7969 }
7970 loop {
7971 if let Some(batch) = self.ctx.take_message_stream_batch(&self.name, max_items)? {
7972 return Ok(batch);
7973 }
7974
7975 self.ctx.record_message_stream_wait(&self.name)?;
7976 let replay_wait_sequence = self.ctx.next_message_stream_wait_sequence()?;
7977 let arguments = self.ctx.wait_runtime_signal(MESSAGE_STREAM_SIGNAL).await?;
7978 self.ctx.buffer_message_stream_delivery(arguments)?;
7979 if let Some(sequence) = replay_wait_sequence {
7980 self.ctx.buffer_message_stream_history_for_wait(sequence)?;
7981 }
7982 }
7983 }
7984
7985 pub async fn receive_one(&self) -> Result<MessageStreamMessage> {
7986 self.receive(1)
7987 .await?
7988 .into_iter()
7989 .next()
7990 .ok_or_else(|| Error::Codec("message stream resumed without a message".to_string()))
7991 }
7992}
7993
7994#[derive(Clone, Debug)]
7995pub struct WorkflowContext {
7996 state: Arc<Mutex<WorkflowState>>,
7997}
7998
7999fn valid_memo_key(key: &str) -> bool {
8000 let numeric_candidate = key.strip_prefix('-').unwrap_or(key);
8001
8002 !key.is_empty()
8003 && key.len() <= 64
8004 && (numeric_candidate.is_empty()
8005 || !numeric_candidate.bytes().all(|byte| byte.is_ascii_digit()))
8006 && key
8007 .bytes()
8008 .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b':' | b'-'))
8009}
8010
8011fn avro_encoded_size(value: &AvroValue) -> Result<usize> {
8012 BASE64
8013 .decode(encode_avro_value(value)?.blob)
8014 .map(|bytes| bytes.len())
8015 .map_err(|error| Error::Codec(format!("memo Avro encoding was not strict base64: {error}")))
8016}
8017
8018fn canonical_memo_entries(value: AvroValue, require_entries: bool) -> Result<AvroValue> {
8019 let AvroValue::Map(entries) = value else {
8020 return Err(Error::InvalidMemoUpdate(
8021 "entries must serialize to an Avro string-keyed map".to_string(),
8022 ));
8023 };
8024 if require_entries && entries.is_empty() {
8025 return Err(Error::InvalidMemoUpdate(
8026 "at least one entry is required".to_string(),
8027 ));
8028 }
8029 if entries.len() > MAX_MEMO_ENTRIES {
8030 return Err(Error::InvalidMemoUpdate(format!(
8031 "at most {MAX_MEMO_ENTRIES} entries are allowed"
8032 )));
8033 }
8034
8035 for (key, value) in &entries {
8036 if !valid_memo_key(&key) {
8037 return Err(Error::InvalidMemoUpdate(
8038 "keys must match ^(?!-?[0-9]+$)[A-Za-z0-9_.:-]{1,64}$".to_string(),
8039 ));
8040 }
8041 if avro_encoded_size(value)? > MAX_MEMO_VALUE_SIZE_BYTES {
8042 return Err(Error::InvalidMemoUpdate(format!(
8043 "value {key:?} exceeds the {MAX_MEMO_VALUE_SIZE_BYTES}-byte limit"
8044 )));
8045 }
8046 }
8047
8048 let value = AvroValue::Map(entries);
8049 if avro_encoded_size(&value)? > MAX_MEMO_TOTAL_SIZE_BYTES {
8050 return Err(Error::InvalidMemoUpdate(format!(
8051 "update exceeds the {MAX_MEMO_TOTAL_SIZE_BYTES}-byte total limit"
8052 )));
8053 }
8054 Ok(value)
8055}
8056
8057fn decode_memo_history_map(envelope: &Value, require_entries: bool) -> Result<AvroValue> {
8058 let object = envelope.as_object().ok_or_else(|| {
8059 Error::InvalidMemoUpdate(
8060 "history field must use the public {codec, blob} payload envelope".to_string(),
8061 )
8062 })?;
8063 if object.len() != 2 || !object.contains_key("codec") || !object.contains_key("blob") {
8064 return Err(Error::InvalidMemoUpdate(
8065 "history field must use exactly the public {codec, blob} payload envelope".to_string(),
8066 ));
8067 }
8068
8069 canonical_memo_entries(
8070 decode_wire_avro_value(envelope, DEFAULT_CODEC)?,
8071 require_entries,
8072 )
8073}
8074
8075impl WorkflowContext {
8076 pub fn message_stream(&self, name: impl Into<String>) -> Result<MessageStream> {
8077 let name = name.into();
8078 if name.is_empty()
8079 || name.len() > 128
8080 || !name.bytes().all(|byte| {
8081 byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b':' | b'-')
8082 })
8083 {
8084 return Err(Error::Codec(
8085 "message stream names must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens"
8086 .to_string(),
8087 ));
8088 }
8089 Ok(MessageStream {
8090 ctx: self.clone(),
8091 name,
8092 })
8093 }
8094
8095 fn record_message_stream_wait(&self, name: &str) -> Result<()> {
8096 let mut state = self
8097 .state
8098 .lock()
8099 .map_err(|_| Error::WorkflowStatePoisoned)?;
8100 let position = state.message_stream_cursors.get(name).copied().unwrap_or(0);
8101 state
8102 .message_stream_waits
8103 .insert(name.to_string(), position);
8104 Ok(())
8105 }
8106
8107 fn buffer_message_stream(&self, message: MessageStreamMessage) -> Result<()> {
8108 let mut state = self
8109 .state
8110 .lock()
8111 .map_err(|_| Error::WorkflowStatePoisoned)?;
8112 let cursor = state
8113 .message_stream_cursors
8114 .get(&message.stream_name)
8115 .copied()
8116 .unwrap_or(0);
8117 if message.position <= cursor {
8118 return Ok(());
8119 }
8120 let pending = state
8121 .message_stream_messages
8122 .entry(message.stream_name.clone())
8123 .or_default();
8124 if pending.iter().any(|candidate| {
8125 candidate.position == message.position || candidate.message_id == message.message_id
8126 }) {
8127 return Ok(());
8128 }
8129 pending.push(message);
8130 pending.sort_by_key(|candidate| candidate.position);
8131 Ok(())
8132 }
8133
8134 fn buffer_message_stream_delivery(&self, arguments: Vec<Value>) -> Result<Option<String>> {
8135 if let Some(delivery) = decode_message_stream_delivery(arguments)? {
8136 match delivery {
8137 MessageStreamDelivery::Message(message) => {
8138 let stream_name = message.stream_name.clone();
8139 self.buffer_message_stream(message)?;
8140 return Ok(Some(stream_name));
8141 }
8142 MessageStreamDelivery::Cursor {
8143 stream_name,
8144 through_position,
8145 } => self.apply_message_stream_cursor(&stream_name, through_position)?,
8146 }
8147 }
8148 Ok(None)
8149 }
8150
8151 fn next_message_stream_wait_sequence(&self) -> Result<Option<u64>> {
8152 let state = self
8153 .state
8154 .lock()
8155 .map_err(|_| Error::WorkflowStatePoisoned)?;
8156 Ok(match state.recorded_commands.get(state.command_cursor) {
8157 Some(RecordedCommand::SignalWait {
8158 sequence,
8159 signal_name,
8160 ..
8161 }) if signal_name == MESSAGE_STREAM_SIGNAL => Some(*sequence),
8162 _ => None,
8163 })
8164 }
8165
8166 fn buffer_message_stream_history_for_wait(&self, wait_sequence: u64) -> Result<()> {
8167 let (history, payload_codec) = {
8168 let state = self
8169 .state
8170 .lock()
8171 .map_err(|_| Error::WorkflowStatePoisoned)?;
8172 (
8173 Arc::clone(&state.history_events),
8174 state.payload_codec.clone(),
8175 )
8176 };
8177
8178 let Some(opened_index) = history.iter().position(|event| {
8179 event.event_type == "SignalWaitOpened"
8180 && durable_event_sequence(event) == Some(wait_sequence)
8181 && event.payload.get("signal_name").and_then(Value::as_str)
8182 == Some(MESSAGE_STREAM_SIGNAL)
8183 }) else {
8184 return Ok(());
8185 };
8186 let boundary_index = history
8187 .iter()
8188 .enumerate()
8189 .skip(opened_index + 1)
8190 .find_map(|(index, event)| {
8191 (durable_event_sequence(event).is_some_and(|sequence| sequence > wait_sequence)
8192 && is_authored_command_open_event(event))
8193 .then_some(index)
8194 })
8195 .unwrap_or(history.len());
8196
8197 for event in history[opened_index + 1..boundary_index]
8198 .iter()
8199 .filter(|event| {
8200 event.event_type == "SignalReceived"
8201 && event.payload.get("signal_name").and_then(Value::as_str)
8202 == Some(MESSAGE_STREAM_SIGNAL)
8203 })
8204 {
8205 let arguments = decode_signal_event_arguments(event, &payload_codec)?
8206 .into_iter()
8207 .map(AvroValue::into_json)
8208 .collect::<Result<Vec<_>>>()?;
8209 self.buffer_message_stream_delivery(arguments)?;
8210 }
8211 Ok(())
8212 }
8213
8214 fn apply_message_stream_cursor(&self, name: &str, through_position: u64) -> Result<()> {
8215 let mut state = self
8216 .state
8217 .lock()
8218 .map_err(|_| Error::WorkflowStatePoisoned)?;
8219 let cursor = state
8220 .message_stream_cursors
8221 .entry(name.to_string())
8222 .or_default();
8223 *cursor = (*cursor).max(through_position);
8224 if let Some(pending) = state.message_stream_messages.get_mut(name) {
8225 pending.retain(|message| message.position > through_position);
8226 }
8227 Ok(())
8228 }
8229
8230 fn take_message_stream_batch(
8231 &self,
8232 name: &str,
8233 max_items: usize,
8234 ) -> Result<Option<Vec<MessageStreamMessage>>> {
8235 let mut state = self
8236 .state
8237 .lock()
8238 .map_err(|_| Error::WorkflowStatePoisoned)?;
8239 let cursor = state.message_stream_cursors.get(name).copied().unwrap_or(0);
8240 let pending = state
8241 .message_stream_messages
8242 .entry(name.to_string())
8243 .or_default();
8244 let count = contiguous_message_stream_count(pending, cursor, max_items);
8245 if count == 0 {
8246 return Ok(None);
8247 }
8248 let batch = pending.drain(..count).collect::<Vec<_>>();
8249 let position = batch.last().map(|message| message.position).unwrap_or(0);
8250 state
8251 .message_stream_cursors
8252 .insert(name.to_string(), position);
8253 state.message_stream_waits.remove(name);
8254 Ok(Some(batch))
8255 }
8256
8257 fn message_stream_metadata(&self) -> Result<(Vec<Value>, Vec<Value>)> {
8258 let state = self
8259 .state
8260 .lock()
8261 .map_err(|_| Error::WorkflowStatePoisoned)?;
8262 let mut cursors = state.message_stream_cursors.iter().collect::<Vec<_>>();
8263 cursors.sort_by_key(|(name, _)| *name);
8264 let mut waits = state.message_stream_waits.iter().collect::<Vec<_>>();
8265 waits.sort_by_key(|(name, _)| *name);
8266 Ok((
8267 cursors
8268 .into_iter()
8269 .map(|(name, position)| json!({"stream_name": name, "through_position": position}))
8270 .collect(),
8271 waits
8272 .into_iter()
8273 .map(|(name, position)| json!({"stream_name": name, "after_position": position}))
8274 .collect(),
8275 ))
8276 }
8277 pub fn workflow_identity(&self) -> Result<WorkflowIdentity> {
8279 let state = self
8280 .state
8281 .lock()
8282 .map_err(|_| Error::WorkflowStatePoisoned)?;
8283 Ok(WorkflowIdentity {
8284 workflow_id: state.workflow_id.clone(),
8285 run_id: state.run_id.clone(),
8286 })
8287 }
8288
8289 pub fn history_budget(&self) -> Result<WorkflowHistoryBudget> {
8291 let state = self
8292 .state
8293 .lock()
8294 .map_err(|_| Error::WorkflowStatePoisoned)?;
8295 Ok(state.history_budget.clone())
8296 }
8297
8298 pub fn continue_as_new<T: Serialize>(&self, args: T) -> Result<Value> {
8304 self.continue_as_new_with_options(ContinueAsNewOptions::new(), args)
8305 }
8306
8307 pub fn continue_as_new_with_options<T: Serialize>(
8309 &self,
8310 options: ContinueAsNewOptions,
8311 args: T,
8312 ) -> Result<Value> {
8313 options.validate()?;
8314 Err(Error::ContinueAsNew(ContinueAsNewRequest {
8315 arguments: normalize_avro_arguments(AvroValue::from_serialize(&args)?),
8316 options,
8317 }))
8318 }
8319
8320 pub fn activity<T: Serialize>(
8321 &self,
8322 activity_type: impl Into<String>,
8323 args: T,
8324 ) -> ActivityCall {
8325 self.activity_with_options(activity_type, ActivityOptions::new(), args)
8326 }
8327
8328 pub fn activity_on_queue<T, Q>(
8329 &self,
8330 activity_type: impl Into<String>,
8331 task_queue: Option<Q>,
8332 args: T,
8333 ) -> ActivityCall
8334 where
8335 T: Serialize,
8336 Q: Into<String>,
8337 {
8338 let mut options = ActivityOptions::new();
8339 options.task_queue = task_queue.map(Into::into);
8340 self.activity_with_options(activity_type, options, args)
8341 }
8342
8343 pub fn activity_with_options<T: Serialize>(
8381 &self,
8382 activity_type: impl Into<String>,
8383 options: ActivityOptions,
8384 args: T,
8385 ) -> ActivityCall {
8386 ActivityCall {
8387 ctx: self.clone(),
8388 activity_type: activity_type.into(),
8389 options,
8390 args: Some(AvroValue::from_serialize(&args)),
8391 scheduled: false,
8392 parallel_group_path: Vec::new(),
8393 }
8394 }
8395
8396 pub async fn activity_avro_value<T: Serialize>(
8397 &self,
8398 activity_type: impl Into<String>,
8399 args: T,
8400 ) -> Result<AvroValue> {
8401 let mut call = self.activity(activity_type, args);
8402 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8403 }
8404
8405 pub async fn activity_avro_value_with_options<T: Serialize>(
8406 &self,
8407 activity_type: impl Into<String>,
8408 options: ActivityOptions,
8409 args: T,
8410 ) -> Result<AvroValue> {
8411 let mut call = self.activity_with_options(activity_type, options, args);
8412 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8413 }
8414
8415 pub async fn activity_typed<I, O>(&self, activity_type: impl Into<String>, args: I) -> Result<O>
8417 where
8418 I: Serialize,
8419 O: DeserializeOwned,
8420 {
8421 self.activity_typed_with_options(activity_type, ActivityOptions::new(), args)
8422 .await
8423 }
8424
8425 pub async fn activity_typed_with_options<I, O>(
8431 &self,
8432 activity_type: impl Into<String>,
8433 options: ActivityOptions,
8434 args: I,
8435 ) -> Result<O>
8436 where
8437 I: Serialize,
8438 O: DeserializeOwned,
8439 {
8440 let activity_type = activity_type.into();
8441 let encoded = AvroValue::from_serialize(&args).map_err(|error| {
8442 handler_type_error::<I>(
8443 HandlerKind::Activity,
8444 &activity_type,
8445 HandlerValueKind::Input,
8446 error.to_string(),
8447 )
8448 });
8449 let mut call = ActivityCall {
8450 ctx: self.clone(),
8451 activity_type: activity_type.clone(),
8452 options,
8453 args: Some(encoded),
8454 scheduled: false,
8455 parallel_group_path: Vec::new(),
8456 };
8457 let result = std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await?;
8458 decode_handler_result(result, HandlerKind::Activity, &activity_type)
8459 }
8460
8461 pub fn parallel(&self, operations: Vec<ParallelOperation>) -> ParallelCall {
8468 ParallelCall::new(self.clone(), operations)
8469 }
8470
8471 pub fn join(&self, operations: Vec<ParallelOperation>) -> ParallelCall {
8473 self.parallel(operations)
8474 }
8475
8476 pub async fn parallel_avro_value(
8478 &self,
8479 operations: Vec<ParallelOperation>,
8480 ) -> Result<Vec<ParallelAvroResult>> {
8481 let mut call = self.parallel(operations);
8482 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8483 }
8484
8485 pub fn select(&self, operations: Vec<ParallelOperation>) -> SelectCall {
8488 let operations = operations
8489 .into_iter()
8490 .enumerate()
8491 .map(|(index, operation)| (SelectionKey::Index(index), operation))
8492 .collect();
8493 SelectCall::new(self.clone(), operations)
8494 }
8495
8496 pub fn select_keyed<K>(&self, operations: Vec<(K, ParallelOperation)>) -> SelectCall
8498 where
8499 K: Into<SelectionKey>,
8500 {
8501 SelectCall::new(
8502 self.clone(),
8503 operations
8504 .into_iter()
8505 .map(|(key, operation)| (key.into(), operation))
8506 .collect(),
8507 )
8508 }
8509
8510 pub fn saga(&self) -> Saga {
8512 Saga::new(self.clone())
8513 }
8514
8515 pub fn is_cancellation_requested(&self) -> Result<bool> {
8520 let state = self
8521 .state
8522 .lock()
8523 .map_err(|_| Error::WorkflowStatePoisoned)?;
8524 Ok(state.cancel_requested)
8525 }
8526
8527 pub fn throw_if_cancellation_requested(&self) -> Result<()> {
8532 if self.is_cancellation_requested()? {
8533 return Err(Error::WorkflowCancellationRequested(
8534 WorkflowCancellationRequested,
8535 ));
8536 }
8537 Ok(())
8538 }
8539
8540 pub fn wait_signal(&self, signal_name: impl Into<String>) -> SignalCall {
8541 SignalCall {
8542 ctx: self.clone(),
8543 signal_name: signal_name.into(),
8544 runtime_reserved_allowed: false,
8545 opened_wait: false,
8546 matched_pending: false,
8547 parallel_group_path: Vec::new(),
8548 }
8549 }
8550
8551 fn wait_runtime_signal(&self, signal_name: impl Into<String>) -> SignalCall {
8552 SignalCall {
8553 ctx: self.clone(),
8554 signal_name: signal_name.into(),
8555 runtime_reserved_allowed: true,
8556 opened_wait: false,
8557 matched_pending: false,
8558 parallel_group_path: Vec::new(),
8559 }
8560 }
8561
8562 pub async fn wait_signal_avro_value(
8563 &self,
8564 signal_name: impl Into<String>,
8565 ) -> Result<Vec<AvroValue>> {
8566 let mut call = self.wait_signal(signal_name);
8567 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8568 }
8569
8570 pub fn signals(&self, signal_name: &str) -> Result<Vec<Vec<Value>>> {
8576 self.signals_avro_value(signal_name)?
8577 .into_iter()
8578 .map(|arguments| {
8579 arguments
8580 .into_iter()
8581 .map(AvroValue::into_json)
8582 .collect::<Result<Vec<_>>>()
8583 })
8584 .collect()
8585 }
8586
8587 pub fn signals_avro_value(&self, signal_name: &str) -> Result<Vec<Vec<AvroValue>>> {
8589 let state = self
8590 .state
8591 .lock()
8592 .map_err(|_| Error::WorkflowStatePoisoned)?;
8593 state
8594 .history_events
8595 .iter()
8596 .filter(|event| {
8597 event.event_type == "SignalReceived"
8598 && event.payload.get("signal_name").and_then(Value::as_str) == Some(signal_name)
8599 })
8600 .map(|event| decode_signal_event_arguments(event, &state.payload_codec))
8601 .collect()
8602 }
8603
8604 pub fn updates(&self, update_name: &str) -> Result<Vec<Vec<Value>>> {
8610 self.updates_avro_value(update_name)?
8611 .into_iter()
8612 .map(|arguments| {
8613 arguments
8614 .into_iter()
8615 .map(AvroValue::into_json)
8616 .collect::<Result<Vec<_>>>()
8617 })
8618 .collect()
8619 }
8620
8621 pub fn updates_avro_value(&self, update_name: &str) -> Result<Vec<Vec<AvroValue>>> {
8623 let state = self
8624 .state
8625 .lock()
8626 .map_err(|_| Error::WorkflowStatePoisoned)?;
8627 let mut seen = Vec::new();
8628 let mut updates = Vec::new();
8629 for event in state.history_events.iter() {
8630 if !matches!(
8631 event.event_type.as_str(),
8632 "UpdateAccepted" | "UpdateApplied"
8633 ) || event.payload.get("update_name").and_then(Value::as_str) != Some(update_name)
8634 || event.payload.get("arguments").is_none()
8635 {
8636 continue;
8637 }
8638 if let Some(update_id) = event.payload.get("update_id").and_then(Value::as_str) {
8639 if seen.iter().any(|recorded| recorded == update_id) {
8640 continue;
8641 }
8642 seen.push(update_id.to_string());
8643 }
8644 updates.push(decode_update_event_arguments(event, &state.payload_codec)?);
8645 }
8646 Ok(updates)
8647 }
8648
8649 pub fn wait_condition<F>(
8657 &self,
8658 options: ConditionWaitOptions,
8659 predicate: F,
8660 ) -> ConditionWaitCall
8661 where
8662 F: Fn() -> Result<bool> + Send + 'static,
8663 {
8664 ConditionWaitCall {
8665 ctx: self.clone(),
8666 options,
8667 predicate: Box::new(predicate),
8668 occurrence_id: None,
8669 opened_wait: false,
8670 parallel_group_path: Vec::new(),
8671 }
8672 }
8673
8674 pub fn sleep(&self, duration: Duration) -> TimerCall {
8695 let delay_seconds = duration
8696 .as_secs()
8697 .checked_add(u64::from(duration.subsec_nanos() > 0));
8698 TimerCall {
8699 ctx: self.clone(),
8700 delay_seconds,
8701 scheduled: false,
8702 matched_pending: false,
8703 parallel_group_path: Vec::new(),
8704 }
8705 }
8706
8707 pub fn start_timer(&self, duration: Duration) -> TimerCall {
8709 self.sleep(duration)
8710 }
8711
8712 pub fn side_effect<T, F>(&self, callback: F) -> Result<T>
8719 where
8720 T: Serialize + DeserializeOwned,
8721 F: FnOnce() -> T,
8722 {
8723 {
8724 let mut state = self
8725 .state
8726 .lock()
8727 .map_err(|_| Error::WorkflowStatePoisoned)?;
8728 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8729 return match recorded {
8730 RecordedCommand::SideEffect { sequence, value } => {
8731 state.command_cursor += 1;
8732 value.deserialize().map_err(|error| {
8733 Error::NonDeterministicReplay(ReplayFailure::new(
8734 "side_effect_type_mismatch",
8735 Some(sequence),
8736 Some(std::any::type_name::<T>().to_string()),
8737 Some(error.to_string()),
8738 "recorded side-effect value is incompatible with the requested Rust type",
8739 ))
8740 })
8741 }
8742 other => Err(command_mismatch(&other, "side effect")),
8743 };
8744 }
8745 }
8746
8747 let value = callback();
8748 let avro_value = AvroValue::from_serialize(&value)?;
8749 let mut state = self
8750 .state
8751 .lock()
8752 .map_err(|_| Error::WorkflowStatePoisoned)?;
8753 let result = encode_typed_envelope(&avro_value, &state.payload_codec)?;
8754 state.commands.push(json!({
8755 "type": "record_side_effect",
8756 "result": result,
8757 }));
8758 Ok(value)
8759 }
8760
8761 pub fn side_effect_avro_value<F>(&self, callback: F) -> Result<AvroValue>
8763 where
8764 F: FnOnce() -> AvroValue,
8765 {
8766 {
8767 let mut state = self
8768 .state
8769 .lock()
8770 .map_err(|_| Error::WorkflowStatePoisoned)?;
8771 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8772 return match recorded {
8773 RecordedCommand::SideEffect { value, .. } => {
8774 state.command_cursor += 1;
8775 Ok(value)
8776 }
8777 other => Err(command_mismatch(&other, "side effect")),
8778 };
8779 }
8780 }
8781
8782 let value = callback();
8783 let mut state = self
8784 .state
8785 .lock()
8786 .map_err(|_| Error::WorkflowStatePoisoned)?;
8787 let result = encode_typed_envelope(&value, &state.payload_codec)?;
8788 state.commands.push(json!({
8789 "type": "record_side_effect",
8790 "result": result,
8791 }));
8792 Ok(value)
8793 }
8794
8795 pub fn append_workflow_stream(
8801 &self,
8802 stream_name: impl Into<String>,
8803 items: &[WorkflowStreamAppendItem],
8804 max_pending_items: Option<u64>,
8805 ) -> Result<()> {
8806 if items.is_empty() {
8807 return Err(Error::Codec(
8808 "workflow_stream_items_empty: append requires at least one item".to_string(),
8809 ));
8810 }
8811 if max_pending_items == Some(0) {
8812 return Err(Error::Codec(
8813 "workflow_stream_pending_limit_invalid: max_pending_items must be positive"
8814 .to_string(),
8815 ));
8816 }
8817 let stream_name = stream_name.into();
8818 if stream_name.is_empty() {
8819 return Err(Error::Codec(
8820 "workflow_stream_name_invalid: stream name must not be empty".to_string(),
8821 ));
8822 }
8823
8824 let mut state = self
8825 .state
8826 .lock()
8827 .map_err(|_| Error::WorkflowStatePoisoned)?;
8828 let command_ordinal = state.workflow_stream_command_counter;
8829 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8830 state.workflow_stream_command_counter += 1;
8831 return match recorded {
8832 RecordedCommand::SideEffect { .. } => {
8833 state.command_cursor += 1;
8834 Ok(())
8835 }
8836 other => Err(command_mismatch(&other, "workflow stream append")),
8837 };
8838 }
8839
8840 let identity = Self::workflow_stream_command_identity(&state)?.to_string();
8841 state.workflow_stream_command_counter += 1;
8842 let wire_items = items
8843 .iter()
8844 .enumerate()
8845 .map(|(item_index, item)| {
8846 item.wire_value(Some(format!(
8847 "dw-stream:{identity}:{command_ordinal}:{item_index}"
8848 )))
8849 })
8850 .collect::<Vec<_>>();
8851 let mut directive = json!({
8852 "operation": "append",
8853 "stream_name": stream_name,
8854 "command_identity": identity,
8855 "command_ordinal": command_ordinal,
8856 "items": wire_items,
8857 });
8858 if let Some(max_pending_items) = max_pending_items {
8859 directive["max_pending_items"] = json!(max_pending_items);
8860 }
8861 let result = encode_typed_envelope(&AvroValue::Null, &state.payload_codec)?;
8862 state.commands.push(json!({
8863 "type": "record_side_effect",
8864 "result": result,
8865 "workflow_stream": directive,
8866 }));
8867 Ok(())
8868 }
8869
8870 pub fn close_workflow_stream(
8872 &self,
8873 stream_name: impl Into<String>,
8874 retention_seconds: Option<u64>,
8875 ) -> Result<()> {
8876 self.finish_workflow_stream(stream_name.into(), None, retention_seconds)
8877 }
8878
8879 pub fn error_workflow_stream(
8881 &self,
8882 stream_name: impl Into<String>,
8883 error_reason: impl Into<String>,
8884 retention_seconds: Option<u64>,
8885 ) -> Result<()> {
8886 let error_reason = error_reason.into();
8887 if error_reason.is_empty() {
8888 return Err(Error::Codec(
8889 "workflow_stream_error_invalid: error reason must not be empty".to_string(),
8890 ));
8891 }
8892 self.finish_workflow_stream(stream_name.into(), Some(error_reason), retention_seconds)
8893 }
8894
8895 fn finish_workflow_stream(
8896 &self,
8897 stream_name: String,
8898 error_reason: Option<String>,
8899 retention_seconds: Option<u64>,
8900 ) -> Result<()> {
8901 if stream_name.is_empty() {
8902 return Err(Error::Codec(
8903 "workflow_stream_name_invalid: stream name must not be empty".to_string(),
8904 ));
8905 }
8906 if retention_seconds == Some(0) {
8907 return Err(Error::Codec(
8908 "workflow_stream_retention_invalid: retention_seconds must be positive".to_string(),
8909 ));
8910 }
8911 let mut state = self
8912 .state
8913 .lock()
8914 .map_err(|_| Error::WorkflowStatePoisoned)?;
8915 let command_ordinal = state.workflow_stream_command_counter;
8916 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8917 state.workflow_stream_command_counter += 1;
8918 return match recorded {
8919 RecordedCommand::SideEffect { .. } => {
8920 state.command_cursor += 1;
8921 Ok(())
8922 }
8923 other => Err(command_mismatch(&other, "workflow stream close")),
8924 };
8925 }
8926 let identity = Self::workflow_stream_command_identity(&state)?.to_string();
8927 state.workflow_stream_command_counter += 1;
8928 let mut directive = json!({
8929 "operation": if error_reason.is_some() { "error" } else { "close" },
8930 "stream_name": stream_name,
8931 "command_identity": identity,
8932 "command_ordinal": command_ordinal,
8933 });
8934 if let Some(error_reason) = error_reason {
8935 directive["error_reason"] = json!(error_reason);
8936 }
8937 if let Some(retention_seconds) = retention_seconds {
8938 directive["retention_seconds"] = json!(retention_seconds);
8939 }
8940 let result = encode_typed_envelope(&AvroValue::Null, &state.payload_codec)?;
8941 state.commands.push(json!({
8942 "type": "record_side_effect",
8943 "result": result,
8944 "workflow_stream": directive,
8945 }));
8946 Ok(())
8947 }
8948
8949 fn workflow_stream_command_identity(state: &WorkflowState) -> Result<&str> {
8950 let identity = state.workflow_command_identity.as_str();
8951 if identity.is_empty() {
8952 return Err(Error::MissingWorkflowCommandIdentity);
8953 }
8954 Ok(identity)
8955 }
8956
8957 pub fn upsert_search_attributes(&self, update: SearchAttributeUpdate) -> Result<()> {
8963 update.validate()?;
8964 let (attributes, attribute_types) = update.into_wire_parts();
8965 let mut state = self
8966 .state
8967 .lock()
8968 .map_err(|_| Error::WorkflowStatePoisoned)?;
8969
8970 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8971 return match recorded {
8972 RecordedCommand::SearchAttributes {
8973 sequence,
8974 attributes: recorded_attributes,
8975 attribute_types: recorded_attribute_types,
8976 } => {
8977 if recorded_attributes != attributes {
8978 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8979 "search_attribute_value_mismatch",
8980 Some(sequence),
8981 Some(recorded_attributes.to_string()),
8982 Some(attributes.to_string()),
8983 "search-attribute values differ from the recorded durable command",
8984 )));
8985 }
8986 if let RecordedSnapshotValue::Known(recorded_types) = recorded_attribute_types {
8987 if recorded_types != attribute_types {
8988 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8989 "search_attribute_type_mismatch",
8990 Some(sequence),
8991 Some(json!(recorded_types).to_string()),
8992 Some(json!(attribute_types).to_string()),
8993 "search-attribute declared types differ from the recorded durable command",
8994 )));
8995 }
8996 }
8997 state.command_cursor += 1;
8998 Ok(())
8999 }
9000 other => Err(command_mismatch(&other, "search-attribute update")),
9001 };
9002 }
9003
9004 let mut command = serde_json::Map::from_iter([
9005 ("type".to_string(), json!("upsert_search_attributes")),
9006 ("attributes".to_string(), attributes),
9007 ]);
9008 if !attribute_types.is_empty() {
9009 command.insert("attribute_types".to_string(), json!(attribute_types));
9010 }
9011 state.commands.push(Value::Object(command));
9012 Ok(())
9013 }
9014
9015 pub fn uuid_v4(&self) -> Result<Uuid> {
9017 self.side_effect(Uuid::new_v4)
9018 }
9019
9020 pub fn get_version(
9023 &self,
9024 change_id: impl Into<String>,
9025 min_supported: i32,
9026 max_supported: i32,
9027 ) -> Result<i32> {
9028 let change_id = change_id.into();
9029 if change_id.trim().is_empty() {
9030 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
9031 "version_change_id_invalid",
9032 None,
9033 Some("non-empty change ID".to_string()),
9034 Some(change_id),
9035 "version markers require a stable non-empty change ID",
9036 )));
9037 }
9038 if min_supported > max_supported {
9039 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
9040 "version_range_invalid",
9041 None,
9042 Some("min_supported <= max_supported".to_string()),
9043 Some(format!("{min_supported}..={max_supported}")),
9044 "version marker supported range is invalid",
9045 )));
9046 }
9047
9048 let mut state = self
9049 .state
9050 .lock()
9051 .map_err(|_| Error::WorkflowStatePoisoned)?;
9052 if let Some((version, sequence)) = state.version_markers.get(&change_id).copied() {
9053 ensure_version_supported(&change_id, version, min_supported, max_supported, sequence)?;
9054 return Ok(version);
9055 }
9056
9057 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
9058 return match recorded {
9059 RecordedCommand::VersionMarker {
9060 sequence,
9061 change_id: recorded_change_id,
9062 version,
9063 ..
9064 } => {
9065 if recorded_change_id != change_id {
9066 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
9067 "version_change_id_mismatch",
9068 Some(sequence),
9069 Some(recorded_change_id),
9070 Some(change_id),
9071 "recorded version marker change ID differs from current workflow code",
9072 )));
9073 }
9074 ensure_version_supported(
9075 &change_id,
9076 version,
9077 min_supported,
9078 max_supported,
9079 sequence,
9080 )?;
9081 state.command_cursor += 1;
9082 state.version_markers.insert(change_id, (version, sequence));
9083 Ok(version)
9084 }
9085 other => Err(command_mismatch(
9086 &other,
9087 format!("version marker:{change_id}"),
9088 )),
9089 };
9090 }
9091
9092 let version = max_supported;
9093 state.commands.push(json!({
9094 "type": "record_version_marker",
9095 "change_id": change_id,
9096 "version": version,
9097 "min_supported": min_supported,
9098 "max_supported": max_supported,
9099 }));
9100 state.version_markers.insert(change_id, (version, 0));
9103 Ok(version)
9104 }
9105
9106 pub fn patched(&self, change_id: impl Into<String>) -> Result<bool> {
9108 Ok(self.get_version(change_id, -1, 1)? == 1)
9109 }
9110
9111 pub fn deprecate_patch(&self, change_id: impl Into<String>) -> Result<()> {
9113 self.get_version(change_id, -1, 1).map(|_| ())
9114 }
9115
9116 pub fn upsert_memo<T: Serialize>(&self, entries: T) -> Result<()> {
9121 let entries = canonical_memo_entries(AvroValue::from_serialize(&entries)?, true)?;
9122 let mut state = self
9123 .state
9124 .lock()
9125 .map_err(|_| Error::WorkflowStatePoisoned)?;
9126
9127 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
9128 return match recorded {
9129 RecordedCommand::Memo {
9130 sequence,
9131 entries: recorded_entries,
9132 } => {
9133 if recorded_entries != entries {
9134 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
9135 "memo_update_mismatch",
9136 Some(sequence),
9137 Some(format!("{recorded_entries:?}")),
9138 Some(format!("{entries:?}")),
9139 "recorded memo entries differ from the current workflow update",
9140 )));
9141 }
9142 state.command_cursor += 1;
9143 Ok(())
9144 }
9145 other => Err(command_mismatch(&other, "memo upsert")),
9146 };
9147 }
9148
9149 let entries_envelope = encode_typed_envelope(&entries, DEFAULT_CODEC)?;
9150 state.commands.push(json!({
9151 "type": "upsert_memo",
9152 "entries": entries_envelope,
9153 }));
9154 Ok(())
9155 }
9156
9157 pub fn start_child_workflow<T: Serialize>(
9184 &self,
9185 workflow_type: impl Into<String>,
9186 options: ChildWorkflowOptions,
9187 args: T,
9188 ) -> ChildWorkflowCall {
9189 ChildWorkflowCall {
9190 ctx: self.clone(),
9191 workflow_type: workflow_type.into(),
9192 options,
9193 args: Some(AvroValue::from_serialize(&args)),
9194 scheduled: false,
9195 matched_pending: false,
9196 parallel_group_path: Vec::new(),
9197 }
9198 }
9199
9200 pub async fn start_child_workflow_avro_value<T: Serialize>(
9201 &self,
9202 workflow_type: impl Into<String>,
9203 options: ChildWorkflowOptions,
9204 args: T,
9205 ) -> Result<ChildWorkflowAvroResult> {
9206 let mut call = self.start_child_workflow(workflow_type, options, args);
9207 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
9208 }
9209
9210 fn take_commands(&self) -> Result<Vec<Value>> {
9211 let mut state = self
9212 .state
9213 .lock()
9214 .map_err(|_| Error::WorkflowStatePoisoned)?;
9215 Ok(std::mem::take(&mut state.commands))
9216 }
9217
9218 fn continue_as_new_command(&self, request: ContinueAsNewRequest) -> Result<Option<Value>> {
9219 let mut state = self
9220 .state
9221 .lock()
9222 .map_err(|_| Error::WorkflowStatePoisoned)?;
9223
9224 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
9225 return Err(command_mismatch(&recorded, "continue as new"));
9226 }
9227 if state.recorded_continue_as_new_sequence.is_some() {
9228 state.continue_as_new_consumed = true;
9229 return Ok(None);
9230 }
9231
9232 let arguments = encode_typed_envelope(&request.arguments, &state.payload_codec)?;
9233 let mut command = serde_json::Map::from_iter([
9234 ("type".to_string(), json!("continue_as_new")),
9235 ("arguments".to_string(), arguments),
9236 ("queue".to_string(), json!(state.task_queue.clone())),
9237 ]);
9238 if let Some(workflow_type) = request.options.workflow_type {
9239 command.insert("workflow_type".to_string(), json!(workflow_type));
9240 }
9241 if let Some(task_queue) = request.options.task_queue {
9242 command.insert("queue".to_string(), json!(task_queue));
9243 }
9244 Ok(Some(Value::Object(command)))
9245 }
9246
9247 fn matched_recorded_pending(&self) -> Result<bool> {
9248 let state = self
9249 .state
9250 .lock()
9251 .map_err(|_| Error::WorkflowStatePoisoned)?;
9252 Ok(state.matched_recorded_pending)
9253 }
9254
9255 fn ensure_history_consumed(&self) -> Result<()> {
9256 let state = self
9257 .state
9258 .lock()
9259 .map_err(|_| Error::WorkflowStatePoisoned)?;
9260 if let Some(command) = state.recorded_commands.get(state.command_cursor) {
9261 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
9262 "recorded_commands_unconsumed",
9263 Some(command.sequence()),
9264 Some(command.shape().to_string()),
9265 Some("workflow completion".to_string()),
9266 "workflow completed before consuming all recorded durable commands",
9267 )));
9268 }
9269 if let Some(sequence) = state
9270 .recorded_continue_as_new_sequence
9271 .filter(|_| !state.continue_as_new_consumed)
9272 {
9273 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
9274 "recorded_continue_as_new_unconsumed",
9275 Some(sequence),
9276 Some("continue as new".to_string()),
9277 Some("workflow completion".to_string()),
9278 "workflow completed without consuming its recorded continue-as-new transition",
9279 )));
9280 }
9281 Ok(())
9282 }
9283}
9284
9285fn contiguous_message_stream_count(
9286 pending: &[MessageStreamMessage],
9287 cursor: u64,
9288 max_items: usize,
9289) -> usize {
9290 pending
9291 .iter()
9292 .take(max_items)
9293 .enumerate()
9294 .take_while(|(offset, message)| {
9295 u64::try_from(*offset)
9296 .ok()
9297 .and_then(|offset| cursor.checked_add(offset + 1))
9298 == Some(message.position)
9299 })
9300 .count()
9301}
9302
9303fn is_authored_command_open_event(event: &HistoryEvent) -> bool {
9304 matches!(
9305 event.event_type.as_str(),
9306 "ActivityScheduled"
9307 | "TimerScheduled"
9308 | "ChildWorkflowScheduled"
9309 | "SignalWaitOpened"
9310 | "ConditionWaitOpened"
9311 | "SearchAttributesUpserted"
9312 | "SideEffectRecorded"
9313 | "VersionMarkerRecorded"
9314 | "MemoUpserted"
9315 | "WorkflowContinuedAsNew"
9316 )
9317}
9318
9319#[derive(Debug)]
9320struct WorkflowState {
9321 workflow_id: Option<String>,
9322 run_id: Option<String>,
9323 task_queue: String,
9324 payload_codec: String,
9325 history_events: Arc<Vec<HistoryEvent>>,
9326 history_budget: WorkflowHistoryBudget,
9327 cancel_requested: bool,
9328 resume_signal: Option<ResumeSignal>,
9329 recorded_commands: Vec<RecordedCommand>,
9330 selection_markers: Vec<SelectionMarker>,
9331 selection_marker_cursor: usize,
9332 cancelled_selection_members: Vec<SelectionCancellation>,
9333 recorded_continue_as_new_sequence: Option<u64>,
9334 continue_as_new_consumed: bool,
9335 command_cursor: usize,
9336 condition_wait_occurrence_counter: u64,
9337 matched_recorded_pending: bool,
9338 version_markers: HashMap<String, (i32, u64)>,
9339 workflow_command_identity: String,
9340 workflow_stream_command_counter: u64,
9341 commands: Vec<Value>,
9342 message_stream_messages: HashMap<String, Vec<MessageStreamMessage>>,
9343 message_stream_cursors: HashMap<String, u64>,
9344 message_stream_waits: HashMap<String, u64>,
9345}
9346
9347impl WorkflowState {
9348 #[cfg(test)]
9349 fn new(
9350 history: Vec<HistoryEvent>,
9351 task_queue: String,
9352 payload_codec: String,
9353 resume_signal: Option<ResumeSignal>,
9354 ) -> Result<Self> {
9355 Self::new_with_identity(
9356 history,
9357 None,
9358 None,
9359 task_queue,
9360 payload_codec,
9361 resume_signal,
9362 )
9363 }
9364
9365 fn new_with_identity(
9366 history: Vec<HistoryEvent>,
9367 workflow_id: Option<String>,
9368 run_id: Option<String>,
9369 task_queue: String,
9370 payload_codec: String,
9371 resume_signal: Option<ResumeSignal>,
9372 ) -> Result<Self> {
9373 let recorded_commands = recorded_commands(
9374 &history,
9375 &payload_codec,
9376 WorkflowIdentity {
9377 workflow_id: workflow_id.clone(),
9378 run_id: run_id.clone(),
9379 },
9380 )?;
9381 let selection_markers = recorded_selection_markers(&history)?;
9382 let cancelled_selection_members = recorded_selection_cancellations(&history)?;
9383 let recorded_continue_as_new = history
9384 .iter()
9385 .filter(|event| event.event_type == "WorkflowContinuedAsNew")
9386 .collect::<Vec<_>>();
9387 if recorded_continue_as_new.len() > 1 {
9388 return Err(invalid_recorded_history(
9389 "duplicate_continue_as_new_transition",
9390 recorded_continue_as_new
9391 .last()
9392 .and_then(|event| durable_event_sequence(event))
9393 .unwrap_or(0),
9394 "one WorkflowContinuedAsNew event",
9395 &format!(
9396 "{} WorkflowContinuedAsNew events",
9397 recorded_continue_as_new.len()
9398 ),
9399 "workflow history records one continue-as-new transition more than once",
9400 ));
9401 }
9402 let recorded_continue_as_new_sequence = recorded_continue_as_new
9403 .first()
9404 .map(|event| {
9405 durable_event_sequence(event).ok_or_else(|| {
9406 Error::NonDeterministicReplay(ReplayFailure::new(
9407 "continue_as_new_sequence_missing",
9408 None,
9409 Some("recorded transition sequence".to_string()),
9410 Some("missing sequence".to_string()),
9411 "WorkflowContinuedAsNew history is missing its recorded sequence",
9412 ))
9413 })
9414 })
9415 .transpose()?;
9416 let mut message_stream_cursors = HashMap::new();
9417 for event in &history {
9418 if !matches!(
9419 event.event_type.as_str(),
9420 "SignalReceived" | "SignalApplied"
9421 ) || event.payload.get("signal_name").and_then(Value::as_str)
9422 != Some(MESSAGE_STREAM_SIGNAL)
9423 {
9424 continue;
9425 }
9426 let arguments = decode_signal_event_arguments(event, &payload_codec)?;
9427 if arguments.len() != 1 {
9428 continue;
9429 }
9430 let envelope = arguments[0].clone().into_json()?;
9431 let Some(envelope) = envelope.as_object() else {
9432 continue;
9433 };
9434 if envelope.get("schema").and_then(Value::as_str) != Some(MESSAGE_STREAM_CURSOR_SCHEMA)
9435 {
9436 continue;
9437 }
9438 let Some(stream_name) = envelope.get("stream_name").and_then(Value::as_str) else {
9439 continue;
9440 };
9441 let Some(through_position) = envelope.get("through_position").and_then(Value::as_u64)
9442 else {
9443 continue;
9444 };
9445 let cursor = message_stream_cursors
9446 .entry(stream_name.to_string())
9447 .or_insert(0);
9448 *cursor = (*cursor).max(through_position);
9449 }
9450 let event_count = u64::try_from(history.len()).unwrap_or(u64::MAX);
9451 let cancel_requested = history.iter().any(|event| {
9452 matches!(
9453 event.event_type.as_str(),
9454 "WorkflowCancellationRequested" | "WorkflowCancelRequested"
9455 )
9456 });
9457 Ok(Self {
9458 workflow_command_identity: String::new(),
9459 workflow_stream_command_counter: 0,
9460 workflow_id,
9461 run_id,
9462 task_queue,
9463 payload_codec,
9464 history_events: Arc::new(history),
9465 history_budget: WorkflowHistoryBudget {
9466 event_count,
9467 ..WorkflowHistoryBudget::default()
9468 },
9469 cancel_requested,
9470 resume_signal,
9471 recorded_commands,
9472 selection_markers,
9473 selection_marker_cursor: 0,
9474 cancelled_selection_members,
9475 recorded_continue_as_new_sequence,
9476 continue_as_new_consumed: false,
9477 command_cursor: 0,
9478 condition_wait_occurrence_counter: 0,
9479 matched_recorded_pending: false,
9480 version_markers: HashMap::new(),
9481 commands: Vec::new(),
9482 message_stream_messages: HashMap::new(),
9483 message_stream_cursors,
9484 message_stream_waits: HashMap::new(),
9485 })
9486 }
9487}
9488
9489enum MessageStreamDelivery {
9490 Message(MessageStreamMessage),
9491 Cursor {
9492 stream_name: String,
9493 through_position: u64,
9494 },
9495}
9496
9497fn decode_message_stream_delivery(arguments: Vec<Value>) -> Result<Option<MessageStreamDelivery>> {
9498 if arguments.len() != 1 {
9499 return Ok(None);
9500 }
9501 let envelope = arguments
9502 .into_iter()
9503 .next()
9504 .expect("one argument was checked");
9505 let Some(envelope) = envelope.as_object() else {
9506 return Ok(None);
9507 };
9508 let Some(stream_name) = envelope.get("stream_name").and_then(Value::as_str) else {
9509 return Ok(None);
9510 };
9511 if envelope.get("schema").and_then(Value::as_str) == Some(MESSAGE_STREAM_CURSOR_SCHEMA) {
9512 let Some(through_position) = envelope.get("through_position").and_then(Value::as_u64)
9513 else {
9514 return Ok(None);
9515 };
9516 return Ok(Some(MessageStreamDelivery::Cursor {
9517 stream_name: stream_name.to_string(),
9518 through_position,
9519 }));
9520 }
9521 if envelope.get("schema").and_then(Value::as_str) != Some(MESSAGE_STREAM_SCHEMA) {
9522 return Ok(None);
9523 }
9524 let Some(message_id) = envelope.get("message_id").and_then(Value::as_str) else {
9525 return Ok(None);
9526 };
9527 let Some(position) = envelope
9528 .get("position")
9529 .and_then(Value::as_u64)
9530 .filter(|value| *value > 0)
9531 else {
9532 return Ok(None);
9533 };
9534 let Some(payload_envelope) = envelope.get("payload_envelope") else {
9535 return Ok(None);
9536 };
9537 let Ok(payload_envelope) = serde_json::from_value::<PayloadEnvelope>(payload_envelope.clone())
9538 else {
9539 return Ok(None);
9540 };
9541 let decoded = decode_avro_value(&payload_envelope)?;
9542 let AvroValue::Array(values) = decoded else {
9543 return Ok(None);
9544 };
9545 Ok(Some(MessageStreamDelivery::Message(MessageStreamMessage {
9546 stream_name: stream_name.to_string(),
9547 message_id: message_id.to_string(),
9548 position,
9549 arguments: values,
9550 })))
9551}
9552
9553#[derive(Clone, Debug)]
9554enum RecordedCommand {
9555 Activity {
9556 sequence: u64,
9557 activity_type: Option<String>,
9558 options: Option<RecordedActivityOptions>,
9559 outcome: Option<ActivityOutcome>,
9560 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9561 },
9562 Timer {
9563 sequence: u64,
9564 delay_seconds: u64,
9565 fired: bool,
9566 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9567 },
9568 ChildWorkflow {
9569 sequence: u64,
9570 workflow_type: Option<String>,
9571 outcome: Option<ChildWorkflowOutcome>,
9572 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9573 },
9574 SignalWait {
9575 sequence: u64,
9576 signal_name: String,
9577 value: Option<Vec<AvroValue>>,
9578 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9579 },
9580 ConditionWait {
9581 sequence: u64,
9582 occurrence_id: String,
9583 condition_key: Option<String>,
9584 predicate_identity: String,
9585 timeout_seconds: Option<u64>,
9586 result: Option<ConditionWaitResult>,
9587 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9588 },
9589 SearchAttributes {
9590 sequence: u64,
9591 attributes: Value,
9592 attribute_types: RecordedSnapshotValue<BTreeMap<String, String>>,
9593 },
9594 SideEffect {
9595 sequence: u64,
9596 value: AvroValue,
9597 },
9598 VersionMarker {
9599 sequence: u64,
9600 change_id: String,
9601 version: i32,
9602 },
9603 Memo {
9604 sequence: u64,
9605 entries: AvroValue,
9606 },
9607}
9608
9609#[derive(Clone, Debug, PartialEq, Eq)]
9610struct SelectionMarker {
9611 selection_group_id: String,
9612 selection_group_base_sequence: u64,
9613 selection_group_size: usize,
9614 member_key: SelectionKey,
9615 member_index: usize,
9616 member_base_sequence: u64,
9617 member_size: usize,
9618 operation_kind: String,
9619 operation_identity: String,
9620 outcome: String,
9621 resolution_event_id: String,
9622 resolution_event_type: String,
9623}
9624
9625#[derive(Clone, Debug, PartialEq, Eq)]
9626struct SelectionCancellation {
9627 selection_group_id: String,
9628 member_key: SelectionKey,
9629 member_index: usize,
9630 member_base_sequence: u64,
9631 member_size: usize,
9632 operation_kind: String,
9633 operation_identity: String,
9634}
9635
9636fn recorded_selection_markers(events: &[HistoryEvent]) -> Result<Vec<SelectionMarker>> {
9637 let mut markers: Vec<SelectionMarker> = Vec::new();
9638 for event in events
9639 .iter()
9640 .filter(|event| event.event_type == "SelectionResolved")
9641 {
9642 let payload = &event.payload;
9643 let base_sequence = required_selection_u64(payload, "selection_group_base_sequence")?;
9644 let group_size = required_selection_usize(payload, "selection_group_size")?;
9645 let member_base_sequence = required_selection_u64(payload, "member_base_sequence")?;
9646 let member_size = required_selection_usize(payload, "member_size")?;
9647 let member_index = required_selection_usize_allow_zero(payload, "member_index")?;
9648 let group_id = payload_string(payload, "selection_group_id").ok_or_else(|| {
9649 invalid_recorded_history(
9650 "selection_marker_invalid",
9651 base_sequence,
9652 "non-empty selection_group_id",
9653 &payload.to_string(),
9654 "selection winner history is missing its durable group identity",
9655 )
9656 })?;
9657 let expected_group_id = format!("select-calls:{base_sequence}:{group_size}");
9658 if group_id != expected_group_id {
9659 return Err(invalid_recorded_history(
9660 "selection_marker_invalid",
9661 base_sequence,
9662 &expected_group_id,
9663 &group_id,
9664 "selection winner history contains an incompatible group identity",
9665 ));
9666 }
9667 let group_end = base_sequence
9668 .checked_add(u64::try_from(group_size).unwrap_or(u64::MAX))
9669 .unwrap_or(u64::MAX);
9670 let member_end = member_base_sequence
9671 .checked_add(u64::try_from(member_size).unwrap_or(u64::MAX))
9672 .unwrap_or(u64::MAX);
9673 if member_index >= group_size
9674 || member_base_sequence < base_sequence
9675 || member_end > group_end
9676 {
9677 return Err(invalid_recorded_history(
9678 "selection_marker_invalid",
9679 base_sequence,
9680 "winner member within selection group bounds",
9681 &payload.to_string(),
9682 "selection winner history contains an invalid member range",
9683 ));
9684 }
9685 let operation_kind = payload_string(payload, "operation_kind").ok_or_else(|| {
9686 invalid_recorded_history(
9687 "selection_marker_invalid",
9688 base_sequence,
9689 "selection operation kind",
9690 &payload.to_string(),
9691 "selection winner history is missing its operation kind",
9692 )
9693 })?;
9694 if !matches!(
9695 operation_kind.as_str(),
9696 "activity" | "child" | "timer" | "signal" | "condition" | "group"
9697 ) {
9698 return Err(invalid_recorded_history(
9699 "selection_marker_invalid",
9700 base_sequence,
9701 "activity, child, timer, signal, condition, or group",
9702 &operation_kind,
9703 "selection winner history contains an unsupported operation kind",
9704 ));
9705 }
9706 let operation_identity =
9707 payload_string(payload, "operation_identity").ok_or_else(|| {
9708 invalid_recorded_history(
9709 "selection_marker_invalid",
9710 base_sequence,
9711 "non-empty operation identity",
9712 &payload.to_string(),
9713 "selection winner history is missing its durable operation identity",
9714 )
9715 })?;
9716 let outcome = payload_string(payload, "outcome").ok_or_else(|| {
9717 invalid_recorded_history(
9718 "selection_marker_invalid",
9719 base_sequence,
9720 "completed or failed selection outcome",
9721 &payload.to_string(),
9722 "selection winner history is missing its outcome",
9723 )
9724 })?;
9725 if !matches!(outcome.as_str(), "completed" | "failed") {
9726 return Err(invalid_recorded_history(
9727 "selection_marker_invalid",
9728 base_sequence,
9729 "completed or failed selection outcome",
9730 &outcome,
9731 "selection winner history contains an unsupported outcome",
9732 ));
9733 }
9734 let marker = SelectionMarker {
9735 selection_group_id: group_id,
9736 selection_group_base_sequence: base_sequence,
9737 selection_group_size: group_size,
9738 member_key: selection_key_from_value(payload.get("member_key"), base_sequence)?,
9739 member_index,
9740 member_base_sequence,
9741 member_size,
9742 operation_kind,
9743 operation_identity,
9744 outcome,
9745 resolution_event_id: payload_string(payload, "resolution_event_id").ok_or_else(
9746 || {
9747 invalid_recorded_history(
9748 "selection_marker_invalid",
9749 base_sequence,
9750 "durable resolution_event_id",
9751 &payload.to_string(),
9752 "selection winner history is missing its terminal event identity",
9753 )
9754 },
9755 )?,
9756 resolution_event_type: payload_string(payload, "resolution_event_type").ok_or_else(
9757 || {
9758 invalid_recorded_history(
9759 "selection_marker_invalid",
9760 base_sequence,
9761 "durable resolution_event_type",
9762 &payload.to_string(),
9763 "selection winner history is missing its terminal event type",
9764 )
9765 },
9766 )?,
9767 };
9768 if let Some(existing) = markers
9769 .iter()
9770 .find(|existing| existing.selection_group_id == marker.selection_group_id)
9771 {
9772 if existing != &marker {
9773 return Err(invalid_recorded_history(
9774 "selection_marker_conflict",
9775 base_sequence,
9776 &format!("one winner for {}", marker.selection_group_id),
9777 &payload.to_string(),
9778 "selection history records conflicting winners for one durable group",
9779 ));
9780 }
9781 continue;
9782 }
9783 markers.push(marker);
9784 }
9785 Ok(markers)
9786}
9787
9788fn recorded_selection_cancellations(events: &[HistoryEvent]) -> Result<Vec<SelectionCancellation>> {
9789 let mut cancelled: Vec<SelectionCancellation> = Vec::new();
9790 for event in events
9791 .iter()
9792 .filter(|event| event.event_type == "SelectionOperationCancelled")
9793 {
9794 let group_id = payload_string(&event.payload, "selection_group_id").ok_or_else(|| {
9795 invalid_recorded_history(
9796 "selection_cancellation_invalid",
9797 0,
9798 "non-empty selection_group_id",
9799 &event.payload.to_string(),
9800 "selection cancellation history is missing its group identity",
9801 )
9802 })?;
9803 let member_base_sequence = required_selection_u64(&event.payload, "member_base_sequence")?;
9804 let marker = SelectionCancellation {
9805 selection_group_id: group_id,
9806 member_key: selection_key_from_value(
9807 event.payload.get("member_key"),
9808 member_base_sequence,
9809 )?,
9810 member_index: required_selection_usize_allow_zero(&event.payload, "member_index")?,
9811 member_base_sequence,
9812 member_size: required_selection_usize(&event.payload, "member_size")?,
9813 operation_kind: payload_string(&event.payload, "operation_kind").ok_or_else(|| {
9814 invalid_recorded_history(
9815 "selection_cancellation_invalid",
9816 member_base_sequence,
9817 "selection operation kind",
9818 &event.payload.to_string(),
9819 "selection cancellation is missing its operation kind",
9820 )
9821 })?,
9822 operation_identity: payload_string(&event.payload, "operation_identity").ok_or_else(
9823 || {
9824 invalid_recorded_history(
9825 "selection_cancellation_invalid",
9826 member_base_sequence,
9827 "selection operation identity",
9828 &event.payload.to_string(),
9829 "selection cancellation is missing its operation identity",
9830 )
9831 },
9832 )?,
9833 };
9834 if let Some(existing) = cancelled.iter().find(|recorded| {
9835 recorded.selection_group_id == marker.selection_group_id
9836 && recorded.member_base_sequence == marker.member_base_sequence
9837 }) {
9838 if existing != &marker {
9839 return Err(invalid_recorded_history(
9840 "selection_cancellation_conflict",
9841 member_base_sequence,
9842 "one stable SelectionOperationCancelled marker",
9843 &event.payload.to_string(),
9844 "selection cancellation history contains conflicting member metadata",
9845 ));
9846 }
9847 } else {
9848 cancelled.push(marker);
9849 }
9850 }
9851 Ok(cancelled)
9852}
9853
9854fn required_selection_u64(payload: &Value, field: &str) -> Result<u64> {
9855 payload
9856 .get(field)
9857 .and_then(value_as_u64)
9858 .filter(|value| *value > 0)
9859 .ok_or_else(|| {
9860 invalid_recorded_history(
9861 "selection_marker_invalid",
9862 0,
9863 &format!("positive integer {field}"),
9864 &payload.to_string(),
9865 "selection history contains invalid durable identity metadata",
9866 )
9867 })
9868}
9869
9870fn required_selection_usize(payload: &Value, field: &str) -> Result<usize> {
9871 required_selection_usize_allow_zero(payload, field).and_then(|value| {
9872 if value > 0 {
9873 Ok(value)
9874 } else {
9875 Err(invalid_recorded_history(
9876 "selection_marker_invalid",
9877 0,
9878 &format!("positive integer {field}"),
9879 &payload.to_string(),
9880 "selection history contains invalid durable identity metadata",
9881 ))
9882 }
9883 })
9884}
9885
9886fn required_selection_usize_allow_zero(payload: &Value, field: &str) -> Result<usize> {
9887 payload
9888 .get(field)
9889 .and_then(value_as_u64)
9890 .and_then(|value| usize::try_from(value).ok())
9891 .ok_or_else(|| {
9892 invalid_recorded_history(
9893 "selection_marker_invalid",
9894 0,
9895 &format!("non-negative integer {field}"),
9896 &payload.to_string(),
9897 "selection history contains invalid durable identity metadata",
9898 )
9899 })
9900}
9901
9902#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9903struct RecordedActivityOptions {
9904 task_queue: RecordedSnapshotValue<Option<String>>,
9905 execution_mode: RecordedSnapshotValue<Option<String>>,
9906 retry_policy: ActivityRetrySnapshot,
9907}
9908
9909#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9910enum RecordedSnapshotValue<T> {
9911 Unknown,
9913 Known(T),
9914}
9915
9916impl<T: PartialEq> RecordedSnapshotValue<T> {
9917 fn matches_current(&self, current: &Self) -> bool {
9918 match self {
9919 Self::Unknown => true,
9920 Self::Known(recorded) => matches!(current, Self::Known(value) if value == recorded),
9921 }
9922 }
9923}
9924
9925#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9926struct ActivityRetrySnapshot {
9927 snapshot_version: RecordedSnapshotValue<Option<u64>>,
9928 max_attempts: RecordedSnapshotValue<Option<u64>>,
9929 backoff_seconds: RecordedSnapshotValue<Vec<u64>>,
9930 start_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
9931 schedule_to_start_timeout: RecordedSnapshotValue<Option<u64>>,
9932 schedule_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
9933 heartbeat_timeout: RecordedSnapshotValue<Option<u64>>,
9934 non_retryable_error_types: RecordedSnapshotValue<Vec<String>>,
9935}
9936
9937impl ActivityRetrySnapshot {
9938 fn matches_current(&self, current: &Self) -> bool {
9939 self.snapshot_version
9940 .matches_current(¤t.snapshot_version)
9941 && self.max_attempts.matches_current(¤t.max_attempts)
9942 && self
9943 .backoff_seconds
9944 .matches_current(¤t.backoff_seconds)
9945 && self
9946 .start_to_close_timeout
9947 .matches_current(¤t.start_to_close_timeout)
9948 && self
9949 .schedule_to_start_timeout
9950 .matches_current(¤t.schedule_to_start_timeout)
9951 && self
9952 .schedule_to_close_timeout
9953 .matches_current(¤t.schedule_to_close_timeout)
9954 && self
9955 .heartbeat_timeout
9956 .matches_current(¤t.heartbeat_timeout)
9957 && self
9958 .non_retryable_error_types
9959 .matches_current(¤t.non_retryable_error_types)
9960 }
9961}
9962
9963fn recorded_optional_u64(
9964 object: Option<&serde_json::Map<String, Value>>,
9965 field: &str,
9966) -> RecordedSnapshotValue<Option<u64>> {
9967 match object.and_then(|object| object.get(field)) {
9968 None => RecordedSnapshotValue::Unknown,
9969 Some(Value::Null) => RecordedSnapshotValue::Known(None),
9970 Some(value) => RecordedSnapshotValue::Known(value_as_u64(value)),
9971 }
9972}
9973
9974fn recorded_optional_string(
9975 object: &serde_json::Map<String, Value>,
9976 field: &str,
9977) -> RecordedSnapshotValue<Option<String>> {
9978 match object.get(field) {
9979 None => RecordedSnapshotValue::Unknown,
9980 Some(Value::Null) => RecordedSnapshotValue::Known(None),
9981 Some(value) => RecordedSnapshotValue::Known(value.as_str().map(str::to_string)),
9982 }
9983}
9984
9985fn recorded_activity_retry_snapshot(policy: Option<&Value>) -> ActivityRetrySnapshot {
9986 let policy = policy.and_then(Value::as_object);
9987 let backoff_seconds = policy
9988 .and_then(|policy| policy.get("backoff_seconds"))
9989 .and_then(Value::as_array)
9990 .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
9991 .map_or(RecordedSnapshotValue::Unknown, RecordedSnapshotValue::Known);
9992 let mut non_retryable_error_types = Vec::new();
9993 for error_type in policy
9994 .and_then(|policy| policy.get("non_retryable_error_types"))
9995 .and_then(Value::as_array)
9996 .into_iter()
9997 .flatten()
9998 .filter_map(Value::as_str)
9999 .map(str::trim)
10000 .filter(|error_type| !error_type.is_empty())
10001 {
10002 if !non_retryable_error_types
10003 .iter()
10004 .any(|recorded| recorded == error_type)
10005 {
10006 non_retryable_error_types.push(error_type.to_string());
10007 }
10008 }
10009
10010 ActivityRetrySnapshot {
10011 snapshot_version: recorded_optional_u64(policy, "snapshot_version"),
10012 max_attempts: recorded_optional_u64(policy, "max_attempts"),
10013 backoff_seconds,
10014 start_to_close_timeout: recorded_optional_u64(policy, "start_to_close_timeout"),
10015 schedule_to_start_timeout: recorded_optional_u64(policy, "schedule_to_start_timeout"),
10016 schedule_to_close_timeout: recorded_optional_u64(policy, "schedule_to_close_timeout"),
10017 heartbeat_timeout: recorded_optional_u64(policy, "heartbeat_timeout"),
10018 non_retryable_error_types: if policy
10019 .is_some_and(|policy| policy.contains_key("non_retryable_error_types"))
10020 {
10021 RecordedSnapshotValue::Known(non_retryable_error_types)
10022 } else {
10023 RecordedSnapshotValue::Unknown
10024 },
10025 }
10026}
10027
10028fn current_activity_retry_snapshot(options: &ValidatedActivityOptions) -> ActivityRetrySnapshot {
10029 let policy = options.retry_policy.as_ref();
10030 let max_attempts = match policy.and_then(|policy| policy.get("max_attempts")) {
10031 Some(Value::Null) => None,
10032 Some(value) => value_as_u64(value),
10033 None => Some(1),
10034 };
10035 let backoff_seconds = policy
10036 .and_then(|policy| policy.get("backoff_seconds"))
10037 .and_then(Value::as_array)
10038 .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
10039 .unwrap_or_default();
10040 let non_retryable_error_types = policy
10041 .and_then(|policy| policy.get("non_retryable_error_types"))
10042 .and_then(Value::as_array)
10043 .into_iter()
10044 .flatten()
10045 .filter_map(Value::as_str)
10046 .map(str::to_string)
10047 .collect();
10048
10049 ActivityRetrySnapshot {
10050 snapshot_version: RecordedSnapshotValue::Known(Some(1)),
10051 max_attempts: RecordedSnapshotValue::Known(max_attempts),
10052 backoff_seconds: RecordedSnapshotValue::Known(backoff_seconds),
10053 start_to_close_timeout: RecordedSnapshotValue::Known(options.start_to_close_timeout),
10054 schedule_to_start_timeout: RecordedSnapshotValue::Known(options.schedule_to_start_timeout),
10055 schedule_to_close_timeout: RecordedSnapshotValue::Known(options.schedule_to_close_timeout),
10056 heartbeat_timeout: RecordedSnapshotValue::Known(options.heartbeat_timeout),
10057 non_retryable_error_types: RecordedSnapshotValue::Known(non_retryable_error_types),
10058 }
10059}
10060
10061fn activity_options_description(options: &RecordedActivityOptions) -> String {
10062 serde_json::to_string(options).unwrap_or_else(|_| format!("{options:?}"))
10063}
10064
10065impl RecordedCommand {
10066 fn sequence(&self) -> u64 {
10067 match self {
10068 Self::Activity { sequence, .. }
10069 | Self::Timer { sequence, .. }
10070 | Self::ChildWorkflow { sequence, .. }
10071 | Self::SignalWait { sequence, .. }
10072 | Self::ConditionWait { sequence, .. }
10073 | Self::SearchAttributes { sequence, .. }
10074 | Self::SideEffect { sequence, .. }
10075 | Self::VersionMarker { sequence, .. }
10076 | Self::Memo { sequence, .. } => *sequence,
10077 }
10078 }
10079
10080 fn shape(&self) -> &'static str {
10081 match self {
10082 Self::Activity { .. } => "activity",
10083 Self::Timer { .. } => "timer",
10084 Self::ChildWorkflow { .. } => "child workflow",
10085 Self::SignalWait { .. } => "signal wait",
10086 Self::ConditionWait { .. } => "condition wait",
10087 Self::SearchAttributes { .. } => "search-attribute update",
10088 Self::SideEffect { .. } => "side effect",
10089 Self::VersionMarker { .. } => "version marker",
10090 Self::Memo { .. } => "memo upsert",
10091 }
10092 }
10093}
10094
10095fn ensure_version_supported(
10096 change_id: &str,
10097 version: i32,
10098 min_supported: i32,
10099 max_supported: i32,
10100 sequence: u64,
10101) -> Result<()> {
10102 if (min_supported..=max_supported).contains(&version) {
10103 return Ok(());
10104 }
10105 Err(Error::NonDeterministicReplay(ReplayFailure::new(
10106 "version_marker_incompatible_range",
10107 (sequence != 0).then_some(sequence),
10108 Some(format!("{min_supported}..={max_supported}")),
10109 Some(format!("{change_id}:{version}")),
10110 "recorded workflow version is outside the range supported by current code",
10111 )))
10112}
10113
10114#[derive(Clone, Debug)]
10115struct ResumeSignal {
10116 signal_name: String,
10117 arguments: Vec<AvroValue>,
10118}
10119
10120const MAX_PARALLEL_OPERATIONS: usize = 1000;
10121
10122fn parallel_group_prefix(kind: &str) -> &'static str {
10123 match kind {
10124 "activity" => "parallel-activities",
10125 "child" => "parallel-children",
10126 "timer" => "parallel-timers",
10127 _ => "parallel-calls",
10128 }
10129}
10130
10131fn parallel_group_entry(
10132 base_sequence: u64,
10133 size: usize,
10134 index: usize,
10135 kind: &str,
10136) -> ParallelGroupMetadata {
10137 ParallelGroupMetadata {
10138 parallel_group_id: format!("{}:{base_sequence}:{size}", parallel_group_prefix(kind)),
10139 parallel_group_kind: kind.to_string(),
10140 parallel_group_base_sequence: base_sequence,
10141 parallel_group_size: size,
10142 parallel_group_index: index,
10143 parallel_group_mode: None,
10144 selection_member_key: None,
10145 selection_member_index: None,
10146 selection_member_base_sequence: None,
10147 selection_member_size: None,
10148 selection_member_kind: None,
10149 }
10150}
10151
10152struct SelectionMemberMetadata {
10153 key: SelectionKey,
10154 index: usize,
10155 base_sequence: u64,
10156 size: usize,
10157 kind: String,
10158}
10159
10160fn selection_group_entry(
10161 base_sequence: u64,
10162 size: usize,
10163 index: usize,
10164 kind: &str,
10165 member: &SelectionMemberMetadata,
10166) -> ParallelGroupMetadata {
10167 ParallelGroupMetadata {
10168 parallel_group_id: format!("select-calls:{base_sequence}:{size}"),
10169 parallel_group_kind: kind.to_string(),
10170 parallel_group_base_sequence: base_sequence,
10171 parallel_group_size: size,
10172 parallel_group_index: index,
10173 parallel_group_mode: Some("select".to_string()),
10174 selection_member_key: Some(member.key.clone()),
10175 selection_member_index: Some(member.index),
10176 selection_member_base_sequence: Some(member.base_sequence),
10177 selection_member_size: Some(member.size),
10178 selection_member_kind: Some(member.kind.clone()),
10179 }
10180}
10181
10182fn apply_parallel_group_path(
10183 command: &mut serde_json::Map<String, Value>,
10184 path: &[ParallelGroupMetadata],
10185) {
10186 let Some(inner) = path.last() else {
10187 return;
10188 };
10189 command.insert(
10190 "parallel_group_id".to_string(),
10191 json!(inner.parallel_group_id),
10192 );
10193 command.insert(
10194 "parallel_group_kind".to_string(),
10195 json!(inner.parallel_group_kind),
10196 );
10197 command.insert(
10198 "parallel_group_base_sequence".to_string(),
10199 json!(inner.parallel_group_base_sequence),
10200 );
10201 command.insert(
10202 "parallel_group_size".to_string(),
10203 json!(inner.parallel_group_size),
10204 );
10205 command.insert(
10206 "parallel_group_index".to_string(),
10207 json!(inner.parallel_group_index),
10208 );
10209 if let Some(mode) = &inner.parallel_group_mode {
10210 command.insert("parallel_group_mode".to_string(), json!(mode));
10211 }
10212 if let Some(key) = &inner.selection_member_key {
10213 command.insert("selection_member_key".to_string(), json!(key));
10214 }
10215 if let Some(index) = inner.selection_member_index {
10216 command.insert("selection_member_index".to_string(), json!(index));
10217 }
10218 if let Some(base_sequence) = inner.selection_member_base_sequence {
10219 command.insert(
10220 "selection_member_base_sequence".to_string(),
10221 json!(base_sequence),
10222 );
10223 }
10224 if let Some(size) = inner.selection_member_size {
10225 command.insert("selection_member_size".to_string(), json!(size));
10226 }
10227 if let Some(kind) = &inner.selection_member_kind {
10228 command.insert("selection_member_kind".to_string(), json!(kind));
10229 }
10230 command.insert("parallel_group_path".to_string(), json!(path));
10231}
10232
10233fn ensure_parallel_path_matches(
10234 sequence: u64,
10235 recorded: Option<&[ParallelGroupMetadata]>,
10236 expected: &[ParallelGroupMetadata],
10237) -> Result<()> {
10238 match (recorded, expected.is_empty()) {
10239 (None, true) => Ok(()),
10240 (Some(recorded), false) if recorded == expected => Ok(()),
10241 (None, false) => Err(invalid_recorded_history(
10242 "parallel_group_metadata_missing",
10243 sequence,
10244 &serde_json::to_string(expected).unwrap_or_default(),
10245 "<missing>",
10246 "recorded parallel member is missing its durable group path",
10247 )),
10248 (Some(recorded), true) => Err(invalid_recorded_history(
10249 "parallel_group_shape_mismatch",
10250 sequence,
10251 "sequential command",
10252 &serde_json::to_string(recorded).unwrap_or_default(),
10253 "recorded command belonged to a parallel group but current code schedules it sequentially",
10254 )),
10255 (Some(recorded), false) => Err(invalid_recorded_history(
10256 "parallel_group_shape_mismatch",
10257 sequence,
10258 &serde_json::to_string(recorded).unwrap_or_default(),
10259 &serde_json::to_string(expected).unwrap_or_default(),
10260 "recorded parallel-group identity or path changed during replay",
10261 )),
10262 }
10263}
10264
10265#[derive(Clone, Debug)]
10266enum ParallelShape {
10267 Leaf,
10268 Group(Vec<ParallelShape>),
10269}
10270
10271struct ParallelDescriptor {
10272 operation: ParallelOperation,
10273 offset: usize,
10274 member_path: Vec<usize>,
10275 group_path: Vec<ParallelGroupMetadata>,
10276}
10277
10278fn parallel_leaf_count(operations: &[ParallelOperation]) -> usize {
10279 operations
10280 .iter()
10281 .map(|operation| match operation {
10282 ParallelOperation::Group(children) => parallel_leaf_count(children),
10283 _ => 1,
10284 })
10285 .sum()
10286}
10287
10288fn parallel_operation_kind(operation: &ParallelOperation) -> Option<&'static str> {
10289 match operation {
10290 ParallelOperation::Activity { .. } => Some("activity"),
10291 ParallelOperation::ChildWorkflow { .. } => Some("child"),
10292 ParallelOperation::Timer(_) => Some("timer"),
10293 ParallelOperation::Signal(_) => Some("signal"),
10294 ParallelOperation::Condition { .. } => Some("condition"),
10295 ParallelOperation::Group(children) => parallel_group_kind(children),
10296 }
10297}
10298
10299fn parallel_group_kind(operations: &[ParallelOperation]) -> Option<&'static str> {
10300 let mut kind = None;
10301 for operation in operations {
10302 let Some(operation_kind) = parallel_operation_kind(operation) else {
10303 continue;
10304 };
10305 match kind {
10306 None => kind = Some(operation_kind),
10307 Some(current) if current == operation_kind => {}
10308 Some(_) => return Some("mixed"),
10309 }
10310 }
10311 kind
10312}
10313
10314fn validate_parallel_operations(
10315 operations: &[ParallelOperation],
10316 member_path: &mut Vec<usize>,
10317 root: bool,
10318) -> Result<()> {
10319 let leaves = parallel_leaf_count(operations);
10320 if leaves > MAX_PARALLEL_OPERATIONS {
10321 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10322 reason: "fan_out_limit_exceeded",
10323 member_path: member_path.clone(),
10324 message: format!(
10325 "group contains {leaves} durable leaves; the limit is {MAX_PARALLEL_OPERATIONS}"
10326 ),
10327 }));
10328 }
10329 if !root && operations.is_empty() {
10330 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10331 reason: "nested_group_empty",
10332 member_path: member_path.clone(),
10333 message: "a nested group must contain at least one durable leaf".to_string(),
10334 }));
10335 }
10336
10337 for (index, operation) in operations.iter().enumerate() {
10338 member_path.push(index);
10339 match operation {
10340 ParallelOperation::Activity {
10341 options, arguments, ..
10342 } => {
10343 options
10344 .validate()
10345 .map_err(|error| Error::InvalidActivityOptions(error))?;
10346 if let Err(error) = arguments {
10347 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10348 reason: "arguments_invalid",
10349 member_path: member_path.clone(),
10350 message: error.to_string(),
10351 }));
10352 }
10353 }
10354 ParallelOperation::ChildWorkflow {
10355 options, arguments, ..
10356 } => {
10357 validate_parallel_child_options(options)?;
10358 if let Err(error) = arguments {
10359 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10360 reason: "arguments_invalid",
10361 member_path: member_path.clone(),
10362 message: error.to_string(),
10363 }));
10364 }
10365 }
10366 ParallelOperation::Timer(duration)
10367 if duration.as_secs() == u64::MAX && duration.subsec_nanos() > 0 =>
10368 {
10369 return Err(Error::TimerDurationOverflow);
10370 }
10371 ParallelOperation::Timer(_) => {}
10372 ParallelOperation::Signal(signal_name) => {
10373 validate_user_signal_name(signal_name)?;
10374 if signal_name.trim().is_empty() {
10375 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10376 reason: "signal_name_empty",
10377 member_path: member_path.clone(),
10378 message: "signal wait name must not be empty".to_string(),
10379 }));
10380 }
10381 }
10382 ParallelOperation::Condition { options, .. } => {
10383 options.validate()?;
10384 }
10385 ParallelOperation::Group(children) => {
10386 validate_parallel_operations(children, member_path, false)?;
10387 }
10388 }
10389 member_path.pop();
10390 }
10391 Ok(())
10392}
10393
10394fn validate_parallel_child_options(options: &ChildWorkflowOptions) -> Result<()> {
10395 if options.task_queue.trim().is_empty() {
10396 return Err(Error::InvalidChildWorkflowOptions(
10397 "task_queue must not be empty".to_string(),
10398 ));
10399 }
10400 for (name, value) in [
10401 (
10402 "execution_timeout_seconds",
10403 options.execution_timeout_seconds,
10404 ),
10405 ("run_timeout_seconds", options.run_timeout_seconds),
10406 ] {
10407 if value == Some(0) {
10408 return Err(Error::InvalidChildWorkflowOptions(format!(
10409 "{name} must be at least 1"
10410 )));
10411 }
10412 }
10413 if options
10414 .retry_policy
10415 .as_ref()
10416 .is_some_and(|policy| policy.max_attempts == Some(0))
10417 {
10418 return Err(Error::InvalidChildWorkflowOptions(
10419 "retry_policy.max_attempts must be at least 1".to_string(),
10420 ));
10421 }
10422 Ok(())
10423}
10424
10425fn parallel_shape(operations: &[ParallelOperation]) -> ParallelShape {
10426 ParallelShape::Group(
10427 operations
10428 .iter()
10429 .map(|operation| match operation {
10430 ParallelOperation::Group(children) => parallel_shape(children),
10431 _ => ParallelShape::Leaf,
10432 })
10433 .collect(),
10434 )
10435}
10436
10437fn parallel_descriptors(
10438 operations: Vec<ParallelOperation>,
10439 base_sequence: u64,
10440) -> Result<Vec<ParallelDescriptor>> {
10441 let size = parallel_leaf_count(&operations);
10442 let kind = parallel_group_kind(&operations).unwrap_or("activity");
10443 let mut descriptors = Vec::with_capacity(size);
10444 let mut cursor = 0;
10445
10446 for (index, operation) in operations.into_iter().enumerate() {
10447 match operation {
10448 ParallelOperation::Group(children) => {
10449 let child_base = base_sequence
10450 .checked_add(u64::try_from(cursor).unwrap_or(u64::MAX))
10451 .ok_or(Error::TimerDurationOverflow)?;
10452 for mut descriptor in parallel_descriptors(children, child_base)? {
10453 let outer_index = cursor + descriptor.offset;
10454 descriptor.group_path.insert(
10455 0,
10456 parallel_group_entry(base_sequence, size, outer_index, kind),
10457 );
10458 descriptor.member_path.insert(0, index);
10459 descriptor.offset = outer_index;
10460 descriptors.push(descriptor);
10461 }
10462 cursor = descriptors.len();
10463 }
10464 operation => {
10465 descriptors.push(ParallelDescriptor {
10466 operation,
10467 offset: cursor,
10468 member_path: vec![index],
10469 group_path: vec![parallel_group_entry(base_sequence, size, cursor, kind)],
10470 });
10471 cursor += 1;
10472 }
10473 }
10474 }
10475 Ok(descriptors)
10476}
10477
10478enum ParallelLeafCall {
10479 Activity(ActivityCall),
10480 ChildWorkflow(ChildWorkflowCall),
10481 Timer(TimerCall),
10482 Signal(SignalCall),
10483 Condition(ConditionWaitCall),
10484}
10485
10486fn parallel_leaf_call(
10487 ctx: &WorkflowContext,
10488 operation: ParallelOperation,
10489 parallel_group_path: Vec<ParallelGroupMetadata>,
10490) -> ParallelLeafCall {
10491 match operation {
10492 ParallelOperation::Activity {
10493 activity_type,
10494 options,
10495 arguments,
10496 } => ParallelLeafCall::Activity(ActivityCall {
10497 ctx: ctx.clone(),
10498 activity_type,
10499 options,
10500 args: Some(arguments),
10501 scheduled: false,
10502 parallel_group_path,
10503 }),
10504 ParallelOperation::ChildWorkflow {
10505 workflow_type,
10506 options,
10507 arguments,
10508 } => ParallelLeafCall::ChildWorkflow(ChildWorkflowCall {
10509 ctx: ctx.clone(),
10510 workflow_type,
10511 options,
10512 args: Some(arguments),
10513 scheduled: false,
10514 matched_pending: false,
10515 parallel_group_path,
10516 }),
10517 ParallelOperation::Timer(duration) => {
10518 let delay_seconds = duration
10519 .as_secs()
10520 .checked_add(u64::from(duration.subsec_nanos() > 0));
10521 ParallelLeafCall::Timer(TimerCall {
10522 ctx: ctx.clone(),
10523 delay_seconds,
10524 scheduled: false,
10525 matched_pending: false,
10526 parallel_group_path,
10527 })
10528 }
10529 ParallelOperation::Signal(signal_name) => ParallelLeafCall::Signal(SignalCall {
10530 ctx: ctx.clone(),
10531 signal_name,
10532 runtime_reserved_allowed: false,
10533 opened_wait: false,
10534 matched_pending: false,
10535 parallel_group_path,
10536 }),
10537 ParallelOperation::Condition { options, predicate } => {
10538 ParallelLeafCall::Condition(ConditionWaitCall {
10539 ctx: ctx.clone(),
10540 options,
10541 predicate,
10542 occurrence_id: None,
10543 opened_wait: false,
10544 parallel_group_path,
10545 })
10546 }
10547 ParallelOperation::Group(_) => {
10548 unreachable!("parallel descriptors contain only durable leaves")
10549 }
10550 }
10551}
10552
10553impl ParallelLeafCall {
10554 fn poll_avro_value(&mut self, cx: &mut TaskContext<'_>) -> Poll<Result<ParallelAvroResult>> {
10555 match self {
10556 Self::Activity(call) => Pin::new(call)
10557 .poll_avro_value(cx)
10558 .map_ok(ParallelAvroResult::Activity),
10559 Self::ChildWorkflow(call) => Pin::new(call)
10560 .poll_avro_value(cx)
10561 .map_ok(ParallelAvroResult::ChildWorkflow),
10562 Self::Timer(call) => Pin::new(call)
10563 .poll(cx)
10564 .map_ok(|()| ParallelAvroResult::Timer),
10565 Self::Signal(call) => Pin::new(call)
10566 .poll_avro_value(cx)
10567 .map_ok(ParallelAvroResult::Signal),
10568 Self::Condition(call) => Pin::new(call)
10569 .poll(cx)
10570 .map_ok(ParallelAvroResult::Condition),
10571 }
10572 }
10573}
10574
10575struct ParallelLeaf {
10576 call: ParallelLeafCall,
10577 member_path: Vec<usize>,
10578 group_path: Vec<ParallelGroupMetadata>,
10579 result: Option<ParallelAvroResult>,
10580}
10581
10582pub struct ParallelCall {
10584 ctx: WorkflowContext,
10585 operations: Option<Vec<ParallelOperation>>,
10586 shape: Option<ParallelShape>,
10587 leaves: Vec<ParallelLeaf>,
10588}
10589
10590impl ParallelCall {
10591 fn new(ctx: WorkflowContext, operations: Vec<ParallelOperation>) -> Self {
10592 Self {
10593 ctx,
10594 operations: Some(operations),
10595 shape: None,
10596 leaves: Vec::new(),
10597 }
10598 }
10599
10600 fn initialize(&mut self) -> Result<()> {
10601 let operations = self.operations.take().unwrap_or_default();
10602 validate_parallel_operations(&operations, &mut Vec::new(), true)?;
10603 self.shape = Some(parallel_shape(&operations));
10604 if operations.is_empty() {
10605 return Ok(());
10606 }
10607
10608 let base_sequence = {
10609 let state = self
10610 .ctx
10611 .state
10612 .lock()
10613 .map_err(|_| Error::WorkflowStatePoisoned)?;
10614 if let Some(recorded) = state.recorded_commands.get(state.command_cursor) {
10615 recorded.sequence()
10616 } else {
10617 let last = state
10618 .recorded_commands
10619 .last()
10620 .map(RecordedCommand::sequence)
10621 .unwrap_or(0);
10622 last.checked_add(u64::try_from(state.commands.len()).unwrap_or(u64::MAX))
10623 .and_then(|sequence| sequence.checked_add(1))
10624 .ok_or_else(|| {
10625 Error::InvalidParallelGroup(ParallelGroupError {
10626 reason: "sequence_overflow",
10627 member_path: Vec::new(),
10628 message: "parallel group sequence identity overflowed u64".to_string(),
10629 })
10630 })?
10631 }
10632 };
10633
10634 self.leaves = parallel_descriptors(operations, base_sequence)?
10635 .into_iter()
10636 .map(|descriptor| {
10637 let call = parallel_leaf_call(
10638 &self.ctx,
10639 descriptor.operation,
10640 descriptor.group_path.clone(),
10641 );
10642 ParallelLeaf {
10643 call,
10644 member_path: descriptor.member_path,
10645 group_path: descriptor.group_path,
10646 result: None,
10647 }
10648 })
10649 .collect();
10650 Ok(())
10651 }
10652
10653 fn poll_avro_value(
10654 mut self: Pin<&mut Self>,
10655 cx: &mut TaskContext<'_>,
10656 ) -> Poll<Result<Vec<ParallelAvroResult>>> {
10657 if self.operations.is_some() {
10658 if let Err(error) = self.initialize() {
10659 return Poll::Ready(Err(error));
10660 }
10661 }
10662 if self.leaves.is_empty() {
10663 return Poll::Ready(Ok(Vec::new()));
10664 }
10665
10666 let mut failures = Vec::new();
10667 let mut pending = false;
10668 for (index, leaf) in self.leaves.iter_mut().enumerate() {
10669 if leaf.result.is_some() {
10670 continue;
10671 }
10672 match leaf.call.poll_avro_value(cx) {
10673 Poll::Ready(Ok(result)) => leaf.result = Some(result),
10674 Poll::Ready(Err(error)) => failures.push((index, error)),
10675 Poll::Pending => pending = true,
10676 }
10677 }
10678
10679 if !failures.is_empty() {
10680 if let Some(position) = failures
10681 .iter()
10682 .position(|(_, error)| workflow_task_integrity_error(error))
10683 {
10684 return Poll::Ready(Err(failures.remove(position).1));
10685 }
10686 failures.sort_by_key(|(index, _)| *index);
10687 let (failed_index, cause) = failures.remove(0);
10688 let failed = &self.leaves[failed_index];
10689 let completed = self
10690 .leaves
10691 .iter()
10692 .filter_map(|leaf| {
10693 leaf.result
10694 .clone()
10695 .and_then(|result| result.into_json_result().ok())
10696 .map(|result| ParallelCompletion {
10697 member_path: leaf.member_path.clone(),
10698 result,
10699 })
10700 })
10701 .collect();
10702 let group_id = failed
10703 .group_path
10704 .first()
10705 .map(|entry| entry.parallel_group_id.clone())
10706 .unwrap_or_default();
10707 return Poll::Ready(Err(Error::ParallelFailed(ParallelFailure {
10708 group_id,
10709 member_path: failed.member_path.clone(),
10710 group_path: failed.group_path.clone(),
10711 completed,
10712 cause: Box::new(cause),
10713 })));
10714 }
10715 if pending {
10716 return Poll::Pending;
10717 }
10718
10719 let mut flat_results = self
10720 .leaves
10721 .iter_mut()
10722 .map(|leaf| leaf.result.take().expect("completed parallel leaf"))
10723 .collect::<Vec<_>>()
10724 .into_iter();
10725 let results = parallel_results_for_shape(
10726 self.shape.as_ref().expect("initialized parallel shape"),
10727 &mut flat_results,
10728 );
10729 Poll::Ready(Ok(match results {
10730 ParallelAvroResult::Group(results) => results,
10731 ParallelAvroResult::Activity(_)
10732 | ParallelAvroResult::ChildWorkflow(_)
10733 | ParallelAvroResult::Timer
10734 | ParallelAvroResult::Signal(_)
10735 | ParallelAvroResult::Condition(_) => {
10736 unreachable!("root parallel shape is a group")
10737 }
10738 }))
10739 }
10740}
10741
10742fn parallel_results_for_shape(
10743 shape: &ParallelShape,
10744 flat_results: &mut impl Iterator<Item = ParallelAvroResult>,
10745) -> ParallelAvroResult {
10746 match shape {
10747 ParallelShape::Leaf => flat_results.next().expect("one result per parallel leaf"),
10748 ParallelShape::Group(children) => ParallelAvroResult::Group(
10749 children
10750 .iter()
10751 .map(|child| parallel_results_for_shape(child, flat_results))
10752 .collect(),
10753 ),
10754 }
10755}
10756
10757impl Future for ParallelCall {
10758 type Output = Result<Vec<ParallelResult>>;
10759
10760 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
10761 self.poll_avro_value(cx)
10762 .map_ok(|results| {
10763 results
10764 .into_iter()
10765 .map(ParallelAvroResult::into_json_result)
10766 .collect::<Result<Vec<_>>>()
10767 })
10768 .map_ok(|result| result)
10769 .flatten_result()
10770 }
10771}
10772
10773#[derive(Clone, Debug)]
10774struct SelectionMemberPlan {
10775 key: SelectionKey,
10776 index: usize,
10777 base_sequence: u64,
10778 size: usize,
10779 kind: String,
10780 shape: ParallelShape,
10781 leaf_start: usize,
10782}
10783
10784fn selection_operation_kind(operation: &ParallelOperation) -> &'static str {
10785 match operation {
10786 ParallelOperation::Activity { .. } => "activity",
10787 ParallelOperation::ChildWorkflow { .. } => "child",
10788 ParallelOperation::Timer(_) => "timer",
10789 ParallelOperation::Signal(_) => "signal",
10790 ParallelOperation::Condition { .. } => "condition",
10791 ParallelOperation::Group(_) => "group",
10792 }
10793}
10794
10795fn selection_operation_shape(operation: &ParallelOperation) -> ParallelShape {
10796 match operation {
10797 ParallelOperation::Group(children) => parallel_shape(children),
10798 _ => ParallelShape::Leaf,
10799 }
10800}
10801
10802fn selection_descriptors(
10803 operations: Vec<(SelectionKey, ParallelOperation)>,
10804 base_sequence: u64,
10805) -> Result<(Vec<ParallelDescriptor>, Vec<SelectionMemberPlan>)> {
10806 if operations.is_empty() {
10807 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10808 reason: "selection_empty",
10809 member_path: Vec::new(),
10810 message: "durable selection requires at least one operation".to_string(),
10811 }));
10812 }
10813 let operation_refs = operations
10814 .iter()
10815 .map(|(_, operation)| operation)
10816 .collect::<Vec<_>>();
10817 let total_size = operation_refs
10818 .iter()
10819 .map(|operation| match operation {
10820 ParallelOperation::Group(children) => parallel_leaf_count(children),
10821 _ => 1,
10822 })
10823 .sum::<usize>();
10824 if total_size > MAX_PARALLEL_OPERATIONS {
10825 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10826 reason: "fan_out_limit_exceeded",
10827 member_path: Vec::new(),
10828 message: format!(
10829 "selection contains {total_size} durable leaves; the limit is {MAX_PARALLEL_OPERATIONS}"
10830 ),
10831 }));
10832 }
10833 let group_kind = {
10834 let mut kind = None;
10835 for operation in &operation_refs {
10836 let operation_kind = parallel_operation_kind(operation).unwrap_or("mixed");
10837 match kind {
10838 None => kind = Some(operation_kind),
10839 Some(current) if current == operation_kind => {}
10840 Some(_) => {
10841 kind = Some("mixed");
10842 break;
10843 }
10844 }
10845 }
10846 kind.unwrap_or("mixed")
10847 };
10848
10849 let mut descriptors = Vec::with_capacity(total_size);
10850 let mut members = Vec::with_capacity(operations.len());
10851 let mut cursor = 0usize;
10852 let mut seen_keys: Vec<SelectionKey> = Vec::new();
10853 for (member_index, (key, operation)) in operations.into_iter().enumerate() {
10854 if matches!(&key, SelectionKey::Name(value) if value.is_empty()) {
10855 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10856 reason: "selection_key_invalid",
10857 member_path: vec![member_index],
10858 message: "selection member keys must be non-empty strings or non-negative integers"
10859 .to_string(),
10860 }));
10861 }
10862 if seen_keys.contains(&key) {
10863 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10864 reason: "selection_key_duplicate",
10865 member_path: vec![member_index],
10866 message: format!("selection member key {key:?} is duplicated"),
10867 }));
10868 }
10869 seen_keys.push(key.clone());
10870 let member_size = match &operation {
10871 ParallelOperation::Group(children) => parallel_leaf_count(children),
10872 _ => 1,
10873 };
10874 if member_size == 0 {
10875 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10876 reason: "selection_member_empty",
10877 member_path: vec![member_index],
10878 message: "a selection member must contain at least one durable leaf".to_string(),
10879 }));
10880 }
10881 let member_base = base_sequence
10882 .checked_add(u64::try_from(cursor).unwrap_or(u64::MAX))
10883 .ok_or(Error::TimerDurationOverflow)?;
10884 let member_kind = selection_operation_kind(&operation).to_string();
10885 let member_shape = selection_operation_shape(&operation);
10886 let leaf_start = descriptors.len();
10887 match operation {
10888 ParallelOperation::Group(children) => {
10889 validate_parallel_operations(&children, &mut vec![member_index], false)?;
10890 for mut descriptor in parallel_descriptors(children, member_base)? {
10891 let flat_index = cursor + descriptor.offset;
10892 descriptor.group_path.insert(
10893 0,
10894 selection_group_entry(
10895 base_sequence,
10896 total_size,
10897 flat_index,
10898 group_kind,
10899 &SelectionMemberMetadata {
10900 key: key.clone(),
10901 index: member_index,
10902 base_sequence: member_base,
10903 size: member_size,
10904 kind: member_kind.clone(),
10905 },
10906 ),
10907 );
10908 descriptor.member_path.insert(0, member_index);
10909 descriptor.offset = flat_index;
10910 descriptors.push(descriptor);
10911 }
10912 }
10913 operation => {
10914 validate_parallel_operations(
10915 std::slice::from_ref(&operation),
10916 &mut Vec::new(),
10917 true,
10918 )?;
10919 descriptors.push(ParallelDescriptor {
10920 operation,
10921 offset: cursor,
10922 member_path: vec![member_index],
10923 group_path: vec![selection_group_entry(
10924 base_sequence,
10925 total_size,
10926 cursor,
10927 group_kind,
10928 &SelectionMemberMetadata {
10929 key: key.clone(),
10930 index: member_index,
10931 base_sequence: member_base,
10932 size: member_size,
10933 kind: member_kind.clone(),
10934 },
10935 )],
10936 });
10937 }
10938 }
10939 members.push(SelectionMemberPlan {
10940 key,
10941 index: member_index,
10942 base_sequence: member_base,
10943 size: member_size,
10944 kind: member_kind,
10945 shape: member_shape,
10946 leaf_start,
10947 });
10948 cursor += member_size;
10949 }
10950 Ok((descriptors, members))
10951}
10952
10953struct SelectionLeaf {
10954 call: ParallelLeafCall,
10955 outcome: Option<Result<ParallelAvroResult>>,
10956}
10957
10958#[derive(Clone)]
10960pub struct DurableOperationHandle {
10961 ctx: WorkflowContext,
10962 pub key: SelectionKey,
10963 pub index: usize,
10964 pub kind: String,
10965 pub identity: String,
10966 pub base_sequence: u64,
10967 pub size: usize,
10968 pub selection_group_id: String,
10969 shape: ParallelShape,
10970}
10971
10972impl std::fmt::Debug for DurableOperationHandle {
10973 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10974 formatter
10975 .debug_struct("DurableOperationHandle")
10976 .field("key", &self.key)
10977 .field("index", &self.index)
10978 .field("kind", &self.kind)
10979 .field("identity", &self.identity)
10980 .field("base_sequence", &self.base_sequence)
10981 .field("size", &self.size)
10982 .field("selection_group_id", &self.selection_group_id)
10983 .finish()
10984 }
10985}
10986
10987impl DurableOperationHandle {
10988 pub fn await_result(&self) -> DurableOperationAwaitCall {
10990 DurableOperationAwaitCall {
10991 handle: self.clone(),
10992 }
10993 }
10994
10995 pub fn cancel(&self) -> CancelDurableOperationCall {
10999 CancelDurableOperationCall {
11000 handle: self.clone(),
11001 emitted: false,
11002 }
11003 }
11004}
11005
11006#[derive(Debug)]
11008pub struct SelectionResult {
11009 pub key: SelectionKey,
11010 pub index: usize,
11011 pub kind: String,
11012 pub identity: String,
11013 pub value: Option<ParallelResult>,
11014 pub failure: Option<Error>,
11015 pub winner: DurableOperationHandle,
11016 pub handles: Vec<DurableOperationHandle>,
11017}
11018
11019impl SelectionResult {
11020 pub fn succeeded(&self) -> bool {
11021 self.failure.is_none()
11022 }
11023
11024 pub fn handle(&self, key: &SelectionKey) -> Option<&DurableOperationHandle> {
11025 self.handles.iter().find(|handle| &handle.key == key)
11026 }
11027
11028 pub fn remaining(&self) -> Vec<&DurableOperationHandle> {
11029 self.handles
11030 .iter()
11031 .filter(|handle| handle.index != self.index)
11032 .collect()
11033 }
11034
11035 pub fn into_result(self) -> Result<ParallelResult> {
11036 match (self.value, self.failure) {
11037 (Some(value), None) => Ok(value),
11038 (_, Some(error)) => Err(error),
11039 _ => Err(Error::WorkerLoop(
11040 "selection result contained neither a value nor a failure".to_string(),
11041 )),
11042 }
11043 }
11044}
11045
11046pub struct SelectCall {
11048 ctx: WorkflowContext,
11049 operations: Option<Vec<(SelectionKey, ParallelOperation)>>,
11050 members: Vec<SelectionMemberPlan>,
11051 leaves: Vec<SelectionLeaf>,
11052 group_id: Option<String>,
11053}
11054
11055impl SelectCall {
11056 fn new(ctx: WorkflowContext, operations: Vec<(SelectionKey, ParallelOperation)>) -> Self {
11057 Self {
11058 ctx,
11059 operations: Some(operations),
11060 members: Vec::new(),
11061 leaves: Vec::new(),
11062 group_id: None,
11063 }
11064 }
11065
11066 fn initialize(&mut self) -> Result<()> {
11067 let operations = self.operations.take().unwrap_or_default();
11068 let base_sequence = {
11069 let state = self
11070 .ctx
11071 .state
11072 .lock()
11073 .map_err(|_| Error::WorkflowStatePoisoned)?;
11074 if let Some(marker) = state.selection_markers.get(state.selection_marker_cursor) {
11075 marker.selection_group_base_sequence
11076 } else if let Some(recorded) = state.recorded_commands.get(state.command_cursor) {
11077 recorded.sequence()
11078 } else {
11079 let last = state
11080 .recorded_commands
11081 .last()
11082 .map(RecordedCommand::sequence)
11083 .unwrap_or(0);
11084 last.checked_add(u64::try_from(state.commands.len()).unwrap_or(u64::MAX))
11085 .and_then(|sequence| sequence.checked_add(1))
11086 .ok_or(Error::TimerDurationOverflow)?
11087 }
11088 };
11089 let (descriptors, members) = selection_descriptors(operations, base_sequence)?;
11090 let group_id = format!("select-calls:{base_sequence}:{}", descriptors.len());
11091 self.leaves = descriptors
11092 .into_iter()
11093 .map(|descriptor| SelectionLeaf {
11094 call: parallel_leaf_call(&self.ctx, descriptor.operation, descriptor.group_path),
11095 outcome: None,
11096 })
11097 .collect();
11098 self.members = members;
11099 self.group_id = Some(group_id);
11100 Ok(())
11101 }
11102}
11103
11104impl Future for SelectCall {
11105 type Output = Result<SelectionResult>;
11106
11107 fn poll(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11108 if self.operations.is_some() {
11109 if let Err(error) = self.initialize() {
11110 return Poll::Ready(Err(error));
11111 }
11112 }
11113
11114 for leaf in &mut self.leaves {
11115 if leaf.outcome.is_some() {
11116 continue;
11117 }
11118 if let Poll::Ready(outcome) = leaf.call.poll_avro_value(cx) {
11119 if outcome
11120 .as_ref()
11121 .err()
11122 .is_some_and(workflow_task_integrity_error)
11123 {
11124 return Poll::Ready(outcome.map(|_| unreachable!()));
11125 }
11126 leaf.outcome = Some(outcome);
11127 }
11128 }
11129
11130 let all_members_terminal = self.leaves.iter().all(|leaf| leaf.outcome.is_some());
11131 let selection_member_range = self
11132 .members
11133 .first()
11134 .map(|member| member.base_sequence)
11135 .zip(self.leaves.len().try_into().ok())
11136 .map(|(base_sequence, size): (u64, u64)| {
11137 base_sequence..base_sequence.saturating_add(size)
11138 });
11139 let marker = {
11140 let mut state = match self.ctx.state.lock() {
11141 Ok(state) => state,
11142 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11143 };
11144 let marker = state
11145 .selection_markers
11146 .get(state.selection_marker_cursor)
11147 .cloned();
11148 if marker.is_none()
11149 && all_members_terminal
11150 && selection_member_range.as_ref().is_some_and(|member_range| {
11151 state
11152 .recorded_commands
11153 .iter()
11154 .any(|command| member_range.contains(&command.sequence()))
11155 })
11156 {
11157 state.matched_recorded_pending = true;
11163 }
11164 marker
11165 };
11166 let Some(marker) = marker else {
11167 return Poll::Pending;
11168 };
11169 if self.group_id.as_deref() != Some(marker.selection_group_id.as_str())
11170 || marker.selection_group_size != self.leaves.len()
11171 || self.members.first().map(|member| member.base_sequence)
11172 != Some(marker.selection_group_base_sequence)
11173 {
11174 return Poll::Ready(Err(invalid_recorded_history(
11175 "selection_group_shape_mismatch",
11176 marker.selection_group_base_sequence,
11177 self.group_id
11178 .as_deref()
11179 .unwrap_or("initialized selection group"),
11180 &marker.selection_group_id,
11181 "recorded selection group differs from current workflow code",
11182 )));
11183 }
11184 let Some(member_position) = self.members.iter().position(|member| {
11185 member.key == marker.member_key
11186 && member.index == marker.member_index
11187 && member.base_sequence == marker.member_base_sequence
11188 && member.size == marker.member_size
11189 && member.kind == marker.operation_kind
11190 }) else {
11191 return Poll::Ready(Err(invalid_recorded_history(
11192 "selection_member_shape_mismatch",
11193 marker.member_base_sequence,
11194 "winner member matching current workflow code",
11195 &format!("{:?}", marker.member_key),
11196 "recorded selection winner differs from the authored member identity",
11197 )));
11198 };
11199 let member = self.members[member_position].clone();
11200 let (handles, resolution_sequence) = {
11201 let mut state = match self.ctx.state.lock() {
11202 Ok(state) => state,
11203 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11204 };
11205 let identities = self
11206 .members
11207 .iter()
11208 .map(|candidate| {
11209 selection_operation_identity(
11210 &state,
11211 &candidate.kind,
11212 candidate.base_sequence,
11213 candidate.size,
11214 )
11215 })
11216 .collect::<Vec<_>>();
11217 if let Some((position, missing)) = identities
11218 .iter()
11219 .enumerate()
11220 .find(|(_, identity)| identity.is_empty())
11221 .map(|(position, identity)| (position, identity.clone()))
11222 {
11223 let candidate = &self.members[position];
11224 return Poll::Ready(Err(invalid_recorded_history(
11225 "selection_operation_identity_missing",
11226 candidate.base_sequence,
11227 &format!(
11228 "durable {} resource identity from scheduled/open history",
11229 candidate.kind
11230 ),
11231 &missing,
11232 "selection member history is missing its canonical durable identity",
11233 )));
11234 }
11235 let expected_winner_identity = &identities[member_position];
11236 let resolution_sequence = match validated_selection_resolution_sequence(
11237 &state,
11238 &marker,
11239 &member,
11240 expected_winner_identity,
11241 ) {
11242 Ok(sequence) => sequence,
11243 Err(error) => return Poll::Ready(Err(error)),
11244 };
11245 let handles = self
11246 .members
11247 .iter()
11248 .zip(identities)
11249 .map(|(member, identity)| DurableOperationHandle {
11250 ctx: self.ctx.clone(),
11251 key: member.key.clone(),
11252 index: member.index,
11253 kind: member.kind.clone(),
11254 identity,
11255 base_sequence: member.base_sequence,
11256 size: member.size,
11257 selection_group_id: marker.selection_group_id.clone(),
11258 shape: member.shape.clone(),
11259 })
11260 .collect::<Vec<_>>();
11261 if let Err(error) = validate_selection_cancellations_for_handles(&state, &handles) {
11262 return Poll::Ready(Err(error));
11263 }
11264 state.selection_marker_cursor += 1;
11265 (handles, resolution_sequence)
11266 };
11267
11268 let mut winner_failure = None;
11269 let mut flat_results = Vec::with_capacity(member.size);
11270 if marker.outcome == "failed" {
11271 let resolution_offset = match resolution_sequence
11272 .checked_sub(member.base_sequence)
11273 .and_then(|offset| usize::try_from(offset).ok())
11274 {
11275 Some(offset) if offset < member.size => offset,
11276 _ => {
11277 return Poll::Ready(Err(invalid_recorded_history(
11278 "selection_resolution_event_mismatch",
11279 member.base_sequence,
11280 "failure event within selected member bounds",
11281 &resolution_sequence.to_string(),
11282 "selection failure event is outside the authored member",
11283 )))
11284 }
11285 };
11286 let leaf = &mut self.leaves[member.leaf_start + resolution_offset];
11287 match leaf.outcome.take() {
11288 Some(Err(error)) => winner_failure = Some(error),
11289 _ => {
11290 return Poll::Ready(Err(invalid_recorded_history(
11291 "selection_winner_outcome_mismatch",
11292 member.base_sequence,
11293 "exact failed terminal history referenced by SelectionResolved",
11294 "missing or successful resolution event",
11295 "selection winner marker disagrees with terminal operation history",
11296 )))
11297 }
11298 }
11299 } else {
11300 for leaf in &mut self.leaves[member.leaf_start..member.leaf_start + member.size] {
11301 match leaf.outcome.take() {
11302 Some(Ok(result)) => flat_results.push(result),
11303 Some(Err(_)) => {
11304 return Poll::Ready(Err(invalid_recorded_history(
11305 "selection_winner_outcome_mismatch",
11306 member.base_sequence,
11307 "fully completed nested selection member",
11308 "failed durable leaf",
11309 "completed selection winner contains a failed leaf",
11310 )))
11311 }
11312 None => {
11313 return Poll::Ready(Err(invalid_recorded_history(
11314 "selection_winner_unresolved",
11315 member.base_sequence,
11316 "terminal history for every completed winner leaf",
11317 "pending member history",
11318 "completed SelectionResolved member has an unfinished durable barrier",
11319 )))
11320 }
11321 }
11322 }
11323 }
11324 let value = if winner_failure.is_none() {
11325 let mut flat_results = flat_results.into_iter();
11326 let value = parallel_results_for_shape(&member.shape, &mut flat_results);
11327 match value.into_json_result() {
11328 Ok(value) => Some(value),
11329 Err(error) => return Poll::Ready(Err(error)),
11330 }
11331 } else {
11332 None
11333 };
11334 let winner = handles[member_position].clone();
11335 Poll::Ready(Ok(SelectionResult {
11336 key: winner.key.clone(),
11337 index: winner.index,
11338 kind: winner.kind.clone(),
11339 identity: winner.identity.clone(),
11340 value,
11341 failure: winner_failure,
11342 winner,
11343 handles,
11344 }))
11345 }
11346}
11347
11348fn selection_operation_identity(
11349 state: &WorkflowState,
11350 kind: &str,
11351 base_sequence: u64,
11352 size: usize,
11353) -> String {
11354 if kind == "group" {
11355 return format!("group:{base_sequence}:{size}");
11356 }
11357 let fields: &[&str] = match kind {
11358 "activity" => &["activity_execution_id"],
11359 "child" => &["child_workflow_run_id"],
11360 "timer" => &["timer_id"],
11361 "signal" => &["signal_wait_id"],
11362 "condition" => &["condition_wait_id"],
11363 _ => &[],
11364 };
11365 for sequence in base_sequence..base_sequence.saturating_add(size as u64) {
11366 for event in state
11367 .history_events
11368 .iter()
11369 .filter(|event| durable_event_sequence(event) == Some(sequence))
11370 {
11371 for field in fields {
11372 if let Some(identity) = event.payload.get(*field).and_then(Value::as_str) {
11373 if !identity.is_empty() {
11374 return identity.to_string();
11375 }
11376 }
11377 }
11378 }
11379 }
11380 String::new()
11381}
11382
11383fn validated_selection_resolution_sequence(
11384 state: &WorkflowState,
11385 marker: &SelectionMarker,
11386 member: &SelectionMemberPlan,
11387 expected_identity: &str,
11388) -> Result<u64> {
11389 if expected_identity.is_empty() {
11390 return Err(invalid_recorded_history(
11391 "selection_operation_identity_missing",
11392 member.base_sequence,
11393 &format!(
11394 "durable {} resource identity from scheduled/open history",
11395 member.kind
11396 ),
11397 "missing operation identity",
11398 "selection member history is missing its canonical durable identity",
11399 ));
11400 }
11401 if marker.operation_identity != expected_identity {
11402 return Err(invalid_recorded_history(
11403 "selection_operation_identity_mismatch",
11404 member.base_sequence,
11405 expected_identity,
11406 &marker.operation_identity,
11407 "selection winner identity does not match durable scheduled/open history",
11408 ));
11409 }
11410
11411 let failure_types = [
11412 "ActivityFailed",
11413 "ActivityCancelled",
11414 "ActivityTimedOut",
11415 "ChildRunFailed",
11416 "ChildRunCancelled",
11417 "ChildRunTerminated",
11418 ];
11419 let success_types = [
11420 "ActivityCompleted",
11421 "ChildRunCompleted",
11422 "TimerFired",
11423 "SignalApplied",
11424 "ConditionWaitSatisfied",
11425 "ConditionWaitTimedOut",
11426 ];
11427 let terminal_types: &[&str] = if marker.outcome == "failed" {
11428 &failure_types
11429 } else {
11430 &success_types
11431 };
11432 let mut candidates = Vec::new();
11433 for event in state.history_events.iter() {
11434 let Some(sequence) = durable_event_sequence(event) else {
11435 continue;
11436 };
11437 if sequence < member.base_sequence
11438 || sequence >= member.base_sequence.saturating_add(member.size as u64)
11439 || !terminal_types.contains(&event.event_type.as_str())
11440 {
11441 continue;
11442 }
11443 let event_id = event
11444 .raw
11445 .get("id")
11446 .or_else(|| event.raw.get("event_id"))
11447 .and_then(Value::as_str)
11448 .filter(|value| !value.is_empty())
11449 .ok_or_else(|| {
11450 invalid_recorded_history(
11451 "selection_resolution_event_id_missing",
11452 member.base_sequence,
11453 "terminal selection history with a durable event id",
11454 &event.payload.to_string(),
11455 "selection terminal history cannot be bound to its winner marker",
11456 )
11457 })?;
11458 candidates.push((event_id.to_string(), event.event_type.clone(), sequence));
11459 }
11460 let resolution = if marker.outcome == "failed" {
11461 candidates.first()
11462 } else {
11463 candidates.last()
11464 };
11465 let Some((event_id, event_type, sequence)) = resolution else {
11466 return Err(invalid_recorded_history(
11467 "selection_resolution_event_missing",
11468 member.base_sequence,
11469 "terminal history for the selected member",
11470 &format!("{:?}", marker.member_key),
11471 "selection winner marker has no matching durable terminal event",
11472 ));
11473 };
11474 if event_id != &marker.resolution_event_id || event_type != &marker.resolution_event_type {
11475 return Err(invalid_recorded_history(
11476 "selection_resolution_event_mismatch",
11477 member.base_sequence,
11478 &format!("{event_type}:{event_id}"),
11479 &format!(
11480 "{}:{}",
11481 marker.resolution_event_type, marker.resolution_event_id
11482 ),
11483 "selection winner marker does not reference the event that made its member terminal",
11484 ));
11485 }
11486 Ok(*sequence)
11487}
11488
11489fn recorded_selection_member_outcome(
11490 state: &WorkflowState,
11491 handle: &DurableOperationHandle,
11492) -> Result<Option<ParallelResult>> {
11493 for event in state.history_events.iter() {
11494 let Some(sequence) = durable_event_sequence(event) else {
11495 continue;
11496 };
11497 if sequence < handle.base_sequence
11498 || sequence >= handle.base_sequence.saturating_add(handle.size as u64)
11499 || !matches!(
11500 event.event_type.as_str(),
11501 "ActivityFailed"
11502 | "ActivityCancelled"
11503 | "ActivityTimedOut"
11504 | "ChildRunFailed"
11505 | "ChildRunCancelled"
11506 | "ChildRunTerminated"
11507 )
11508 {
11509 continue;
11510 }
11511 let Some(command) = state
11512 .recorded_commands
11513 .iter()
11514 .find(|command| command.sequence() == sequence)
11515 else {
11516 continue;
11517 };
11518 match command {
11519 RecordedCommand::Activity {
11520 outcome: Some(Err(failure)),
11521 ..
11522 } => return Err(Error::ActivityFailed(failure.clone())),
11523 RecordedCommand::ChildWorkflow {
11524 outcome: Some(Err(failure)),
11525 ..
11526 } => return Err(Error::ChildWorkflowFailed(failure.clone())),
11527 _ => {}
11528 }
11529 }
11530
11531 let mut results = Vec::with_capacity(handle.size);
11532 for sequence in handle.base_sequence..handle.base_sequence.saturating_add(handle.size as u64) {
11533 let Some(command) = state
11534 .recorded_commands
11535 .iter()
11536 .find(|command| command.sequence() == sequence)
11537 else {
11538 return Ok(None);
11539 };
11540 let result = match command {
11541 RecordedCommand::Activity { outcome, .. } => match outcome {
11542 Some(Ok(value)) => ParallelAvroResult::Activity(value.clone()),
11543 Some(Err(failure)) => return Err(Error::ActivityFailed(failure.clone())),
11544 None => return Ok(None),
11545 },
11546 RecordedCommand::Timer { fired, .. } => {
11547 if !fired {
11548 return Ok(None);
11549 }
11550 ParallelAvroResult::Timer
11551 }
11552 RecordedCommand::ChildWorkflow { outcome, .. } => match outcome {
11553 Some(Ok(value)) => ParallelAvroResult::ChildWorkflow(value.clone()),
11554 Some(Err(failure)) => return Err(Error::ChildWorkflowFailed(failure.clone())),
11555 None => return Ok(None),
11556 },
11557 RecordedCommand::SignalWait { value, .. } => match value {
11558 Some(value) => ParallelAvroResult::Signal(value.clone()),
11559 None => return Ok(None),
11560 },
11561 RecordedCommand::ConditionWait { result, .. } => match result {
11562 Some(result) => ParallelAvroResult::Condition(*result),
11563 None => return Ok(None),
11564 },
11565 other => {
11566 return Err(command_mismatch(
11567 other,
11568 format!("selected {} member", handle.kind),
11569 ))
11570 }
11571 };
11572 results.push(result);
11573 }
11574 let mut results = results.into_iter();
11575 parallel_results_for_shape(&handle.shape, &mut results)
11576 .into_json_result()
11577 .map(Some)
11578}
11579
11580fn recorded_selection_member_is_terminal(
11581 state: &WorkflowState,
11582 handle: &DurableOperationHandle,
11583) -> bool {
11584 let mut completed = 0usize;
11585 let mut all_completed = true;
11586 for sequence in handle.base_sequence..handle.base_sequence.saturating_add(handle.size as u64) {
11587 let Some(command) = state
11588 .recorded_commands
11589 .iter()
11590 .find(|command| command.sequence() == sequence)
11591 else {
11592 all_completed = false;
11593 continue;
11594 };
11595 let terminal = match command {
11596 RecordedCommand::Activity {
11597 outcome: Some(Err(_)),
11598 ..
11599 }
11600 | RecordedCommand::ChildWorkflow {
11601 outcome: Some(Err(_)),
11602 ..
11603 } => return true,
11604 RecordedCommand::Activity { outcome, .. } => outcome.is_some(),
11605 RecordedCommand::ChildWorkflow { outcome, .. } => outcome.is_some(),
11606 RecordedCommand::Timer { fired, .. } => *fired,
11607 RecordedCommand::SignalWait { value, .. } => value.is_some(),
11608 RecordedCommand::ConditionWait { result, .. } => result.is_some(),
11609 RecordedCommand::SearchAttributes { .. }
11610 | RecordedCommand::SideEffect { .. }
11611 | RecordedCommand::VersionMarker { .. }
11612 | RecordedCommand::Memo { .. } => false,
11613 };
11614 if !terminal {
11615 all_completed = false;
11616 continue;
11617 }
11618 completed += 1;
11619 }
11620 all_completed && completed == handle.size
11621}
11622
11623fn selection_cancellation_for_handle(
11624 state: &WorkflowState,
11625 handle: &DurableOperationHandle,
11626) -> Result<bool> {
11627 let Some(marker) = state.cancelled_selection_members.iter().find(|recorded| {
11628 recorded.selection_group_id == handle.selection_group_id
11629 && recorded.member_base_sequence == handle.base_sequence
11630 }) else {
11631 return Ok(false);
11632 };
11633 validate_selection_cancellation_marker(marker, handle)?;
11634 Ok(true)
11635}
11636
11637fn validate_selection_cancellations_for_handles(
11638 state: &WorkflowState,
11639 handles: &[DurableOperationHandle],
11640) -> Result<()> {
11641 let Some(group_id) = handles
11642 .first()
11643 .map(|handle| handle.selection_group_id.as_str())
11644 else {
11645 return Ok(());
11646 };
11647 for marker in state
11648 .cancelled_selection_members
11649 .iter()
11650 .filter(|marker| marker.selection_group_id == group_id)
11651 {
11652 let Some(handle) = handles
11653 .iter()
11654 .find(|handle| handle.base_sequence == marker.member_base_sequence)
11655 else {
11656 return Err(invalid_recorded_history(
11657 "selection_cancellation_member_mismatch",
11658 marker.member_base_sequence,
11659 "SelectionOperationCancelled matching an authored selection handle",
11660 &format!("{marker:?}"),
11661 "selection cancellation member base does not name an authored member",
11662 ));
11663 };
11664 validate_selection_cancellation_marker(marker, handle)?;
11665 }
11666 Ok(())
11667}
11668
11669fn validate_selection_cancellation_marker(
11670 marker: &SelectionCancellation,
11671 handle: &DurableOperationHandle,
11672) -> Result<()> {
11673 if marker.selection_group_id != handle.selection_group_id
11674 || marker.member_key != handle.key
11675 || marker.member_index != handle.index
11676 || marker.member_base_sequence != handle.base_sequence
11677 || marker.member_size != handle.size
11678 || marker.operation_kind != handle.kind
11679 || marker.operation_identity != handle.identity
11680 {
11681 return Err(invalid_recorded_history(
11682 "selection_cancellation_member_mismatch",
11683 handle.base_sequence,
11684 "SelectionOperationCancelled matching the authored selection handle",
11685 &format!("{marker:?}"),
11686 "selection cancellation history targets different authored member metadata",
11687 ));
11688 }
11689 Ok(())
11690}
11691
11692pub struct DurableOperationAwaitCall {
11694 handle: DurableOperationHandle,
11695}
11696
11697impl Future for DurableOperationAwaitCall {
11698 type Output = Result<ParallelResult>;
11699
11700 fn poll(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11701 let state = match self.handle.ctx.state.lock() {
11702 Ok(state) => state,
11703 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11704 };
11705 match selection_cancellation_for_handle(&state, &self.handle) {
11706 Err(error) => return Poll::Ready(Err(error)),
11707 Ok(false) => {}
11708 Ok(true) => {
11709 return Poll::Ready(Err(Error::DurableOperationCancelled(
11710 DurableOperationCancelled {
11711 selection_group_id: self.handle.selection_group_id.clone(),
11712 member_key: self.handle.key.clone(),
11713 member_index: self.handle.index,
11714 operation_kind: self.handle.kind.clone(),
11715 operation_identity: self.handle.identity.clone(),
11716 },
11717 )));
11718 }
11719 }
11720 match recorded_selection_member_outcome(&state, &self.handle) {
11721 Ok(Some(result)) => Poll::Ready(Ok(result)),
11722 Ok(None) => Poll::Pending,
11723 Err(error) => Poll::Ready(Err(error)),
11724 }
11725 }
11726}
11727
11728pub struct CancelDurableOperationCall {
11730 handle: DurableOperationHandle,
11731 emitted: bool,
11732}
11733
11734impl Future for CancelDurableOperationCall {
11735 type Output = Result<()>;
11736
11737 fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11738 let ctx = self.handle.ctx.clone();
11739 let mut state = match ctx.state.lock() {
11740 Ok(state) => state,
11741 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11742 };
11743 match selection_cancellation_for_handle(&state, &self.handle) {
11744 Err(error) => return Poll::Ready(Err(error)),
11745 Ok(true) => return Poll::Ready(Ok(())),
11746 Ok(false) => {}
11747 }
11748 if recorded_selection_member_is_terminal(&state, &self.handle) {
11749 return Poll::Ready(Ok(()));
11750 }
11751 if !self.emitted {
11752 state.commands.push(json!({
11753 "type": "cancel_selection_operation",
11754 "selection_group_id": self.handle.selection_group_id,
11755 "member_key": self.handle.key,
11756 "member_index": self.handle.index,
11757 "member_base_sequence": self.handle.base_sequence,
11758 "member_size": self.handle.size,
11759 "operation_kind": self.handle.kind,
11760 "operation_identity": self.handle.identity,
11761 }));
11762 self.emitted = true;
11763 }
11764 Poll::Pending
11768 }
11769}
11770
11771trait PollNestedResultExt<T> {
11772 fn flatten_result(self) -> Poll<Result<T>>;
11773}
11774
11775impl<T> PollNestedResultExt<T> for Poll<Result<Result<T>>> {
11776 fn flatten_result(self) -> Poll<Result<T>> {
11777 match self {
11778 Poll::Ready(Ok(result)) => Poll::Ready(result),
11779 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
11780 Poll::Pending => Poll::Pending,
11781 }
11782 }
11783}
11784
11785struct SagaCompensation {
11786 activity_type: String,
11787 options: ActivityOptions,
11788 arguments: AvroValue,
11789 registration_order: usize,
11790}
11791
11792pub struct Saga {
11800 ctx: WorkflowContext,
11801 compensations: Vec<SagaCompensation>,
11802}
11803
11804impl Saga {
11805 fn new(ctx: WorkflowContext) -> Self {
11806 Self {
11807 ctx,
11808 compensations: Vec::new(),
11809 }
11810 }
11811
11812 pub fn add_compensation<T: Serialize>(
11813 &mut self,
11814 activity_type: impl Into<String>,
11815 args: T,
11816 ) -> Result<&mut Self> {
11817 self.add_compensation_with_options(activity_type, ActivityOptions::new(), args)
11818 }
11819
11820 pub fn add_compensation_with_options<T: Serialize>(
11821 &mut self,
11822 activity_type: impl Into<String>,
11823 options: ActivityOptions,
11824 args: T,
11825 ) -> Result<&mut Self> {
11826 let activity_type = activity_type.into();
11827 if activity_type.trim().is_empty() || activity_type.trim() != activity_type {
11828 return Err(Error::Codec(
11829 "saga compensation activity type must be non-empty without surrounding whitespace"
11830 .to_string(),
11831 ));
11832 }
11833 options.validate().map_err(Error::InvalidActivityOptions)?;
11834 let arguments = AvroValue::from_serialize(&args)?;
11835 let registration_order = self.compensations.len() + 1;
11836 self.compensations.push(SagaCompensation {
11837 activity_type,
11838 options,
11839 arguments,
11840 registration_order,
11841 });
11842 Ok(self)
11843 }
11844
11845 pub async fn compensate(mut self, initiating_failure: Error) -> Error {
11847 while let Some(compensation) = self.compensations.pop() {
11848 if let Err(compensation_failure) = self
11849 .ctx
11850 .activity_with_options(
11851 compensation.activity_type.clone(),
11852 compensation.options,
11853 compensation.arguments,
11854 )
11855 .await
11856 {
11857 if workflow_task_integrity_error(&compensation_failure) {
11858 return compensation_failure;
11859 }
11860 return Error::SagaCompensationFailed(SagaCompensationFailure {
11861 initiating_failure: Box::new(initiating_failure),
11862 compensation_failure: Box::new(compensation_failure),
11863 compensation_activity_type: compensation.activity_type,
11864 compensation_registration_order: compensation.registration_order,
11865 });
11866 }
11867 }
11868 initiating_failure
11869 }
11870
11871 pub async fn finish<T>(self, outcome: Result<T>) -> Result<T> {
11873 match outcome {
11874 Ok(value) => Ok(value),
11875 Err(error) => Err(self.compensate(error).await),
11876 }
11877 }
11878}
11879
11880pub struct ActivityCall {
11881 ctx: WorkflowContext,
11882 activity_type: String,
11883 options: ActivityOptions,
11884 args: Option<Result<AvroValue>>,
11885 scheduled: bool,
11886 parallel_group_path: Vec<ParallelGroupMetadata>,
11887}
11888
11889impl ActivityCall {
11890 fn poll_avro_value(
11891 mut self: Pin<&mut Self>,
11892 _cx: &mut TaskContext<'_>,
11893 ) -> Poll<Result<AvroValue>> {
11894 let ctx = self.ctx.clone();
11895 let mut state = match ctx.state.lock() {
11896 Ok(state) => state,
11897 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11898 };
11899
11900 if self.scheduled {
11901 return Poll::Pending;
11902 }
11903
11904 let options = match self.options.validate() {
11905 Ok(options) => options,
11906 Err(error) => {
11907 return Poll::Ready(Err(Error::InvalidActivityOptions(error)));
11908 }
11909 };
11910 let task_queue = options
11911 .task_queue
11912 .clone()
11913 .unwrap_or_else(|| state.task_queue.clone());
11914 let current_recorded_options = RecordedActivityOptions {
11915 task_queue: RecordedSnapshotValue::Known(Some(task_queue.clone())),
11916 execution_mode: RecordedSnapshotValue::Known(None),
11919 retry_policy: current_activity_retry_snapshot(&options),
11920 };
11921
11922 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
11923 let sequence = recorded.sequence();
11924 match recorded {
11925 RecordedCommand::Activity {
11926 activity_type,
11927 options: recorded_options,
11928 outcome,
11929 parallel_group_path,
11930 ..
11931 } => {
11932 if let Err(error) = ensure_parallel_path_matches(
11933 sequence,
11934 parallel_group_path.as_deref(),
11935 &self.parallel_group_path,
11936 ) {
11937 return Poll::Ready(Err(error));
11938 }
11939 if let Some(recorded_type) = activity_type {
11940 if recorded_type != self.activity_type {
11941 return Poll::Ready(Err(Error::NonDeterministicReplay(
11942 ReplayFailure::new(
11943 "recorded_command_detail_mismatch",
11944 Some(sequence),
11945 Some(format!("activity:{recorded_type}")),
11946 Some(format!("activity:{}", self.activity_type)),
11947 "recorded activity type differs from the current workflow command",
11948 ),
11949 )));
11950 }
11951 }
11952 if let Some(recorded_options) = recorded_options {
11953 if !recorded_options
11954 .task_queue
11955 .matches_current(¤t_recorded_options.task_queue)
11956 {
11957 return Poll::Ready(Err(Error::NonDeterministicReplay(
11958 ReplayFailure::new(
11959 "activity_task_queue_mismatch",
11960 Some(sequence),
11961 Some(activity_options_description(&recorded_options)),
11962 Some(activity_options_description(¤t_recorded_options)),
11963 "recorded activity task queue differs from the current workflow command",
11964 ),
11965 )));
11966 }
11967 if !recorded_options
11968 .execution_mode
11969 .matches_current(¤t_recorded_options.execution_mode)
11970 {
11971 return Poll::Ready(Err(Error::NonDeterministicReplay(
11972 ReplayFailure::new(
11973 "activity_execution_mode_mismatch",
11974 Some(sequence),
11975 Some(activity_options_description(&recorded_options)),
11976 Some(activity_options_description(¤t_recorded_options)),
11977 "recorded activity execution mode differs from the current workflow command",
11978 ),
11979 )));
11980 }
11981 if !recorded_options
11982 .retry_policy
11983 .matches_current(¤t_recorded_options.retry_policy)
11984 {
11985 return Poll::Ready(Err(Error::NonDeterministicReplay(
11986 ReplayFailure::new(
11987 "activity_retry_policy_mismatch",
11988 Some(sequence),
11989 Some(activity_options_description(&recorded_options)),
11990 Some(activity_options_description(¤t_recorded_options)),
11991 "recorded activity retry policy differs from the current workflow command",
11992 ),
11993 )));
11994 }
11995 }
11996 state.command_cursor += 1;
11997 if let Some(outcome) = outcome {
11998 return Poll::Ready(outcome.map_err(Error::ActivityFailed));
11999 }
12000 state.matched_recorded_pending = true;
12001 self.scheduled = true;
12002 return Poll::Pending;
12003 }
12004 other => {
12005 return Poll::Ready(Err(command_mismatch(
12006 &other,
12007 format!("activity:{}", self.activity_type),
12008 )));
12009 }
12010 }
12011 }
12012
12013 if !self.scheduled {
12014 let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
12015 Ok(args) => args,
12016 Err(error) => return Poll::Ready(Err(error)),
12017 };
12018 let arguments = normalize_avro_arguments(args);
12019 let envelope = match encode_typed_envelope(&arguments, &state.payload_codec) {
12020 Ok(envelope) => envelope,
12021 Err(error) => return Poll::Ready(Err(error)),
12022 };
12023
12024 let mut command = serde_json::Map::from_iter([
12025 ("type".to_string(), json!("schedule_activity")),
12026 (
12027 "activity_type".to_string(),
12028 json!(self.activity_type.clone()),
12029 ),
12030 ("queue".to_string(), json!(task_queue)),
12031 ("arguments".to_string(), envelope),
12032 ]);
12033 for (field, value) in [
12034 ("start_to_close_timeout", options.start_to_close_timeout),
12035 (
12036 "schedule_to_start_timeout",
12037 options.schedule_to_start_timeout,
12038 ),
12039 (
12040 "schedule_to_close_timeout",
12041 options.schedule_to_close_timeout,
12042 ),
12043 ("heartbeat_timeout", options.heartbeat_timeout),
12044 ] {
12045 if let Some(value) = value {
12046 command.insert(field.to_string(), json!(value));
12047 }
12048 }
12049 if let Some(retry_policy) = options.retry_policy {
12050 command.insert("retry_policy".to_string(), retry_policy);
12051 }
12052 apply_parallel_group_path(&mut command, &self.parallel_group_path);
12053 state.commands.push(Value::Object(command));
12054 self.scheduled = true;
12055 }
12056
12057 Poll::Pending
12058 }
12059}
12060
12061impl Future for ActivityCall {
12062 type Output = Result<Value>;
12063
12064 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12065 match self.poll_avro_value(cx) {
12066 Poll::Ready(Ok(value)) => Poll::Ready(value.into_json()),
12067 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
12068 Poll::Pending => Poll::Pending,
12069 }
12070 }
12071}
12072
12073pub struct TimerCall {
12075 ctx: WorkflowContext,
12076 delay_seconds: Option<u64>,
12077 scheduled: bool,
12078 matched_pending: bool,
12079 parallel_group_path: Vec<ParallelGroupMetadata>,
12080}
12081
12082impl Future for TimerCall {
12083 type Output = Result<()>;
12084
12085 fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12086 if self.matched_pending {
12087 return Poll::Pending;
12088 }
12089
12090 let ctx = self.ctx.clone();
12091 let Some(requested_delay) = self.delay_seconds else {
12092 return Poll::Ready(Err(Error::TimerDurationOverflow));
12093 };
12094 let mut state = match ctx.state.lock() {
12095 Ok(state) => state,
12096 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12097 };
12098
12099 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
12100 match recorded {
12101 RecordedCommand::Timer {
12102 sequence,
12103 delay_seconds,
12104 fired,
12105 parallel_group_path,
12106 ..
12107 } => {
12108 if let Err(error) = ensure_parallel_path_matches(
12109 sequence,
12110 parallel_group_path.as_deref(),
12111 &self.parallel_group_path,
12112 ) {
12113 return Poll::Ready(Err(error));
12114 }
12115 if delay_seconds != requested_delay {
12116 return Poll::Ready(Err(Error::NonDeterministicReplay(
12117 ReplayFailure::new(
12118 "timer_delay_mismatch",
12119 Some(sequence),
12120 Some(format!("timer:{delay_seconds}s")),
12121 Some(format!("timer:{requested_delay}s")),
12122 "recorded timer delay differs from the current workflow command",
12123 ),
12124 )));
12125 }
12126 state.command_cursor += 1;
12127 if fired {
12128 return Poll::Ready(Ok(()));
12129 }
12130 state.matched_recorded_pending = true;
12131 self.scheduled = true;
12132 self.matched_pending = true;
12133 return Poll::Pending;
12134 }
12135 other => return Poll::Ready(Err(command_mismatch(&other, "timer"))),
12136 }
12137 }
12138
12139 if !self.scheduled {
12140 let mut command = serde_json::Map::from_iter([
12141 ("type".to_string(), json!("start_timer")),
12142 ("delay_seconds".to_string(), json!(requested_delay)),
12143 ]);
12144 apply_parallel_group_path(&mut command, &self.parallel_group_path);
12145 state.commands.push(Value::Object(command));
12146 self.scheduled = true;
12147 }
12148
12149 Poll::Pending
12150 }
12151}
12152
12153pub struct ConditionWaitCall {
12155 ctx: WorkflowContext,
12156 options: ConditionWaitOptions,
12157 predicate: Box<dyn Fn() -> Result<bool> + Send + 'static>,
12158 occurrence_id: Option<String>,
12159 opened_wait: bool,
12160 parallel_group_path: Vec<ParallelGroupMetadata>,
12161}
12162
12163impl Future for ConditionWaitCall {
12164 type Output = Result<ConditionWaitResult>;
12165
12166 fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12167 if self.opened_wait {
12168 return Poll::Pending;
12169 }
12170
12171 let options = match self.options.validate() {
12172 Ok(options) => options,
12173 Err(error) => return Poll::Ready(Err(Error::InvalidConditionWaitOptions(error))),
12174 };
12175 let ctx = self.ctx.clone();
12176 let occurrence_id = match self.occurrence_id.as_ref() {
12177 Some(occurrence_id) => occurrence_id.clone(),
12178 None => {
12179 let mut state = match ctx.state.lock() {
12180 Ok(state) => state,
12181 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12182 };
12183 let ordinal = state.condition_wait_occurrence_counter;
12184 state.condition_wait_occurrence_counter = match ordinal.checked_add(1) {
12185 Some(next) => next,
12186 None => {
12187 return Poll::Ready(Err(Error::WorkerLoop(
12188 "condition wait occurrence counter overflowed".to_string(),
12189 )))
12190 }
12191 };
12192 let occurrence_id = format!("{CONDITION_WAIT_OCCURRENCE_PREFIX}{ordinal}");
12193 drop(state);
12194 self.occurrence_id = Some(occurrence_id.clone());
12195 occurrence_id
12196 }
12197 };
12198
12199 let recorded_result = {
12200 let mut state = match ctx.state.lock() {
12201 Ok(state) => state,
12202 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12203 };
12204 let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() else {
12205 drop(state);
12206 return self.poll_new_condition(options);
12207 };
12208 if !matches!(recorded, RecordedCommand::ConditionWait { .. }) {
12209 return Poll::Ready(Err(command_mismatch(&recorded, "condition wait")));
12210 }
12211
12212 let mut cursor = state.command_cursor;
12213 let mut result = None;
12214 loop {
12215 let Some(RecordedCommand::ConditionWait {
12216 sequence,
12217 occurrence_id: recorded_occurrence_id,
12218 condition_key,
12219 predicate_identity,
12220 timeout_seconds,
12221 result: recorded_result,
12222 parallel_group_path,
12223 ..
12224 }) = state.recorded_commands.get(cursor)
12225 else {
12226 break;
12227 };
12228
12229 if cursor > state.command_cursor && recorded_occurrence_id != &occurrence_id {
12230 break;
12231 }
12232 if let Err(error) = ensure_parallel_path_matches(
12233 *sequence,
12234 parallel_group_path.as_deref(),
12235 &self.parallel_group_path,
12236 ) {
12237 return Poll::Ready(Err(error));
12238 }
12239 if let Err(error) = validate_recorded_condition_wait(
12240 *sequence,
12241 recorded_occurrence_id,
12242 condition_key.as_deref(),
12243 predicate_identity,
12244 *timeout_seconds,
12245 &occurrence_id,
12246 &options,
12247 ) {
12248 return Poll::Ready(Err(error));
12249 }
12250 if result == Some(ConditionWaitResult::TimedOut) {
12251 return Poll::Ready(Err(Error::NonDeterministicReplay(ReplayFailure::new(
12252 "condition_wait_reopened_after_timeout",
12253 Some(*sequence),
12254 Some("timed-out condition is terminal".to_string()),
12255 Some("another physical wait-open".to_string()),
12256 "condition history reopened one logical wait after its durable timeout",
12257 ))));
12258 }
12259 result = *recorded_result;
12260 cursor += 1;
12261 }
12262 state.command_cursor = cursor;
12263 result
12264 };
12265
12266 if let Some(result) = recorded_result {
12267 return Poll::Ready(Ok(result));
12268 }
12269
12270 self.poll_open_condition(options)
12271 }
12272}
12273
12274impl ConditionWaitCall {
12275 fn poll_new_condition(
12276 self: Pin<&mut Self>,
12277 options: ValidatedConditionWaitOptions,
12278 ) -> Poll<Result<ConditionWaitResult>> {
12279 self.poll_open_condition(options)
12280 }
12281
12282 fn poll_open_condition(
12283 mut self: Pin<&mut Self>,
12284 options: ValidatedConditionWaitOptions,
12285 ) -> Poll<Result<ConditionWaitResult>> {
12286 let selection_member = self
12287 .parallel_group_path
12288 .first()
12289 .is_some_and(|entry| entry.parallel_group_mode.as_deref() == Some("select"));
12290 match (self.predicate)() {
12291 Ok(true) if !selection_member => {
12292 return Poll::Ready(Ok(ConditionWaitResult::Satisfied))
12293 }
12294 Ok(_) => {}
12295 Err(error) => return Poll::Ready(Err(error)),
12296 }
12297 if options.timeout_seconds == Some(0) && !selection_member {
12298 return Poll::Ready(Ok(ConditionWaitResult::TimedOut));
12299 }
12300
12301 let ctx = self.ctx.clone();
12302 let mut state = match ctx.state.lock() {
12303 Ok(state) => state,
12304 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12305 };
12306 let mut command = serde_json::Map::from_iter([
12307 ("type".to_string(), json!("open_condition_wait")),
12308 (
12309 "condition_wait_occurrence_id".to_string(),
12310 json!(self.occurrence_id.as_deref().unwrap_or_default()),
12311 ),
12312 ("condition_key".to_string(), json!(options.condition_key)),
12313 (
12314 "condition_definition_fingerprint".to_string(),
12315 json!(options.predicate_identity),
12316 ),
12317 ]);
12318 if let Some(timeout_seconds) = options.timeout_seconds {
12319 command.insert("timeout_seconds".to_string(), json!(timeout_seconds));
12320 }
12321 apply_parallel_group_path(&mut command, &self.parallel_group_path);
12322 state.commands.push(Value::Object(command));
12323 drop(state);
12324 self.opened_wait = true;
12325 Poll::Pending
12326 }
12327}
12328
12329fn validate_recorded_condition_wait(
12330 sequence: u64,
12331 recorded_occurrence_id: &str,
12332 recorded_key: Option<&str>,
12333 recorded_predicate_identity: &str,
12334 recorded_timeout_seconds: Option<u64>,
12335 current_occurrence_id: &str,
12336 current: &ValidatedConditionWaitOptions,
12337) -> Result<()> {
12338 if recorded_occurrence_id != current_occurrence_id {
12339 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12340 "condition_wait_occurrence_mismatch",
12341 Some(sequence),
12342 Some(recorded_occurrence_id.to_string()),
12343 Some(current_occurrence_id.to_string()),
12344 "recorded condition occurrence differs from the current authored wait position",
12345 )));
12346 }
12347 if recorded_key != Some(current.condition_key.as_str()) {
12348 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12349 "condition_wait_key_mismatch",
12350 Some(sequence),
12351 recorded_key.map(str::to_string),
12352 Some(current.condition_key.clone()),
12353 "recorded condition identity differs from the current workflow wait",
12354 )));
12355 }
12356 if recorded_predicate_identity != current.predicate_identity {
12357 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12358 "condition_wait_predicate_mismatch",
12359 Some(sequence),
12360 Some(recorded_predicate_identity.to_string()),
12361 Some(current.predicate_identity.clone()),
12362 "recorded condition predicate behavior differs from current workflow code",
12363 )));
12364 }
12365 if recorded_timeout_seconds != current.timeout_seconds {
12366 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12367 "condition_wait_timeout_mismatch",
12368 Some(sequence),
12369 recorded_timeout_seconds.map(|seconds| format!("{seconds}s")),
12370 current.timeout_seconds.map(|seconds| format!("{seconds}s")),
12371 "recorded condition timeout differs from the current workflow wait",
12372 )));
12373 }
12374 Ok(())
12375}
12376
12377pub struct ChildWorkflowCall {
12379 ctx: WorkflowContext,
12380 workflow_type: String,
12381 options: ChildWorkflowOptions,
12382 args: Option<Result<AvroValue>>,
12383 scheduled: bool,
12384 matched_pending: bool,
12385 parallel_group_path: Vec<ParallelGroupMetadata>,
12386}
12387
12388impl ChildWorkflowCall {
12389 fn poll_avro_value(
12390 mut self: Pin<&mut Self>,
12391 _cx: &mut TaskContext<'_>,
12392 ) -> Poll<Result<ChildWorkflowAvroResult>> {
12393 if self.matched_pending {
12394 return Poll::Pending;
12395 }
12396
12397 let ctx = self.ctx.clone();
12398 let mut state = match ctx.state.lock() {
12399 Ok(state) => state,
12400 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12401 };
12402
12403 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
12404 let sequence = recorded.sequence();
12405 match recorded {
12406 RecordedCommand::ChildWorkflow {
12407 workflow_type,
12408 outcome,
12409 parallel_group_path,
12410 ..
12411 } => {
12412 if let Err(error) = ensure_parallel_path_matches(
12413 sequence,
12414 parallel_group_path.as_deref(),
12415 &self.parallel_group_path,
12416 ) {
12417 return Poll::Ready(Err(error));
12418 }
12419 if let Some(recorded_type) = workflow_type {
12420 if recorded_type != self.workflow_type {
12421 return Poll::Ready(Err(Error::NonDeterministicReplay(
12422 ReplayFailure::new(
12423 "recorded_command_detail_mismatch",
12424 Some(sequence),
12425 Some(format!("child workflow:{recorded_type}")),
12426 Some(format!("child workflow:{}", self.workflow_type)),
12427 "recorded child workflow type differs from the current workflow command",
12428 ),
12429 )));
12430 }
12431 }
12432 state.command_cursor += 1;
12433 if let Some(outcome) = outcome {
12434 return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed));
12435 }
12436 state.matched_recorded_pending = true;
12437 self.scheduled = true;
12438 self.matched_pending = true;
12439 return Poll::Pending;
12440 }
12441 other => {
12442 return Poll::Ready(Err(command_mismatch(
12443 &other,
12444 format!("child workflow:{}", self.workflow_type),
12445 )));
12446 }
12447 }
12448 }
12449
12450 if !self.scheduled {
12451 if self.options.task_queue.trim().is_empty() {
12452 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
12453 "task_queue must not be empty".to_string(),
12454 )));
12455 }
12456 for (name, value) in [
12457 (
12458 "execution_timeout_seconds",
12459 self.options.execution_timeout_seconds,
12460 ),
12461 ("run_timeout_seconds", self.options.run_timeout_seconds),
12462 ] {
12463 if value == Some(0) {
12464 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(format!(
12465 "{name} must be at least 1"
12466 ))));
12467 }
12468 }
12469
12470 let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
12471 Ok(args) => args,
12472 Err(error) => return Poll::Ready(Err(error)),
12473 };
12474 let arguments = match encode_typed_envelope(
12475 &normalize_avro_arguments(args),
12476 &state.payload_codec,
12477 ) {
12478 Ok(arguments) => arguments,
12479 Err(error) => return Poll::Ready(Err(error)),
12480 };
12481 let mut command = json!({
12482 "type": "start_child_workflow",
12483 "workflow_type": self.workflow_type,
12484 "queue": self.options.task_queue,
12485 "parent_close_policy": self.options.parent_close_policy.as_str(),
12486 "arguments": arguments,
12487 });
12488 let object = command
12489 .as_object_mut()
12490 .expect("child workflow command is always an object");
12491 if let Some(policy) = &self.options.retry_policy {
12492 let mut retry_policy = serde_json::Map::new();
12493 if let Some(max_attempts) = policy.max_attempts {
12494 if max_attempts == 0 {
12495 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
12496 "retry_policy.max_attempts must be at least 1".to_string(),
12497 )));
12498 }
12499 retry_policy.insert("max_attempts".to_string(), json!(max_attempts));
12500 }
12501 if !policy.backoff_seconds.is_empty() {
12502 retry_policy
12503 .insert("backoff_seconds".to_string(), json!(policy.backoff_seconds));
12504 }
12505 if !policy.non_retryable_error_types.is_empty() {
12506 retry_policy.insert(
12507 "non_retryable_error_types".to_string(),
12508 json!(policy.non_retryable_error_types),
12509 );
12510 }
12511 if retry_policy.is_empty() {
12512 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
12513 "retry_policy must configure at least one field".to_string(),
12514 )));
12515 }
12516 object.insert("retry_policy".to_string(), Value::Object(retry_policy));
12517 }
12518 if let Some(seconds) = self.options.execution_timeout_seconds {
12519 object.insert("execution_timeout_seconds".to_string(), json!(seconds));
12520 }
12521 if let Some(seconds) = self.options.run_timeout_seconds {
12522 object.insert("run_timeout_seconds".to_string(), json!(seconds));
12523 }
12524 apply_parallel_group_path(object, &self.parallel_group_path);
12525 state.commands.push(command);
12526 self.scheduled = true;
12527 }
12528
12529 Poll::Pending
12530 }
12531}
12532
12533impl Future for ChildWorkflowCall {
12534 type Output = Result<ChildWorkflowResult>;
12535
12536 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12537 match self.poll_avro_value(cx) {
12538 Poll::Ready(Ok(result)) => match result.result.into_json() {
12539 Ok(projected) => Poll::Ready(Ok(ChildWorkflowResult {
12540 parent: result.parent,
12541 child: result.child,
12542 child_workflow_type: result.child_workflow_type,
12543 result: projected,
12544 })),
12545 Err(error) => Poll::Ready(Err(error)),
12546 },
12547 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
12548 Poll::Pending => Poll::Pending,
12549 }
12550 }
12551}
12552
12553fn command_mismatch(recorded: &RecordedCommand, actual: impl Into<String>) -> Error {
12554 Error::NonDeterministicReplay(ReplayFailure::new(
12555 "recorded_command_mismatch",
12556 Some(recorded.sequence()),
12557 Some(recorded.shape().to_string()),
12558 Some(actual.into()),
12559 "current workflow command does not match the recorded durable command sequence",
12560 ))
12561}
12562
12563pub struct SignalCall {
12564 ctx: WorkflowContext,
12565 signal_name: String,
12566 runtime_reserved_allowed: bool,
12567 opened_wait: bool,
12568 matched_pending: bool,
12569 parallel_group_path: Vec<ParallelGroupMetadata>,
12570}
12571
12572impl SignalCall {
12573 fn poll_avro_value(
12574 mut self: Pin<&mut Self>,
12575 _cx: &mut TaskContext<'_>,
12576 ) -> Poll<Result<Vec<AvroValue>>> {
12577 if self.matched_pending {
12578 return Poll::Pending;
12579 }
12580 if !self.runtime_reserved_allowed {
12581 if let Err(error) = validate_user_signal_name(&self.signal_name) {
12582 return Poll::Ready(Err(error));
12583 }
12584 }
12585
12586 let ctx = self.ctx.clone();
12587 let mut state = match ctx.state.lock() {
12588 Ok(state) => state,
12589 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12590 };
12591
12592 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
12593 match recorded {
12594 RecordedCommand::SignalWait {
12595 sequence,
12596 signal_name,
12597 value,
12598 parallel_group_path,
12599 } => {
12600 if let Err(error) = ensure_parallel_path_matches(
12601 sequence,
12602 parallel_group_path.as_deref(),
12603 &self.parallel_group_path,
12604 ) {
12605 return Poll::Ready(Err(error));
12606 }
12607 if signal_name != self.signal_name {
12608 return Poll::Ready(Err(Error::NonDeterministicReplay(
12609 ReplayFailure::new(
12610 "recorded_command_detail_mismatch",
12611 Some(sequence),
12612 Some(format!("signal wait:{signal_name}")),
12613 Some(format!("signal wait:{}", self.signal_name)),
12614 "recorded signal name differs from the current workflow command",
12615 ),
12616 )));
12617 }
12618
12619 state.command_cursor += 1;
12620 if let Some(value) = value {
12621 return Poll::Ready(Ok(value));
12622 }
12623 if state
12624 .resume_signal
12625 .as_ref()
12626 .is_some_and(|signal| signal.signal_name == self.signal_name)
12627 {
12628 let signal = state
12629 .resume_signal
12630 .take()
12631 .expect("matching resume signal is present");
12632 return Poll::Ready(Ok(signal.arguments));
12633 }
12634
12635 state.matched_recorded_pending = true;
12636 self.opened_wait = true;
12637 self.matched_pending = true;
12638 return Poll::Pending;
12639 }
12640 other => {
12641 return Poll::Ready(Err(command_mismatch(
12642 &other,
12643 format!("signal wait:{}", self.signal_name),
12644 )));
12645 }
12646 }
12647 }
12648
12649 if state
12650 .resume_signal
12651 .as_ref()
12652 .is_some_and(|signal| signal.signal_name == self.signal_name)
12653 {
12654 let signal = state
12655 .resume_signal
12656 .take()
12657 .expect("matching resume signal is present");
12658 return Poll::Ready(Ok(signal.arguments));
12659 }
12660
12661 if !self.opened_wait {
12662 let mut command = serde_json::Map::from_iter([
12663 ("type".to_string(), json!("open_signal_wait")),
12664 ("signal_name".to_string(), json!(self.signal_name)),
12665 ]);
12666 apply_parallel_group_path(&mut command, &self.parallel_group_path);
12667 state.commands.push(Value::Object(command));
12668 self.opened_wait = true;
12669 }
12670
12671 Poll::Pending
12672 }
12673}
12674
12675impl Future for SignalCall {
12676 type Output = Result<Vec<Value>>;
12677
12678 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12679 match self.poll_avro_value(cx) {
12680 Poll::Ready(Ok(values)) => Poll::Ready(
12681 values
12682 .into_iter()
12683 .map(AvroValue::into_json)
12684 .collect::<Result<Vec<_>>>(),
12685 ),
12686 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
12687 Poll::Pending => Poll::Pending,
12688 }
12689 }
12690}
12691
12692#[derive(Clone, Debug)]
12693pub struct ActivityContext {
12694 client: Client,
12695 pub task_id: String,
12696 pub activity_attempt_id: String,
12697 pub lease_owner: String,
12698 pub activity_type: String,
12699 pub attempt_number: u64,
12700 pub task_queue: String,
12701 pub worker_id: String,
12702}
12703
12704impl ActivityContext {
12705 pub async fn heartbeat<T: Serialize>(&self, details: T) -> Result<ActivityHeartbeatResponse> {
12706 self.client
12707 .heartbeat_activity_task(
12708 &self.task_id,
12709 &self.activity_attempt_id,
12710 &self.lease_owner,
12711 details,
12712 )
12713 .await
12714 }
12715}
12716
12717fn decode_task_avro_arguments(value: Option<&Value>, codec: &str) -> Result<AvroValue> {
12718 validate_payload_codec(codec)?;
12719 match value {
12720 Some(value) => Ok(normalize_avro_arguments(decode_wire_avro_value(
12721 value, codec,
12722 )?)),
12723 None => Ok(AvroValue::Array(Vec::new())),
12724 }
12725}
12726
12727fn decode_resume_signal(task: &WorkflowTask) -> Result<Option<ResumeSignal>> {
12728 let Some(signal_name) = task
12729 .signal_name
12730 .as_deref()
12731 .filter(|value| !value.is_empty())
12732 else {
12733 return Ok(None);
12734 };
12735 let decoded = decode_task_avro_arguments(task.signal_arguments.as_ref(), &task.payload_codec)?;
12736 let AvroValue::Array(arguments) = decoded else {
12737 unreachable!("normalize_avro_arguments always returns an array");
12738 };
12739
12740 Ok(Some(ResumeSignal {
12741 signal_name: signal_name.to_string(),
12742 arguments,
12743 }))
12744}
12745
12746fn validate_workflow_task_payloads(task: &WorkflowTask) -> Result<()> {
12747 validate_payload_codec(&task.payload_codec)?;
12748 validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)?;
12749 validate_optional_inbound_payload(task.signal_arguments.as_ref(), &task.payload_codec)?;
12750 for event in &task.history_events {
12751 validate_history_event_payloads(event, &task.payload_codec)?;
12752 }
12753 Ok(())
12754}
12755
12756fn validate_activity_task_payloads(task: &ActivityTask) -> Result<()> {
12757 validate_payload_codec(&task.payload_codec)?;
12758 validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)
12759}
12760
12761fn validate_query_task_payloads(task: &QueryTask) -> Result<()> {
12762 validate_payload_codec(&task.payload_codec)?;
12763 validate_optional_inbound_payload(task.workflow_arguments.as_ref(), &task.payload_codec)?;
12764 validate_optional_inbound_payload(task.query_arguments.as_ref(), &task.payload_codec)?;
12765 for event in &task.history_events {
12766 validate_history_event_payloads(event, &task.payload_codec)?;
12767 }
12768
12769 let Some(export) = task.history_export.as_ref() else {
12770 return Ok(());
12771 };
12772 let export_codec = match export.get("payloads") {
12773 Some(payloads) => declared_payload_codec(payloads, "codec")?,
12774 None => None,
12775 }
12776 .unwrap_or(&task.payload_codec);
12777 validate_payload_codec(export_codec)?;
12778
12779 if let Some(events) = export.get("history_events").and_then(Value::as_array) {
12780 for event in events {
12781 let event_type = event
12782 .get("event_type")
12783 .or_else(|| event.get("type"))
12784 .and_then(Value::as_str)
12785 .unwrap_or_default();
12786 if let Some(payload) = event.get("payload") {
12787 validate_history_payloads(event_type, payload, export_codec)?;
12788 }
12789 }
12790 }
12791 for signal in export
12792 .get("signals")
12793 .and_then(Value::as_array)
12794 .into_iter()
12795 .flatten()
12796 {
12797 let codec = declared_payload_codec(signal, "payload_codec")?.unwrap_or(export_codec);
12798 validate_payload_codec(codec)?;
12799 validate_optional_inbound_payload(signal.get("arguments"), codec)?;
12800 }
12801 for activity in export
12802 .get("activities")
12803 .and_then(Value::as_array)
12804 .into_iter()
12805 .flatten()
12806 {
12807 let codec = declared_payload_codec(activity, "payload_codec")?.unwrap_or(export_codec);
12808 validate_payload_codec(codec)?;
12809 validate_optional_inbound_payload(activity.get("arguments"), codec)?;
12810 validate_optional_inbound_payload(activity.get("result"), codec)?;
12811 }
12812 Ok(())
12813}
12814
12815fn validate_history_event_payloads(event: &HistoryEvent, fallback_codec: &str) -> Result<()> {
12816 validate_history_payloads(&event.event_type, &event.payload, fallback_codec)
12817}
12818
12819fn validate_history_payloads(
12820 event_type: &str,
12821 payload: &Value,
12822 fallback_codec: &str,
12823) -> Result<()> {
12824 let codec = declared_payload_codec(payload, "payload_codec")?.unwrap_or(fallback_codec);
12825 validate_payload_codec(codec)?;
12826 for field in history_payload_fields(event_type) {
12827 validate_optional_inbound_payload(payload.get(*field), codec)?;
12828 }
12829 Ok(())
12830}
12831
12832const SIGNAL_HISTORY_PAYLOAD_FIELDS: &[&str] = &["value", "input", "arguments"];
12833
12834fn history_payload_fields(event_type: &str) -> &'static [&'static str] {
12835 match event_type {
12836 "ActivityCompleted" => &["result"],
12837 "SignalReceived" | "SignalApplied" => SIGNAL_HISTORY_PAYLOAD_FIELDS,
12838 "UpdateAccepted" | "UpdateRejected" | "UpdateApplied" => &["arguments"],
12839 "UpdateCompleted" | "SideEffectRecorded" => &["result"],
12840 "ChildRunCompleted" => &["result", "output"],
12841 "WorkflowCompleted" => &["output"],
12842 "ServiceCallStarted"
12843 | "ServiceCallCompleted"
12844 | "ServiceCallFailed"
12845 | "ServiceCallCancelled" => &["request_payload", "response_payload"],
12846 _ => &[],
12847 }
12848}
12849
12850fn signal_history_payload(payload: &Value) -> Option<&Value> {
12851 SIGNAL_HISTORY_PAYLOAD_FIELDS
12852 .iter()
12853 .find_map(|field| payload.get(*field))
12854}
12855
12856fn declared_payload_codec<'a>(value: &'a Value, field: &str) -> Result<Option<&'a str>> {
12857 match value.get(field) {
12858 None => Ok(None),
12859 Some(Value::String(codec)) => Ok(Some(codec)),
12860 Some(_) => Err(invalid_payload_envelope()),
12861 }
12862}
12863
12864fn validate_optional_inbound_payload(value: Option<&Value>, codec: &str) -> Result<()> {
12865 validate_payload_codec(codec)?;
12866 if let Some(value) = value.filter(|value| !value.is_null()) {
12867 decode_wire_avro_value(value, codec)?;
12868 }
12869 Ok(())
12870}
12871
12872fn recorded_parallel_group_entry(payload: &Value, sequence: u64) -> Result<ParallelGroupMetadata> {
12873 let group_id = payload_string(payload, "parallel_group_id").ok_or_else(|| {
12874 invalid_recorded_history(
12875 "parallel_group_metadata_invalid",
12876 sequence,
12877 "non-empty parallel_group_id",
12878 &payload.to_string(),
12879 "parallel-group history is missing its stable identity",
12880 )
12881 })?;
12882 let kind = payload_string(payload, "parallel_group_kind").ok_or_else(|| {
12883 invalid_recorded_history(
12884 "parallel_group_metadata_invalid",
12885 sequence,
12886 "activity, child, timer, signal, condition, or mixed group kind",
12887 &payload.to_string(),
12888 "parallel-group history is missing its group kind",
12889 )
12890 })?;
12891 if !matches!(
12892 kind.as_str(),
12893 "activity" | "child" | "timer" | "signal" | "condition" | "mixed"
12894 ) {
12895 return Err(invalid_recorded_history(
12896 "parallel_group_metadata_invalid",
12897 sequence,
12898 "activity, child, timer, signal, condition, or mixed group kind",
12899 &kind,
12900 "parallel-group history contains an unsupported group kind",
12901 ));
12902 }
12903 let base_sequence = payload
12904 .get("parallel_group_base_sequence")
12905 .and_then(value_as_u64)
12906 .filter(|value| *value > 0)
12907 .ok_or_else(|| {
12908 invalid_recorded_history(
12909 "parallel_group_metadata_invalid",
12910 sequence,
12911 "positive parallel_group_base_sequence",
12912 &payload.to_string(),
12913 "parallel-group history contains an invalid base sequence",
12914 )
12915 })?;
12916 let size = payload
12917 .get("parallel_group_size")
12918 .and_then(value_as_u64)
12919 .and_then(|value| usize::try_from(value).ok())
12920 .filter(|value| (1..=MAX_PARALLEL_OPERATIONS).contains(value))
12921 .ok_or_else(|| {
12922 invalid_recorded_history(
12923 "parallel_group_metadata_invalid",
12924 sequence,
12925 "bounded positive parallel_group_size",
12926 &payload.to_string(),
12927 "parallel-group history contains an invalid group size",
12928 )
12929 })?;
12930 let index = payload
12931 .get("parallel_group_index")
12932 .and_then(value_as_u64)
12933 .and_then(|value| usize::try_from(value).ok())
12934 .filter(|value| *value < size)
12935 .ok_or_else(|| {
12936 invalid_recorded_history(
12937 "parallel_group_metadata_invalid",
12938 sequence,
12939 "parallel_group_index within group bounds",
12940 &payload.to_string(),
12941 "parallel-group history contains an invalid member index",
12942 )
12943 })?;
12944 if base_sequence.checked_add(u64::try_from(index).unwrap_or(u64::MAX)) != Some(sequence) {
12945 return Err(invalid_recorded_history(
12946 "parallel_group_metadata_invalid",
12947 sequence,
12948 "base sequence plus member index equals workflow sequence",
12949 &payload.to_string(),
12950 "parallel-group path does not preserve durable workflow position",
12951 ));
12952 }
12953 let mode = payload
12954 .get("parallel_group_mode")
12955 .and_then(Value::as_str)
12956 .unwrap_or("all");
12957 if !matches!(mode, "all" | "select") {
12958 return Err(invalid_recorded_history(
12959 "parallel_group_metadata_invalid",
12960 sequence,
12961 "parallel group mode all or select",
12962 mode,
12963 "parallel-group history contains an unsupported group mode",
12964 ));
12965 }
12966 let expected_id = if mode == "select" {
12967 format!("select-calls:{base_sequence}:{size}")
12968 } else {
12969 format!("{}:{base_sequence}:{size}", parallel_group_prefix(&kind))
12970 };
12971 if group_id != expected_id {
12972 return Err(invalid_recorded_history(
12973 "parallel_group_metadata_invalid",
12974 sequence,
12975 &expected_id,
12976 &group_id,
12977 "parallel-group history contains an incompatible stable group ID",
12978 ));
12979 }
12980 let selection_member_key = if mode == "select" {
12981 Some(selection_key_from_value(
12982 payload.get("selection_member_key"),
12983 sequence,
12984 )?)
12985 } else {
12986 None
12987 };
12988 let selection_member_index = if mode == "select" {
12989 Some(required_parallel_usize(
12990 payload,
12991 "selection_member_index",
12992 sequence,
12993 )?)
12994 } else {
12995 None
12996 };
12997 let selection_member_base_sequence = if mode == "select" {
12998 Some(
12999 payload
13000 .get("selection_member_base_sequence")
13001 .and_then(value_as_u64)
13002 .filter(|value| *value >= base_sequence)
13003 .ok_or_else(|| {
13004 invalid_recorded_history(
13005 "parallel_group_metadata_invalid",
13006 sequence,
13007 "selection member base within its group",
13008 &payload.to_string(),
13009 "selection history contains an invalid member base sequence",
13010 )
13011 })?,
13012 )
13013 } else {
13014 None
13015 };
13016 let selection_member_size = if mode == "select" {
13017 let member_size = required_parallel_usize(payload, "selection_member_size", sequence)?;
13018 if member_size == 0 {
13019 return Err(invalid_recorded_history(
13020 "parallel_group_metadata_invalid",
13021 sequence,
13022 "positive selection member size",
13023 &payload.to_string(),
13024 "selection history contains an invalid member size",
13025 ));
13026 }
13027 Some(member_size)
13028 } else {
13029 None
13030 };
13031 let selection_member_kind = if mode == "select" {
13032 let kind = payload_string(payload, "selection_member_kind").ok_or_else(|| {
13033 invalid_recorded_history(
13034 "parallel_group_metadata_invalid",
13035 sequence,
13036 "selection member operation kind",
13037 &payload.to_string(),
13038 "selection history is missing its authored member kind",
13039 )
13040 })?;
13041 if !matches!(
13042 kind.as_str(),
13043 "activity" | "child" | "timer" | "signal" | "condition" | "group"
13044 ) {
13045 return Err(invalid_recorded_history(
13046 "parallel_group_metadata_invalid",
13047 sequence,
13048 "activity, child, timer, signal, condition, or group selection member kind",
13049 &kind,
13050 "selection history contains an unsupported member kind",
13051 ));
13052 }
13053 Some(kind)
13054 } else {
13055 None
13056 };
13057 if let (Some(member_base), Some(member_size)) =
13058 (selection_member_base_sequence, selection_member_size)
13059 {
13060 let member_end = member_base
13061 .checked_add(u64::try_from(member_size).unwrap_or(u64::MAX))
13062 .ok_or_else(|| {
13063 invalid_recorded_history(
13064 "parallel_group_metadata_invalid",
13065 sequence,
13066 "bounded selection member range",
13067 &payload.to_string(),
13068 "selection member range overflowed",
13069 )
13070 })?;
13071 let group_end = base_sequence
13072 .checked_add(u64::try_from(size).unwrap_or(u64::MAX))
13073 .unwrap_or(u64::MAX);
13074 if sequence < member_base || sequence >= member_end || member_end > group_end {
13075 return Err(invalid_recorded_history(
13076 "parallel_group_metadata_invalid",
13077 sequence,
13078 "workflow sequence within one bounded selection member",
13079 &payload.to_string(),
13080 "selection member range does not contain its durable leaf",
13081 ));
13082 }
13083 }
13084 Ok(ParallelGroupMetadata {
13085 parallel_group_id: group_id,
13086 parallel_group_kind: kind,
13087 parallel_group_base_sequence: base_sequence,
13088 parallel_group_size: size,
13089 parallel_group_index: index,
13090 parallel_group_mode: (mode == "select").then(|| "select".to_string()),
13091 selection_member_key,
13092 selection_member_index,
13093 selection_member_base_sequence,
13094 selection_member_size,
13095 selection_member_kind,
13096 })
13097}
13098
13099fn required_parallel_usize(payload: &Value, field: &str, sequence: u64) -> Result<usize> {
13100 payload
13101 .get(field)
13102 .and_then(value_as_u64)
13103 .and_then(|value| usize::try_from(value).ok())
13104 .ok_or_else(|| {
13105 invalid_recorded_history(
13106 "parallel_group_metadata_invalid",
13107 sequence,
13108 &format!("non-negative integer {field}"),
13109 &payload.to_string(),
13110 "selection history contains invalid member metadata",
13111 )
13112 })
13113}
13114
13115fn selection_key_from_value(value: Option<&Value>, sequence: u64) -> Result<SelectionKey> {
13116 match value {
13117 Some(Value::String(value)) if !value.is_empty() => Ok(SelectionKey::Name(value.clone())),
13118 Some(value) => value_as_u64(value)
13119 .and_then(|value| usize::try_from(value).ok())
13120 .map(SelectionKey::Index)
13121 .ok_or_else(|| {
13122 invalid_recorded_history(
13123 "selection_member_key_invalid",
13124 sequence,
13125 "non-empty string or non-negative integer member key",
13126 &value.to_string(),
13127 "selection history contains an invalid member key",
13128 )
13129 }),
13130 None => Err(invalid_recorded_history(
13131 "selection_member_key_missing",
13132 sequence,
13133 "selection_member_key",
13134 "<missing>",
13135 "selection history is missing its stable member key",
13136 )),
13137 }
13138}
13139
13140fn recorded_parallel_group_path(
13141 events: &[&HistoryEvent],
13142 sequence: u64,
13143) -> Result<Option<Vec<ParallelGroupMetadata>>> {
13144 let mut recorded: Option<Vec<ParallelGroupMetadata>> = None;
13145 for event in events {
13146 let payload = &event.payload;
13147 let has_metadata = payload.get("parallel_group_path").is_some()
13148 || payload.get("parallel_group_id").is_some()
13149 || payload.get("parallel_group_kind").is_some()
13150 || payload.get("parallel_group_base_sequence").is_some()
13151 || payload.get("parallel_group_size").is_some()
13152 || payload.get("parallel_group_index").is_some()
13153 || payload.get("parallel_group_mode").is_some()
13154 || payload.get("selection_member_key").is_some();
13155 if !has_metadata {
13156 continue;
13157 }
13158
13159 let top_level = recorded_parallel_group_entry(payload, sequence)?;
13160 let path = match payload.get("parallel_group_path") {
13161 None => vec![top_level.clone()],
13162 Some(Value::Array(entries)) if !entries.is_empty() => entries
13163 .iter()
13164 .map(|entry| recorded_parallel_group_entry(entry, sequence))
13165 .collect::<Result<Vec<_>>>()?,
13166 Some(value) => {
13167 return Err(invalid_recorded_history(
13168 "parallel_group_metadata_invalid",
13169 sequence,
13170 "non-empty parallel_group_path list",
13171 &value.to_string(),
13172 "parallel-group history contains an invalid group path",
13173 ));
13174 }
13175 };
13176 if path.last() != Some(&top_level) {
13177 return Err(invalid_recorded_history(
13178 "parallel_group_metadata_invalid",
13179 sequence,
13180 &serde_json::to_string(&path.last()).unwrap_or_default(),
13181 &serde_json::to_string(&top_level).unwrap_or_default(),
13182 "parallel-group top-level fields do not match the innermost path entry",
13183 ));
13184 }
13185 if recorded.as_ref().is_some_and(|existing| existing != &path) {
13186 return Err(invalid_recorded_history(
13187 "parallel_group_history_conflict",
13188 sequence,
13189 &serde_json::to_string(&recorded.as_ref()).unwrap_or_default(),
13190 &serde_json::to_string(&path).unwrap_or_default(),
13191 "parallel-group metadata changed between scheduling and resolution history",
13192 ));
13193 }
13194 recorded = Some(path);
13195 }
13196 Ok(recorded)
13197}
13198
13199fn recorded_commands(
13200 events: &[HistoryEvent],
13201 fallback_codec: &str,
13202 parent: WorkflowIdentity,
13203) -> Result<Vec<RecordedCommand>> {
13204 let mut events_by_sequence: BTreeMap<u64, Vec<&HistoryEvent>> = BTreeMap::new();
13205 let mut last_new_sequence = None;
13206
13207 for event in events {
13208 let is_activity = matches!(
13209 event.event_type.as_str(),
13210 "ActivityScheduled"
13211 | "ActivityStarted"
13212 | "ActivityHeartbeatRecorded"
13213 | "ActivityRetryScheduled"
13214 | "ActivityCompleted"
13215 | "ActivityFailed"
13216 | "ActivityCancelled"
13217 | "ActivityTimedOut"
13218 );
13219 let is_workflow_timer = matches!(
13220 event.event_type.as_str(),
13221 "TimerScheduled" | "TimerCancelled" | "TimerFired"
13222 ) && !is_internal_timer_event(event);
13223 let is_child_workflow = matches!(
13224 event.event_type.as_str(),
13225 "ChildWorkflowScheduled"
13226 | "ChildRunCompleted"
13227 | "ChildRunFailed"
13228 | "ChildRunCancelled"
13229 | "ChildRunTerminated"
13230 );
13231 let is_signal_wait = is_recorded_signal_wait_event(event);
13232 let is_condition_wait = is_recorded_condition_wait_event(event);
13233 let is_search_attributes = event.event_type == "SearchAttributesUpserted";
13234 let is_side_effect = event.event_type == "SideEffectRecorded";
13235 let is_version_marker = event.event_type == "VersionMarkerRecorded";
13236 let is_memo = event.event_type == "MemoUpserted";
13237 if !is_activity
13238 && !is_workflow_timer
13239 && !is_child_workflow
13240 && !is_signal_wait
13241 && !is_condition_wait
13242 && !is_search_attributes
13243 && !is_side_effect
13244 && !is_version_marker
13245 && !is_memo
13246 {
13247 continue;
13248 }
13249
13250 let sequence = durable_event_sequence(event).ok_or_else(|| {
13251 Error::NonDeterministicReplay(ReplayFailure::new(
13252 "durable_command_sequence_missing",
13253 None,
13254 Some("positive workflow sequence".to_string()),
13255 Some(event.event_type.clone()),
13256 "durable command history event has no workflow sequence",
13257 ))
13258 })?;
13259 if sequence == 0 {
13260 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
13261 "durable_command_sequence_invalid",
13262 Some(sequence),
13263 Some("positive workflow sequence".to_string()),
13264 Some(sequence.to_string()),
13265 "durable command history uses an invalid workflow sequence",
13266 )));
13267 }
13268 if !events_by_sequence.contains_key(&sequence) {
13269 if let Some(previous) = last_new_sequence {
13270 if sequence < previous {
13271 return Err(invalid_recorded_history(
13272 "durable_command_sequence_mismatch",
13273 sequence,
13274 &format!("workflow sequence greater than {previous}"),
13275 &sequence.to_string(),
13276 "durable commands are not strictly ordered by their recorded workflow sequence",
13277 ));
13278 }
13279 }
13280 last_new_sequence = Some(sequence);
13281 }
13282 events_by_sequence.entry(sequence).or_default().push(event);
13283 }
13284
13285 let commands: Vec<RecordedCommand> = events_by_sequence
13286 .into_iter()
13287 .map(|(sequence, sequence_events)| {
13288 let activity_events: Vec<_> = sequence_events
13289 .iter()
13290 .copied()
13291 .filter(|event| event.event_type.starts_with("Activity"))
13292 .collect();
13293 let timer_events: Vec<_> = sequence_events
13294 .iter()
13295 .copied()
13296 .filter(|event| event.event_type.starts_with("Timer"))
13297 .collect();
13298 let child_events: Vec<_> = sequence_events
13299 .iter()
13300 .copied()
13301 .filter(|event| {
13302 event.event_type == "ChildWorkflowScheduled"
13303 || event.event_type.starts_with("ChildRun")
13304 })
13305 .collect();
13306 let signal_wait_events: Vec<_> = sequence_events
13307 .iter()
13308 .copied()
13309 .filter(|event| is_recorded_signal_wait_event(event))
13310 .collect();
13311 let condition_wait_events: Vec<_> = sequence_events
13312 .iter()
13313 .copied()
13314 .filter(|event| is_recorded_condition_wait_event(event))
13315 .collect();
13316 let search_attribute_events: Vec<_> = sequence_events
13317 .iter()
13318 .copied()
13319 .filter(|event| event.event_type == "SearchAttributesUpserted")
13320 .collect();
13321 let side_effect_events: Vec<_> = sequence_events
13322 .iter()
13323 .copied()
13324 .filter(|event| event.event_type == "SideEffectRecorded")
13325 .collect();
13326 let version_marker_events: Vec<_> = sequence_events
13327 .iter()
13328 .copied()
13329 .filter(|event| event.event_type == "VersionMarkerRecorded")
13330 .collect();
13331 let memo_events: Vec<_> = sequence_events
13332 .iter()
13333 .copied()
13334 .filter(|event| event.event_type == "MemoUpserted")
13335 .collect();
13336
13337 let command_kind_count = usize::from(!activity_events.is_empty())
13338 + usize::from(!timer_events.is_empty())
13339 + usize::from(!child_events.is_empty())
13340 + usize::from(!signal_wait_events.is_empty())
13341 + usize::from(!condition_wait_events.is_empty())
13342 + usize::from(!search_attribute_events.is_empty())
13343 + usize::from(!side_effect_events.is_empty())
13344 + usize::from(!version_marker_events.is_empty())
13345 + usize::from(!memo_events.is_empty());
13346 if command_kind_count > 1 {
13347 let actual = [
13348 (!activity_events.is_empty()).then_some("activity"),
13349 (!timer_events.is_empty()).then_some("timer"),
13350 (!child_events.is_empty()).then_some("child workflow"),
13351 (!signal_wait_events.is_empty()).then_some("signal wait"),
13352 (!condition_wait_events.is_empty()).then_some("condition wait"),
13353 (!search_attribute_events.is_empty()).then_some("search-attribute update"),
13354 (!side_effect_events.is_empty()).then_some("side effect"),
13355 (!version_marker_events.is_empty()).then_some("version marker"),
13356 (!memo_events.is_empty()).then_some("memo upsert"),
13357 ]
13358 .into_iter()
13359 .flatten()
13360 .collect::<Vec<_>>()
13361 .join(" and ");
13362 return Err(invalid_recorded_history(
13363 "durable_command_sequence_collision",
13364 sequence,
13365 "one durable command kind",
13366 &actual,
13367 "one workflow sequence records more than one durable command kind",
13368 ));
13369 }
13370
13371 if !activity_events.is_empty() {
13372 let parallel_group_path =
13373 recorded_parallel_group_path(&activity_events, sequence)?;
13374 let scheduled_count = activity_events
13375 .iter()
13376 .filter(|event| event.event_type == "ActivityScheduled")
13377 .count();
13378 if scheduled_count > 1 {
13379 return Err(invalid_recorded_history(
13380 "duplicate_activity_schedule",
13381 sequence,
13382 "at most one ActivityScheduled event",
13383 "multiple ActivityScheduled events",
13384 "activity history schedules more than one command at one workflow sequence",
13385 ));
13386 }
13387 let activity_type = activity_events.iter().find_map(|event| {
13388 event
13389 .payload
13390 .get("activity_type")
13391 .or_else(|| event.payload.get("activity_name"))
13392 .and_then(Value::as_str)
13393 .map(str::to_string)
13394 });
13395 if activity_events.iter().filter_map(|event| {
13396 event
13397 .payload
13398 .get("activity_type")
13399 .or_else(|| event.payload.get("activity_name"))
13400 .and_then(Value::as_str)
13401 }).any(|candidate| Some(candidate) != activity_type.as_deref()) {
13402 return Err(invalid_recorded_history(
13403 "activity_identity_mismatch",
13404 sequence,
13405 activity_type.as_deref().unwrap_or("one activity identity"),
13406 "conflicting activity identities",
13407 "activity lifecycle events at one workflow sequence disagree on identity",
13408 ));
13409 }
13410 let terminal: Vec<_> = activity_events
13411 .iter()
13412 .copied()
13413 .filter(|event| {
13414 matches!(
13415 event.event_type.as_str(),
13416 "ActivityCompleted"
13417 | "ActivityFailed"
13418 | "ActivityCancelled"
13419 | "ActivityTimedOut"
13420 )
13421 })
13422 .collect();
13423 let duplicate_delivery = terminal.first().is_some_and(|first| {
13424 terminal.iter().all(|event| {
13425 event.event_type == first.event_type && event.payload == first.payload
13426 })
13427 });
13428 if terminal.len() > 1 && !duplicate_delivery {
13429 return Err(invalid_recorded_history(
13430 "duplicate_activity_terminal_event",
13431 sequence,
13432 "at most one terminal activity event",
13433 "multiple terminal activity events",
13434 "activity history settles one command more than once",
13435 ));
13436 }
13437 let outcome = terminal
13438 .first()
13439 .map(|event| activity_outcome(event, fallback_codec, activity_type.clone()))
13440 .transpose()?;
13441 let options = activity_events
13442 .iter()
13443 .find(|event| event.event_type == "ActivityScheduled")
13444 .and_then(|event| event.payload.get("activity"))
13445 .and_then(Value::as_object)
13446 .map(|activity| RecordedActivityOptions {
13447 task_queue: recorded_optional_string(activity, "queue"),
13448 execution_mode: recorded_optional_string(activity, "execution_mode"),
13449 retry_policy: recorded_activity_retry_snapshot(
13450 activity.get("retry_policy"),
13451 ),
13452 });
13453 return Ok(RecordedCommand::Activity {
13454 sequence,
13455 activity_type,
13456 options,
13457 outcome,
13458 parallel_group_path,
13459 });
13460 }
13461
13462 if !child_events.is_empty() {
13463 let parallel_group_path = recorded_parallel_group_path(&child_events, sequence)?;
13464 let scheduled: Vec<_> = child_events
13465 .iter()
13466 .copied()
13467 .filter(|event| event.event_type == "ChildWorkflowScheduled")
13468 .collect();
13469 if scheduled.len() != 1 {
13470 return Err(invalid_recorded_history(
13471 "child_workflow_schedule_missing_or_duplicate",
13472 sequence,
13473 "one ChildWorkflowScheduled event",
13474 &format!("{} ChildWorkflowScheduled events", scheduled.len()),
13475 "child workflow replay requires exactly one recorded schedule event",
13476 ));
13477 }
13478 let workflow_type = child_events.iter().find_map(|event| {
13479 event
13480 .payload
13481 .get("child_workflow_type")
13482 .or_else(|| event.payload.get("workflow_type"))
13483 .and_then(Value::as_str)
13484 .filter(|value| !value.is_empty())
13485 .map(str::to_string)
13486 });
13487 if child_events
13488 .iter()
13489 .filter_map(|event| {
13490 event
13491 .payload
13492 .get("child_workflow_type")
13493 .or_else(|| event.payload.get("workflow_type"))
13494 .and_then(Value::as_str)
13495 })
13496 .any(|candidate| Some(candidate) != workflow_type.as_deref())
13497 {
13498 return Err(invalid_recorded_history(
13499 "child_workflow_identity_mismatch",
13500 sequence,
13501 workflow_type
13502 .as_deref()
13503 .unwrap_or("one child workflow type"),
13504 "conflicting child workflow types",
13505 "child workflow lifecycle events at one sequence disagree on type",
13506 ));
13507 }
13508 let mut outcomes = child_workflow_outcomes(
13509 &child_events.iter().map(|event| (*event).clone()).collect::<Vec<_>>(),
13510 fallback_codec,
13511 parent.clone(),
13512 )?;
13513 let terminal_events = child_events
13514 .iter()
13515 .copied()
13516 .filter(|event| event.event_type.starts_with("ChildRun"))
13517 .collect::<Vec<_>>();
13518 let duplicate_delivery = terminal_events.first().is_some_and(|first| {
13519 terminal_events.iter().all(|event| {
13520 event.event_type == first.event_type && event.payload == first.payload
13521 })
13522 });
13523 if outcomes.len() > 1 && !duplicate_delivery {
13524 return Err(invalid_recorded_history(
13525 "duplicate_child_workflow_terminal_event",
13526 sequence,
13527 "at most one terminal child event",
13528 "multiple terminal child events",
13529 "child workflow history settles one command more than once",
13530 ));
13531 }
13532 return Ok(RecordedCommand::ChildWorkflow {
13533 sequence,
13534 workflow_type,
13535 outcome: outcomes.pop(),
13536 parallel_group_path,
13537 });
13538 }
13539
13540 if !signal_wait_events.is_empty() {
13541 let opened: Vec<_> = signal_wait_events
13542 .iter()
13543 .copied()
13544 .filter(|event| event.event_type == "SignalWaitOpened")
13545 .collect();
13546 if opened.len() != 1 {
13547 return Err(invalid_recorded_history(
13548 "signal_wait_open_missing_or_duplicate",
13549 sequence,
13550 "one SignalWaitOpened event",
13551 &format!("{} SignalWaitOpened events", opened.len()),
13552 "signal replay requires exactly one canonical wait-open event",
13553 ));
13554 }
13555
13556 let applied: Vec<_> = signal_wait_events
13557 .iter()
13558 .copied()
13559 .filter(|event| event.event_type == "SignalApplied")
13560 .collect();
13561 if applied.len() > 1 {
13562 return Err(invalid_recorded_history(
13563 "duplicate_signal_wait_apply",
13564 sequence,
13565 "at most one SignalApplied event",
13566 "multiple SignalApplied events",
13567 "signal history applies one durable wait more than once",
13568 ));
13569 }
13570
13571 let signal_names = signal_wait_events
13572 .iter()
13573 .map(|event| required_signal_wait_name(event, sequence))
13574 .collect::<Result<Vec<_>>>()?;
13575 let signal_name = signal_names
13576 .first()
13577 .expect("signal wait events are not empty")
13578 .clone();
13579 if signal_names.iter().any(|candidate| candidate != &signal_name) {
13580 return Err(invalid_recorded_history(
13581 "signal_wait_identity_mismatch",
13582 sequence,
13583 &signal_name,
13584 "conflicting signal names",
13585 "signal wait lifecycle events at one workflow sequence disagree on identity",
13586 ));
13587 }
13588 let value = applied
13589 .first()
13590 .map(|event| decode_signal_event_arguments(event, fallback_codec))
13591 .transpose()?;
13592 return Ok(RecordedCommand::SignalWait {
13593 sequence,
13594 signal_name,
13595 value,
13596 parallel_group_path: recorded_parallel_group_path(
13597 &signal_wait_events,
13598 sequence,
13599 )?,
13600 });
13601 }
13602
13603 if !condition_wait_events.is_empty() {
13604 return recorded_condition_wait(
13605 sequence,
13606 &condition_wait_events,
13607 events,
13608 );
13609 }
13610
13611 if !search_attribute_events.is_empty() {
13612 if search_attribute_events.len() != 1 {
13613 return Err(invalid_recorded_history(
13614 "duplicate_search_attribute_update",
13615 sequence,
13616 "one SearchAttributesUpserted event",
13617 &format!(
13618 "{} SearchAttributesUpserted events",
13619 search_attribute_events.len()
13620 ),
13621 "search-attribute history records one workflow command more than once",
13622 ));
13623 }
13624 let payload = &search_attribute_events[0].payload;
13625 let attributes = payload
13626 .get("attributes")
13627 .filter(|value| value.as_object().is_some_and(|values| !values.is_empty()))
13628 .cloned()
13629 .ok_or_else(|| {
13630 invalid_recorded_history(
13631 "search_attribute_update_missing",
13632 sequence,
13633 "non-empty attributes object",
13634 "missing or invalid attributes",
13635 "search-attribute history is missing its recorded mutation",
13636 )
13637 })?;
13638 let attribute_types =
13639 recorded_search_attribute_types(payload, &attributes, sequence)?;
13640 return Ok(RecordedCommand::SearchAttributes {
13641 sequence,
13642 attributes,
13643 attribute_types,
13644 });
13645 }
13646
13647 if !side_effect_events.is_empty() {
13648 if side_effect_events.len() != 1 {
13649 return Err(invalid_recorded_history(
13650 "duplicate_side_effect_record",
13651 sequence,
13652 "one SideEffectRecorded event",
13653 &format!("{} SideEffectRecorded events", side_effect_events.len()),
13654 "side-effect history records one workflow command more than once",
13655 ));
13656 }
13657 let event = side_effect_events[0];
13658 let result = event.payload.get("result").ok_or_else(|| {
13659 invalid_recorded_history(
13660 "side_effect_result_missing",
13661 sequence,
13662 "recorded result payload",
13663 "missing result",
13664 "side-effect history is missing its recorded value",
13665 )
13666 })?;
13667 let has_published_envelope = result.as_str().is_some()
13668 || result.as_object().is_some_and(|envelope| {
13669 envelope.get("codec").and_then(Value::as_str).is_some()
13670 && envelope.get("blob").and_then(Value::as_str).is_some()
13671 });
13672 if !has_published_envelope {
13673 return Err(invalid_recorded_history(
13674 "side_effect_payload_malformed",
13675 sequence,
13676 "payload blob or {codec, blob} envelope",
13677 &result.to_string(),
13678 "side-effect history result does not use a published payload envelope",
13679 ));
13680 }
13681 let codec = event
13682 .payload
13683 .get("payload_codec")
13684 .and_then(Value::as_str)
13685 .unwrap_or(fallback_codec);
13686 let value = decode_wire_avro_value(result, codec).map_err(|error| {
13687 if error.to_string().contains("unsupported_payload_codec") {
13688 return error;
13689 }
13690
13691 invalid_recorded_history(
13692 "side_effect_payload_incompatible",
13693 sequence,
13694 &format!("valid {codec} payload envelope"),
13695 &error.to_string(),
13696 "side-effect history payload cannot be decoded with its recorded codec",
13697 )
13698 })?;
13699 return Ok(RecordedCommand::SideEffect { sequence, value });
13700 }
13701
13702 if !version_marker_events.is_empty() {
13703 if version_marker_events.len() != 1 {
13704 return Err(invalid_recorded_history(
13705 "duplicate_version_marker_record",
13706 sequence,
13707 "one VersionMarkerRecorded event",
13708 &format!("{} VersionMarkerRecorded events", version_marker_events.len()),
13709 "version-marker history records one workflow command more than once",
13710 ));
13711 }
13712 let payload = &version_marker_events[0].payload;
13713 let change_id = payload
13714 .get("change_id")
13715 .and_then(Value::as_str)
13716 .filter(|value| !value.is_empty())
13717 .map(str::to_string)
13718 .ok_or_else(|| {
13719 invalid_recorded_history(
13720 "version_marker_field_missing",
13721 sequence,
13722 "non-empty change_id",
13723 "missing or invalid change_id",
13724 "version-marker history is missing its stable change ID",
13725 )
13726 })?;
13727 let version = required_version_i32(payload, "version", sequence)?;
13728 let min_supported = required_version_i32(payload, "min_supported", sequence)?;
13729 let max_supported = required_version_i32(payload, "max_supported", sequence)?;
13730 if min_supported > max_supported || version < min_supported || version > max_supported {
13731 return Err(invalid_recorded_history(
13732 "version_marker_history_range_invalid",
13733 sequence,
13734 "min_supported <= version <= max_supported",
13735 &format!("{min_supported} <= {version} <= {max_supported}"),
13736 "recorded version marker contains an internally incompatible range",
13737 ));
13738 }
13739 return Ok(RecordedCommand::VersionMarker {
13740 sequence,
13741 change_id,
13742 version,
13743 });
13744 }
13745
13746 if !memo_events.is_empty() {
13747 if memo_events.len() != 1 {
13748 return Err(invalid_recorded_history(
13749 "duplicate_memo_upsert_record",
13750 sequence,
13751 "one MemoUpserted event",
13752 &format!("{} MemoUpserted events", memo_events.len()),
13753 "memo history records one workflow update more than once",
13754 ));
13755 }
13756 let payload = &memo_events[0].payload;
13757 let entries = payload.get("entries").cloned().ok_or_else(|| {
13758 invalid_recorded_history(
13759 "memo_entries_missing",
13760 sequence,
13761 "memo entries object",
13762 "missing entries",
13763 "MemoUpserted history is missing replay identity entries",
13764 )
13765 })?;
13766 let entries = decode_memo_history_map(&entries, true).map_err(|error| {
13767 invalid_recorded_history(
13768 "memo_entries_invalid",
13769 sequence,
13770 "valid canonical memo entries",
13771 &error.to_string(),
13772 "MemoUpserted history contains invalid replay identity entries",
13773 )
13774 })?;
13775 let merged = payload.get("merged").cloned().ok_or_else(|| {
13776 invalid_recorded_history(
13777 "memo_merged_projection_missing",
13778 sequence,
13779 "merged memo projection",
13780 "missing merged",
13781 "MemoUpserted history is missing its merged projection",
13782 )
13783 })?;
13784 decode_memo_history_map(&merged, false).map_err(|error| {
13785 invalid_recorded_history(
13786 "memo_merged_projection_invalid",
13787 sequence,
13788 "valid merged memo projection",
13789 &error.to_string(),
13790 "MemoUpserted history contains an invalid merged projection",
13791 )
13792 })?;
13793
13794 return Ok(RecordedCommand::Memo { sequence, entries });
13795 }
13796 let scheduled: Vec<_> = timer_events
13797 .iter()
13798 .copied()
13799 .filter(|event| event.event_type == "TimerScheduled")
13800 .collect();
13801 let fired: Vec<_> = timer_events
13802 .iter()
13803 .copied()
13804 .filter(|event| event.event_type == "TimerFired")
13805 .collect();
13806 if scheduled.len() != 1 {
13807 return Err(invalid_recorded_history(
13808 "timer_schedule_missing_or_duplicate",
13809 sequence,
13810 "one TimerScheduled event",
13811 &format!("{} TimerScheduled events", scheduled.len()),
13812 "timer replay requires exactly one recorded schedule event",
13813 ));
13814 }
13815 if fired.len() > 1 {
13816 return Err(invalid_recorded_history(
13817 "duplicate_timer_fire",
13818 sequence,
13819 "at most one TimerFired event",
13820 "multiple TimerFired events",
13821 "timer history contains more than one fire event for a workflow sequence",
13822 ));
13823 }
13824
13825 let scheduled = scheduled[0];
13826 let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
13827 let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
13828 if let Some(fired) = fired.first() {
13829 let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
13830 if fired_timer_id != timer_id {
13831 return Err(invalid_recorded_history(
13832 "timer_identity_mismatch",
13833 sequence,
13834 &timer_id,
13835 &fired_timer_id,
13836 "TimerFired does not correspond to the recorded TimerScheduled event",
13837 ));
13838 }
13839 let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
13840 if fired_delay != delay_seconds {
13841 return Err(invalid_recorded_history(
13842 "timer_history_delay_mismatch",
13843 sequence,
13844 &delay_seconds.to_string(),
13845 &fired_delay.to_string(),
13846 "TimerScheduled and TimerFired record different delays",
13847 ));
13848 }
13849 }
13850
13851 Ok(RecordedCommand::Timer {
13852 sequence,
13853 delay_seconds,
13854 fired: !fired.is_empty(),
13855 parallel_group_path: recorded_parallel_group_path(&timer_events, sequence)?,
13856 })
13857 })
13858 .collect::<Result<_>>()?;
13859
13860 let mut marker_sequences = HashMap::new();
13861 for command in &commands {
13862 if let RecordedCommand::VersionMarker {
13863 sequence,
13864 change_id,
13865 ..
13866 } = command
13867 {
13868 if let Some(first_sequence) = marker_sequences.insert(change_id.clone(), *sequence) {
13869 return Err(invalid_recorded_history(
13870 "duplicate_version_marker",
13871 *sequence,
13872 &format!("one marker for change ID {change_id:?}"),
13873 &format!("markers at sequences {first_sequence} and {sequence}"),
13874 "workflow history contains duplicate markers for one stable change ID",
13875 ));
13876 }
13877 }
13878 }
13879
13880 Ok(commands)
13881}
13882
13883fn required_version_i32(payload: &Value, field: &str, sequence: u64) -> Result<i32> {
13884 payload
13885 .get(field)
13886 .and_then(Value::as_i64)
13887 .and_then(|value| i32::try_from(value).ok())
13888 .ok_or_else(|| {
13889 invalid_recorded_history(
13890 "version_marker_field_missing",
13891 sequence,
13892 &format!("integer {field}"),
13893 "missing or out-of-range integer",
13894 "version-marker history is missing a required integer field",
13895 )
13896 })
13897}
13898
13899fn durable_event_sequence(event: &HistoryEvent) -> Option<u64> {
13900 event
13901 .payload
13902 .get("sequence")
13903 .or_else(|| event.payload.get("workflow_sequence"))
13904 .or_else(|| event.raw.get("sequence"))
13905 .or_else(|| event.raw.get("workflow_sequence"))
13906 .and_then(value_as_u64)
13907}
13908
13909fn is_internal_timer_event(event: &HistoryEvent) -> bool {
13910 matches!(
13911 event
13912 .payload
13913 .get("timer_kind")
13914 .or_else(|| event.raw.get("timer_kind"))
13915 .and_then(Value::as_str),
13916 Some("condition_timeout" | "signal_timeout")
13917 )
13918}
13919
13920fn is_recorded_condition_wait_event(event: &HistoryEvent) -> bool {
13921 matches!(
13922 event.event_type.as_str(),
13923 "ConditionWaitOpened" | "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
13924 )
13925}
13926
13927fn recorded_condition_wait(
13928 sequence: u64,
13929 condition_events: &[&HistoryEvent],
13930 all_events: &[HistoryEvent],
13931) -> Result<RecordedCommand> {
13932 let opened = condition_events
13933 .iter()
13934 .copied()
13935 .filter(|event| event.event_type == "ConditionWaitOpened")
13936 .collect::<Vec<_>>();
13937 if opened.len() != 1 {
13938 return Err(invalid_recorded_history(
13939 "condition_wait_open_missing_or_duplicate",
13940 sequence,
13941 "one ConditionWaitOpened event",
13942 &format!("{} ConditionWaitOpened events", opened.len()),
13943 "condition replay requires exactly one canonical wait-open event",
13944 ));
13945 }
13946 let terminal = condition_events
13947 .iter()
13948 .copied()
13949 .filter(|event| {
13950 matches!(
13951 event.event_type.as_str(),
13952 "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
13953 )
13954 })
13955 .collect::<Vec<_>>();
13956 if terminal.len() > 1 {
13957 return Err(invalid_recorded_history(
13958 "duplicate_condition_wait_terminal_event",
13959 sequence,
13960 "at most one condition terminal event",
13961 "multiple condition terminal events",
13962 "condition history settles one durable wait more than once",
13963 ));
13964 }
13965
13966 let opened = opened[0];
13967 let condition_wait_id = required_condition_wait_id(opened, sequence)?;
13968 let occurrence_id = required_condition_wait_occurrence_id(opened, sequence)?;
13969 for event in condition_events
13970 .iter()
13971 .copied()
13972 .filter(|event| !std::ptr::eq(*event, opened))
13973 {
13974 let event_wait_id = required_condition_wait_id(event, sequence)?;
13975 if event_wait_id != condition_wait_id {
13976 return Err(invalid_recorded_history(
13977 "condition_wait_id_mismatch",
13978 sequence,
13979 &condition_wait_id,
13980 &event_wait_id,
13981 "condition lifecycle events at one sequence disagree on wait identity",
13982 ));
13983 }
13984 let event_occurrence_id = required_condition_wait_occurrence_id(event, sequence)?;
13985 if event_occurrence_id != occurrence_id {
13986 return Err(invalid_recorded_history(
13987 "condition_wait_occurrence_history_mismatch",
13988 sequence,
13989 &occurrence_id,
13990 &event_occurrence_id,
13991 "condition lifecycle events at one sequence disagree on authored occurrence identity",
13992 ));
13993 }
13994 }
13995
13996 let condition_key = optional_non_empty_history_string(opened, "condition_key");
13997 let predicate_identity = opened
13998 .payload
13999 .get("condition_definition_fingerprint")
14000 .and_then(Value::as_str)
14001 .filter(|value| !value.is_empty())
14002 .map(str::to_string)
14003 .ok_or_else(|| {
14004 invalid_recorded_history(
14005 "condition_wait_predicate_fingerprint_missing",
14006 sequence,
14007 "non-empty condition_definition_fingerprint",
14008 &opened.event_type,
14009 "canonical condition history is missing its predicate identity",
14010 )
14011 })?;
14012 let timeout_seconds = optional_history_u64(opened, "timeout_seconds", sequence)?;
14013 for event in condition_events
14014 .iter()
14015 .copied()
14016 .filter(|event| !std::ptr::eq(*event, opened))
14017 {
14018 for (field, opened_value) in [
14019 ("condition_key", condition_key.as_deref()),
14020 (
14021 "condition_definition_fingerprint",
14022 Some(predicate_identity.as_str()),
14023 ),
14024 ] {
14025 if let Some(value) = optional_non_empty_history_string(event, field) {
14026 if opened_value.is_some_and(|opened_value| opened_value != value) {
14027 return Err(invalid_recorded_history(
14028 "condition_wait_definition_history_mismatch",
14029 sequence,
14030 opened_value.unwrap_or_default(),
14031 &value,
14032 "condition lifecycle events disagree on the recorded definition",
14033 ));
14034 }
14035 }
14036 }
14037 if let Some(event_timeout) = optional_history_u64(event, "timeout_seconds", sequence)? {
14038 if timeout_seconds.is_some_and(|opened_timeout| opened_timeout != event_timeout) {
14039 return Err(invalid_recorded_history(
14040 "condition_wait_definition_history_mismatch",
14041 sequence,
14042 &format!("{}s", timeout_seconds.unwrap_or_default()),
14043 &format!("{event_timeout}s"),
14044 "condition lifecycle events disagree on the recorded timeout",
14045 ));
14046 }
14047 }
14048 }
14049
14050 let timeout_timer_events = all_events
14051 .iter()
14052 .filter(|event| {
14053 matches!(
14054 event.event_type.as_str(),
14055 "TimerScheduled" | "TimerCancelled" | "TimerFired"
14056 ) && event.payload.get("timer_kind").and_then(Value::as_str)
14057 == Some("condition_timeout")
14058 && event
14059 .payload
14060 .get("condition_wait_id")
14061 .and_then(Value::as_str)
14062 == Some(condition_wait_id.as_str())
14063 })
14064 .collect::<Vec<_>>();
14065 let scheduled = timeout_timer_events
14066 .iter()
14067 .copied()
14068 .filter(|event| event.event_type == "TimerScheduled")
14069 .collect::<Vec<_>>();
14070 let fired = timeout_timer_events
14071 .iter()
14072 .copied()
14073 .filter(|event| event.event_type == "TimerFired")
14074 .collect::<Vec<_>>();
14075 if scheduled.len() > 1 || fired.len() > 1 || (!fired.is_empty() && scheduled.len() != 1) {
14076 return Err(invalid_recorded_history(
14077 "condition_wait_timeout_history_invalid",
14078 sequence,
14079 "one timeout schedule and at most one fire",
14080 &format!("{} schedules and {} fires", scheduled.len(), fired.len()),
14081 "condition timeout history has a missing or duplicate lifecycle event",
14082 ));
14083 }
14084 if let Some(scheduled) = scheduled.first() {
14085 let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
14086 let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
14087 if timeout_seconds.is_some_and(|timeout| timeout != delay_seconds) {
14088 return Err(invalid_recorded_history(
14089 "condition_wait_timeout_delay_mismatch",
14090 sequence,
14091 &format!("{}s", timeout_seconds.unwrap_or_default()),
14092 &format!("{delay_seconds}s"),
14093 "condition timeout timer differs from the wait definition",
14094 ));
14095 }
14096 if let Some(fired) = fired.first() {
14097 let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
14098 let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
14099 if fired_timer_id != timer_id || fired_delay != delay_seconds {
14100 return Err(invalid_recorded_history(
14101 "condition_wait_timeout_identity_mismatch",
14102 sequence,
14103 &format!("{timer_id}:{delay_seconds}s"),
14104 &format!("{fired_timer_id}:{fired_delay}s"),
14105 "condition timeout fire does not match its durable schedule",
14106 ));
14107 }
14108 }
14109 }
14110
14111 let result = terminal.first().map(|event| {
14112 if event.event_type == "ConditionWaitTimedOut" {
14113 ConditionWaitResult::TimedOut
14114 } else {
14115 ConditionWaitResult::Satisfied
14116 }
14117 });
14118 let result = if !fired.is_empty() {
14119 if result == Some(ConditionWaitResult::Satisfied) {
14120 return Err(invalid_recorded_history(
14121 "condition_wait_terminal_conflict",
14122 sequence,
14123 "one satisfied or timed-out outcome",
14124 "satisfied event and fired timeout",
14125 "condition history records conflicting terminal outcomes",
14126 ));
14127 }
14128 Some(ConditionWaitResult::TimedOut)
14129 } else {
14130 result
14131 };
14132
14133 Ok(RecordedCommand::ConditionWait {
14134 sequence,
14135 occurrence_id,
14136 condition_key,
14137 predicate_identity,
14138 timeout_seconds,
14139 result,
14140 parallel_group_path: recorded_parallel_group_path(condition_events, sequence)?,
14141 })
14142}
14143
14144fn required_condition_wait_occurrence_id(event: &HistoryEvent, sequence: u64) -> Result<String> {
14145 event
14146 .payload
14147 .get("condition_wait_occurrence_id")
14148 .and_then(Value::as_str)
14149 .filter(|value| !value.is_empty())
14150 .map(str::to_string)
14151 .ok_or_else(|| {
14152 invalid_recorded_history(
14153 "condition_wait_occurrence_id_missing",
14154 sequence,
14155 "non-empty condition_wait_occurrence_id",
14156 &event.event_type,
14157 "condition history is missing authored occurrence identity",
14158 )
14159 })
14160}
14161
14162fn required_condition_wait_id(event: &HistoryEvent, sequence: u64) -> Result<String> {
14163 event
14164 .payload
14165 .get("condition_wait_id")
14166 .and_then(Value::as_str)
14167 .filter(|value| !value.is_empty())
14168 .map(str::to_string)
14169 .ok_or_else(|| {
14170 invalid_recorded_history(
14171 "condition_wait_id_missing",
14172 sequence,
14173 "non-empty condition_wait_id",
14174 &event.event_type,
14175 "canonical condition history is missing its durable wait identity",
14176 )
14177 })
14178}
14179
14180fn optional_non_empty_history_string(event: &HistoryEvent, field: &str) -> Option<String> {
14181 event
14182 .payload
14183 .get(field)
14184 .and_then(Value::as_str)
14185 .filter(|value| !value.is_empty())
14186 .map(str::to_string)
14187}
14188
14189fn optional_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<Option<u64>> {
14190 match event.payload.get(field) {
14191 None | Some(Value::Null) => Ok(None),
14192 Some(value) => value_as_u64(value).map(Some).ok_or_else(|| {
14193 invalid_recorded_history(
14194 "condition_wait_definition_invalid",
14195 sequence,
14196 &format!("non-negative integer {field}"),
14197 &value.to_string(),
14198 "condition history contains an invalid numeric definition field",
14199 )
14200 }),
14201 }
14202}
14203
14204fn required_signal_wait_name(event: &HistoryEvent, sequence: u64) -> Result<String> {
14205 event
14206 .payload
14207 .get("signal_name")
14208 .or_else(|| event.raw.get("signal_name"))
14209 .and_then(Value::as_str)
14210 .filter(|value| !value.is_empty())
14211 .map(str::to_string)
14212 .ok_or_else(|| {
14213 invalid_recorded_history(
14214 "signal_wait_name_missing",
14215 sequence,
14216 "non-empty signal_name",
14217 &event.event_type,
14218 "canonical signal-wait history is missing its signal identity",
14219 )
14220 })
14221}
14222
14223fn is_recorded_signal_wait_event(event: &HistoryEvent) -> bool {
14224 matches!(
14225 event.event_type.as_str(),
14226 "SignalWaitOpened" | "SignalApplied"
14227 )
14228}
14229
14230fn required_history_string(event: &HistoryEvent, field: &str, sequence: u64) -> Result<String> {
14231 event
14232 .payload
14233 .get(field)
14234 .and_then(Value::as_str)
14235 .filter(|value| !value.is_empty())
14236 .map(str::to_string)
14237 .ok_or_else(|| {
14238 invalid_recorded_history(
14239 "timer_history_field_missing",
14240 sequence,
14241 field,
14242 &event.event_type,
14243 "timer history is missing a required identity field",
14244 )
14245 })
14246}
14247
14248fn required_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<u64> {
14249 event
14250 .payload
14251 .get(field)
14252 .and_then(value_as_u64)
14253 .ok_or_else(|| {
14254 invalid_recorded_history(
14255 "timer_history_field_missing",
14256 sequence,
14257 field,
14258 &event.event_type,
14259 "timer history is missing a required numeric field",
14260 )
14261 })
14262}
14263
14264fn recorded_search_attribute_types(
14265 payload: &Value,
14266 attributes: &Value,
14267 sequence: u64,
14268) -> Result<RecordedSnapshotValue<BTreeMap<String, String>>> {
14269 let Some(raw_types) = payload.get("attribute_types") else {
14270 return Ok(RecordedSnapshotValue::Unknown);
14274 };
14275 let Some(raw_types) = raw_types.as_object() else {
14276 return Err(invalid_recorded_history(
14277 "search_attribute_types_malformed",
14278 sequence,
14279 "canonical attribute type map",
14280 &raw_types.to_string(),
14281 "search-attribute history contains malformed type identity",
14282 ));
14283 };
14284 let attribute_keys = attributes
14285 .as_object()
14286 .expect("recorded search attributes were validated as an object");
14287 let mut types = BTreeMap::new();
14288 for (key, value) in raw_types {
14289 let Some(attribute_type) = value.as_str() else {
14290 return Err(invalid_recorded_history(
14291 "search_attribute_types_malformed",
14292 sequence,
14293 "canonical string type name",
14294 &value.to_string(),
14295 "search-attribute history contains a non-string type identity",
14296 ));
14297 };
14298 if !attribute_keys.contains_key(key)
14299 || !matches!(
14300 attribute_type,
14301 "string" | "keyword" | "keyword_list" | "int" | "float" | "bool" | "datetime"
14302 )
14303 {
14304 return Err(invalid_recorded_history(
14305 "search_attribute_types_malformed",
14306 sequence,
14307 "canonical types for keys present in attributes",
14308 &format!("{key}:{attribute_type}"),
14309 "search-attribute history contains unsupported or orphaned type identity",
14310 ));
14311 }
14312 types.insert(key.clone(), attribute_type.to_string());
14313 }
14314 Ok(RecordedSnapshotValue::Known(types))
14315}
14316
14317fn invalid_recorded_history(
14318 reason: &str,
14319 sequence: u64,
14320 expected: &str,
14321 actual: &str,
14322 message: &str,
14323) -> Error {
14324 Error::NonDeterministicReplay(ReplayFailure::new(
14325 reason,
14326 Some(sequence),
14327 Some(expected.to_string()),
14328 Some(actual.to_string()),
14329 message,
14330 ))
14331}
14332
14333type ActivityOutcome = std::result::Result<AvroValue, ActivityFailure>;
14334
14335fn activity_outcome(
14336 event: &HistoryEvent,
14337 fallback_codec: &str,
14338 recorded_activity_type: Option<String>,
14339) -> Result<ActivityOutcome> {
14340 if event.event_type == "ActivityCompleted" {
14341 let codec = event
14342 .payload
14343 .get("payload_codec")
14344 .and_then(Value::as_str)
14345 .unwrap_or(fallback_codec);
14346 return Ok(Ok(decode_wire_avro_value(
14347 event.payload.get("result").unwrap_or(&Value::Null),
14348 codec,
14349 )?));
14350 }
14351
14352 let payload = &event.payload;
14353 let (kind, fallback_reason, fallback_message) = match event.event_type.as_str() {
14354 "ActivityFailed" => (ActivityFailureKind::Failed, "activity", "activity failed"),
14355 "ActivityCancelled" => (
14356 ActivityFailureKind::Cancelled,
14357 "cancelled",
14358 "activity was cancelled",
14359 ),
14360 "ActivityTimedOut" => (
14361 ActivityFailureKind::TimedOut,
14362 "timeout",
14363 "activity timed out",
14364 ),
14365 _ => unreachable!("activity_outcome is called only for terminal activity events"),
14366 };
14367 let exception = payload
14368 .get("exception")
14369 .filter(|value| !value.is_null())
14370 .cloned();
14371 let failure_category = payload_string(payload, "failure_category");
14372 let timeout_kind = payload_string(payload, "timeout_kind");
14373 let reason = payload_string(payload, "reason").unwrap_or_else(|| match kind {
14374 ActivityFailureKind::Failed => failure_category
14375 .clone()
14376 .unwrap_or_else(|| fallback_reason.to_string()),
14377 ActivityFailureKind::Cancelled => fallback_reason.to_string(),
14378 ActivityFailureKind::TimedOut => timeout_kind
14379 .clone()
14380 .unwrap_or_else(|| fallback_reason.to_string()),
14381 });
14382 let message = payload_string(payload, "message")
14383 .or_else(|| {
14384 exception
14385 .as_ref()
14386 .and_then(|value| payload_string(value, "message"))
14387 })
14388 .unwrap_or_else(|| fallback_message.to_string());
14389
14390 Ok(Err(ActivityFailure {
14391 kind,
14392 reason,
14393 message,
14394 activity_execution_id: payload_string(payload, "activity_execution_id"),
14395 activity_attempt_id: payload_string(payload, "activity_attempt_id"),
14396 activity_type: payload_string(payload, "activity_type")
14397 .or_else(|| payload_string(payload, "activity_name"))
14398 .or(recorded_activity_type),
14399 activity_class: payload_string(payload, "activity_class"),
14400 attempt_number: payload.get("attempt_number").and_then(value_as_u64),
14401 failure_id: payload_string(payload, "failure_id"),
14402 failure_category,
14403 timeout_kind,
14404 non_retryable: payload
14405 .get("non_retryable")
14406 .and_then(Value::as_bool)
14407 .unwrap_or(false),
14408 exception_type: payload_string(payload, "exception_type").or_else(|| {
14409 exception
14410 .as_ref()
14411 .and_then(|value| payload_string(value, "type"))
14412 }),
14413 exception_class: payload_string(payload, "exception_class").or_else(|| {
14414 exception
14415 .as_ref()
14416 .and_then(|value| payload_string(value, "class"))
14417 }),
14418 code: payload
14419 .get("code")
14420 .filter(|value| !value.is_null())
14421 .cloned(),
14422 exception,
14423 }))
14424}
14425
14426type ChildWorkflowOutcome = std::result::Result<ChildWorkflowAvroResult, ChildWorkflowFailure>;
14427
14428fn child_workflow_outcomes(
14429 events: &[HistoryEvent],
14430 fallback_codec: &str,
14431 parent: WorkflowIdentity,
14432) -> Result<Vec<ChildWorkflowOutcome>> {
14433 let mut outcomes = Vec::new();
14434
14435 for event in events {
14436 let kind = match event.event_type.as_str() {
14437 "ChildRunCompleted" => None,
14438 "ChildRunFailed" => Some((
14439 ChildWorkflowFailureKind::Failed,
14440 "child_workflow",
14441 "child workflow failed",
14442 )),
14443 "ChildRunCancelled" => Some((
14444 ChildWorkflowFailureKind::Cancelled,
14445 "cancelled",
14446 "child workflow was cancelled",
14447 )),
14448 "ChildRunTerminated" => Some((
14449 ChildWorkflowFailureKind::Terminated,
14450 "terminated",
14451 "child workflow was terminated",
14452 )),
14453 _ => continue,
14454 };
14455 let payload = &event.payload;
14456 let child_workflow_id = payload_string(payload, "child_workflow_instance_id");
14457 let child_workflow_run_id = payload_string(payload, "child_workflow_run_id");
14458 let child_workflow_type = payload_string(payload, "child_workflow_type");
14459
14460 if let Some((kind, reason, fallback_message)) = kind {
14461 let exception = payload
14462 .get("exception")
14463 .filter(|value| !value.is_null())
14464 .cloned();
14465 let message = payload_string(payload, "message")
14466 .or_else(|| {
14467 exception
14468 .as_ref()
14469 .and_then(|value| payload_string(value, "message"))
14470 })
14471 .unwrap_or_else(|| fallback_message.to_string());
14472 let exception_type = payload_string(payload, "exception_type").or_else(|| {
14473 exception
14474 .as_ref()
14475 .and_then(|value| payload_string(value, "type"))
14476 });
14477 let exception_class = payload_string(payload, "exception_class").or_else(|| {
14478 exception
14479 .as_ref()
14480 .and_then(|value| payload_string(value, "class"))
14481 });
14482 outcomes.push(Err(ChildWorkflowFailure {
14483 kind,
14484 reason: reason.to_string(),
14485 message,
14486 parent_workflow_id: parent.workflow_id.clone(),
14487 parent_workflow_run_id: parent.run_id.clone(),
14488 child_workflow_id,
14489 child_workflow_run_id,
14490 child_workflow_type,
14491 failure_id: payload_string(payload, "failure_id"),
14492 failure_category: payload_string(payload, "failure_category"),
14493 exception_type,
14494 exception_class,
14495 non_retryable: payload
14496 .get("non_retryable")
14497 .and_then(Value::as_bool)
14498 .unwrap_or(false),
14499 code: payload
14500 .get("code")
14501 .filter(|value| !value.is_null())
14502 .cloned(),
14503 exception,
14504 }));
14505 continue;
14506 }
14507
14508 let codec = payload
14509 .get("payload_codec")
14510 .and_then(Value::as_str)
14511 .unwrap_or(fallback_codec);
14512 let result = payload
14513 .get("result")
14514 .or_else(|| payload.get("output"))
14515 .unwrap_or(&Value::Null);
14516 outcomes.push(Ok(ChildWorkflowAvroResult {
14517 parent: parent.clone(),
14518 child: WorkflowIdentity {
14519 workflow_id: child_workflow_id,
14520 run_id: child_workflow_run_id,
14521 },
14522 child_workflow_type,
14523 result: decode_wire_avro_value(result, codec)?,
14524 }));
14525 }
14526
14527 Ok(outcomes)
14528}
14529
14530fn payload_string(payload: &Value, key: &str) -> Option<String> {
14531 payload
14532 .get(key)
14533 .and_then(Value::as_str)
14534 .filter(|value| !value.is_empty())
14535 .map(str::to_string)
14536}
14537
14538fn recorded_activity_failure_boundary(
14539 ctx: &WorkflowContext,
14540 error: &Error,
14541) -> Option<(u64, String)> {
14542 let Error::ActivityFailed(failure) = error else {
14543 return None;
14544 };
14545 if failure.kind != ActivityFailureKind::Failed {
14546 return None;
14547 }
14548 let activity_execution_id = failure
14549 .activity_execution_id
14550 .as_deref()
14551 .filter(|value| !value.is_empty())?;
14552 let state = ctx.state.lock().ok()?;
14553 let mut matches = state.recorded_commands.iter().filter_map(|command| {
14554 let RecordedCommand::Activity {
14555 sequence,
14556 outcome: Some(Err(recorded_failure)),
14557 ..
14558 } = command
14559 else {
14560 return None;
14561 };
14562 (*sequence > 0 && recorded_failure == failure)
14563 .then_some((*sequence, activity_execution_id.to_string()))
14564 });
14565 let boundary = matches.next()?;
14566 matches.next().is_none().then_some(boundary)
14567}
14568
14569fn workflow_failure_command(
14570 error: &Error,
14571 failed_activity_boundary: Option<(u64, String)>,
14572) -> Value {
14573 let (exception_type, exception_class, properties) = match error {
14574 Error::ActivityFailed(failure) => (
14575 match failure.kind {
14576 ActivityFailureKind::Failed => "ActivityFailed",
14577 ActivityFailureKind::Cancelled => "ActivityCancelled",
14578 ActivityFailureKind::TimedOut => "ActivityTimedOut",
14579 },
14580 "durable_workflow::ActivityFailure",
14581 json!({
14582 "reason": failure.reason,
14583 "activity_execution_id": failure.activity_execution_id,
14584 "activity_attempt_id": failure.activity_attempt_id,
14585 "activity_type": failure.activity_type,
14586 "activity_class": failure.activity_class,
14587 "attempt_number": failure.attempt_number,
14588 "failure_id": failure.failure_id,
14589 "failure_category": failure.failure_category,
14590 "timeout_kind": failure.timeout_kind,
14591 "activity_non_retryable": failure.non_retryable,
14592 "activity_exception_type": failure.exception_type,
14593 "activity_exception_class": failure.exception_class,
14594 "activity_code": failure.code,
14595 "activity_exception": failure.exception,
14596 }),
14597 ),
14598 Error::ChildWorkflowFailed(failure) => (
14599 match failure.kind {
14600 ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
14601 ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
14602 ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
14603 },
14604 "durable_workflow::ChildWorkflowFailure",
14605 json!({
14606 "reason": failure.reason,
14607 "parent_workflow_id": failure.parent_workflow_id,
14608 "parent_workflow_run_id": failure.parent_workflow_run_id,
14609 "child_workflow_id": failure.child_workflow_id,
14610 "child_workflow_run_id": failure.child_workflow_run_id,
14611 "child_workflow_type": failure.child_workflow_type,
14612 "failure_id": failure.failure_id,
14613 "failure_category": failure.failure_category,
14614 "child_exception_type": failure.exception_type,
14615 "child_exception_class": failure.exception_class,
14616 "child_non_retryable": failure.non_retryable,
14617 "child_code": failure.code,
14618 "child_exception": failure.exception,
14619 }),
14620 ),
14621 Error::ParallelFailed(failure) => (
14622 "ParallelFailed",
14623 "durable_workflow::ParallelFailure",
14624 json!({
14625 "parallel_group_id": failure.group_id,
14626 "parallel_member_path": failure.member_path,
14627 "parallel_group_path": failure.group_path,
14628 "completed_members": failure.completed.iter().map(|completion| &completion.member_path).collect::<Vec<_>>(),
14629 "cause_type": workflow_error_type(&failure.cause),
14630 "cause_message": failure.cause.to_string(),
14631 }),
14632 ),
14633 Error::SagaCompensationFailed(failure) => (
14634 "SagaCompensationFailed",
14635 "durable_workflow::SagaCompensationFailure",
14636 json!({
14637 "initiating_failure_type": workflow_error_type(&failure.initiating_failure),
14638 "initiating_failure_message": failure.initiating_failure.to_string(),
14639 "compensation_activity_type": failure.compensation_activity_type,
14640 "compensation_registration_order": failure.compensation_registration_order,
14641 "compensation_failure_type": workflow_error_type(&failure.compensation_failure),
14642 "compensation_failure_message": failure.compensation_failure.to_string(),
14643 }),
14644 ),
14645 Error::WorkflowCancellationRequested(_) => (
14646 "WorkflowCancellationRequested",
14647 "durable_workflow::WorkflowCancellationRequested",
14648 json!({"reason": "cancelled"}),
14649 ),
14650 Error::NonDeterministicReplay(_) => (
14651 "NonDeterministicReplay",
14652 "durable_workflow::Error",
14653 Value::Null,
14654 ),
14655 _ => ("RustWorkflowError", "durable_workflow::Error", Value::Null),
14656 };
14657 let non_retryable = match error {
14658 Error::ActivityFailed(failure) => failure.non_retryable,
14659 Error::ChildWorkflowFailed(failure) => failure.non_retryable,
14660 Error::ParallelFailed(failure) => workflow_error_non_retryable(&failure.cause),
14661 Error::SagaCompensationFailed(failure) => {
14662 workflow_error_non_retryable(&failure.compensation_failure)
14663 }
14664 Error::WorkflowCancellationRequested(_) => true,
14665 Error::NonDeterministicReplay(_) => true,
14666 _ => false,
14667 };
14668
14669 let mut command = json!({
14670 "type": "fail_workflow",
14671 "message": error.to_string(),
14672 "exception_type": exception_type,
14673 "exception_class": exception_class,
14674 "non_retryable": non_retryable,
14675 "exception": {
14676 "type": exception_type,
14677 "class": exception_class,
14678 "message": error.to_string(),
14679 "properties": properties,
14680 }
14681 });
14682
14683 if let Some((sequence, activity_execution_id)) = failed_activity_boundary {
14684 command["failed_step_sequence"] = json!(sequence);
14685 command["failed_activity_execution_id"] = json!(activity_execution_id);
14686 }
14687
14688 command
14689}
14690
14691fn workflow_error_type(error: &Error) -> &'static str {
14692 match error {
14693 Error::ActivityFailed(failure) => match failure.kind {
14694 ActivityFailureKind::Failed => "ActivityFailed",
14695 ActivityFailureKind::Cancelled => "ActivityCancelled",
14696 ActivityFailureKind::TimedOut => "ActivityTimedOut",
14697 },
14698 Error::ChildWorkflowFailed(failure) => match failure.kind {
14699 ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
14700 ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
14701 ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
14702 },
14703 Error::ParallelFailed(_) => "ParallelFailed",
14704 Error::SagaCompensationFailed(_) => "SagaCompensationFailed",
14705 Error::WorkflowCancellationRequested(_) => "WorkflowCancellationRequested",
14706 Error::NonDeterministicReplay(_) => "NonDeterministicReplay",
14707 _ => "RustWorkflowError",
14708 }
14709}
14710
14711fn workflow_error_non_retryable(error: &Error) -> bool {
14712 match error {
14713 Error::ActivityFailed(failure) => failure.non_retryable,
14714 Error::ChildWorkflowFailed(failure) => failure.non_retryable,
14715 Error::ParallelFailed(failure) => workflow_error_non_retryable(&failure.cause),
14716 Error::SagaCompensationFailed(failure) => {
14717 workflow_error_non_retryable(&failure.compensation_failure)
14718 }
14719 Error::WorkflowCancellationRequested(_) | Error::NonDeterministicReplay(_) => true,
14720 _ => false,
14721 }
14722}
14723
14724fn workflow_task_integrity_error(error: &Error) -> bool {
14725 matches!(
14726 error,
14727 Error::NonDeterministicReplay(_)
14728 | Error::Protocol(_)
14729 | Error::MissingWorkflowCommandIdentity
14730 | Error::WorkflowStatePoisoned
14731 )
14732}
14733
14734fn decode_signal_event_arguments(
14735 event: &HistoryEvent,
14736 fallback_codec: &str,
14737) -> Result<Vec<AvroValue>> {
14738 let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
14739 validate_payload_codec(codec)?;
14740 let raw = signal_history_payload(&event.payload);
14741 let decoded = match raw.filter(|value| !value.is_null()) {
14742 Some(value) => decode_wire_avro_value(value, codec)?,
14743 None => AvroValue::Array(Vec::new()),
14744 };
14745 let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
14746 unreachable!("normalize_avro_arguments always returns an array");
14747 };
14748 Ok(arguments)
14749}
14750
14751fn decode_update_event_arguments(
14752 event: &HistoryEvent,
14753 fallback_codec: &str,
14754) -> Result<Vec<AvroValue>> {
14755 let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
14756 validate_payload_codec(codec)?;
14757 let decoded = match event
14758 .payload
14759 .get("arguments")
14760 .filter(|value| !value.is_null())
14761 {
14762 Some(value) => decode_wire_avro_value(value, codec)?,
14763 None => AvroValue::Array(Vec::new()),
14764 };
14765 let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
14766 unreachable!("normalize_avro_arguments always returns an array");
14767 };
14768 Ok(arguments)
14769}
14770
14771fn hydrate_query_history_from_export(task: &mut QueryTask) -> Result<()> {
14772 let Some(export_events) = task
14773 .history_export
14774 .as_ref()
14775 .and_then(|export| export.get("history_events"))
14776 .and_then(Value::as_array)
14777 else {
14778 return Ok(());
14779 };
14780
14781 if export_events.len() > task.history_events.len() {
14782 task.history_events = serde_json::from_value(Value::Array(export_events.clone()))?;
14783 }
14784
14785 Ok(())
14786}
14787
14788fn enrich_query_history_from_export(task: &mut QueryTask) -> Result<()> {
14789 let Some(export) = task.history_export.as_ref() else {
14790 return Ok(());
14791 };
14792 let signals = export
14793 .get("signals")
14794 .and_then(Value::as_array)
14795 .cloned()
14796 .unwrap_or_default();
14797 let activities = export
14798 .get("activities")
14799 .and_then(Value::as_array)
14800 .cloned()
14801 .unwrap_or_default();
14802 let export_codec = export
14803 .get("payloads")
14804 .and_then(|payloads| payloads.get("codec"))
14805 .and_then(Value::as_str)
14806 .unwrap_or(&task.payload_codec)
14807 .to_string();
14808 let mut signal_name_offsets: HashMap<String, usize> = HashMap::new();
14809
14810 for event in &mut task.history_events {
14811 if event.event_type == "ActivityCompleted" {
14812 let sequence = event
14813 .payload
14814 .get("sequence")
14815 .or_else(|| event.payload.get("workflow_sequence"))
14816 .and_then(value_as_u64);
14817 let Some(activity) = sequence.and_then(|sequence| {
14818 activities.iter().find(|activity| {
14819 activity.get("sequence").and_then(value_as_u64) == Some(sequence)
14820 })
14821 }) else {
14822 continue;
14823 };
14824 let Some(payload) = event.payload.as_object_mut() else {
14825 continue;
14826 };
14827 if missing_payload(payload.get("result")) {
14828 if let Some(result) = activity
14829 .get("result")
14830 .filter(|value| !missing_payload(Some(value)))
14831 {
14832 payload.insert("result".to_string(), result.clone());
14833 }
14834 }
14835 for field in ["payload_codec", "activity_type"] {
14836 if payload
14837 .get(field)
14838 .and_then(Value::as_str)
14839 .unwrap_or_default()
14840 .is_empty()
14841 {
14842 if let Some(value) = activity.get(field) {
14843 payload.insert(field.to_string(), value.clone());
14844 }
14845 }
14846 }
14847 continue;
14848 }
14849
14850 if event.event_type != "SignalReceived" && event.event_type != "SignalApplied" {
14851 continue;
14852 }
14853 let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
14854 let command_id = event
14855 .payload
14856 .get("workflow_command_id")
14857 .or_else(|| event.raw.get("workflow_command_id"))
14858 .and_then(Value::as_str);
14859 let signal_name = event
14860 .payload
14861 .get("signal_name")
14862 .and_then(Value::as_str)
14863 .unwrap_or_default()
14864 .to_string();
14865 let matched = signals
14866 .iter()
14867 .find(|signal| {
14868 signal_id.is_some() && signal.get("id").and_then(Value::as_str) == signal_id
14869 })
14870 .or_else(|| {
14871 signals.iter().find(|signal| {
14872 command_id.is_some()
14873 && signal.get("command_id").and_then(Value::as_str) == command_id
14874 })
14875 })
14876 .or_else(|| {
14877 let offset = signal_name_offsets.entry(signal_name.clone()).or_default();
14878 let signal = signals
14879 .iter()
14880 .filter(|signal| {
14881 signal.get("name").and_then(Value::as_str) == Some(signal_name.as_str())
14882 })
14883 .nth(*offset);
14884 if signal.is_some() {
14885 *offset += 1;
14886 }
14887 signal
14888 });
14889 let Some(signal) = matched else {
14890 continue;
14891 };
14892 let signal_codec = signal
14893 .get("payload_codec")
14894 .and_then(Value::as_str)
14895 .unwrap_or(&export_codec);
14896 let Some(payload) = event.payload.as_object_mut() else {
14897 continue;
14898 };
14899 if missing_payload(payload.get("arguments")) {
14900 if let Some(arguments) = signal
14901 .get("arguments")
14902 .filter(|value| !missing_payload(Some(value)))
14903 {
14904 let envelope = match arguments {
14905 Value::String(blob) => json!({"codec": signal_codec, "blob": blob}),
14906 other => other.clone(),
14907 };
14908 payload.insert("arguments".to_string(), envelope);
14909 }
14910 }
14911 if payload
14912 .get("payload_codec")
14913 .and_then(Value::as_str)
14914 .unwrap_or_default()
14915 .is_empty()
14916 {
14917 payload.insert("payload_codec".to_string(), json!(signal_codec));
14918 }
14919 }
14920
14921 Ok(())
14922}
14923
14924fn missing_payload(value: Option<&Value>) -> bool {
14925 match value {
14926 None | Some(Value::Null) => true,
14927 Some(Value::String(value)) => value.is_empty(),
14928 Some(_) => false,
14929 }
14930}
14931
14932fn query_signal_events(task: &QueryTask) -> Result<Vec<QuerySignal>> {
14933 let export_signals = task
14934 .history_export
14935 .as_ref()
14936 .and_then(|export| export.get("signals"))
14937 .and_then(Value::as_array)
14938 .cloned()
14939 .unwrap_or_default();
14940 let export_codec = task
14941 .history_export
14942 .as_ref()
14943 .and_then(|export| export.get("payloads"))
14944 .and_then(|payloads| payloads.get("codec"))
14945 .and_then(Value::as_str)
14946 .unwrap_or(&task.payload_codec);
14947 let mut name_offsets: HashMap<String, usize> = HashMap::new();
14948 let mut signals = Vec::new();
14949
14950 for event in &task.history_events {
14951 if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
14952 continue;
14953 }
14954
14955 let name = event
14956 .payload
14957 .get("signal_name")
14958 .and_then(Value::as_str)
14959 .unwrap_or_default();
14960 if name.is_empty() {
14961 continue;
14962 }
14963 let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
14964 let command_id = event
14965 .payload
14966 .get("workflow_command_id")
14967 .or_else(|| event.raw.get("workflow_command_id"))
14968 .and_then(Value::as_str);
14969 let matched_export = export_signals
14970 .iter()
14971 .find(|candidate| {
14972 signal_id.is_some() && candidate.get("id").and_then(Value::as_str) == signal_id
14973 })
14974 .or_else(|| {
14975 export_signals.iter().find(|candidate| {
14976 command_id.is_some()
14977 && candidate.get("command_id").and_then(Value::as_str) == command_id
14978 })
14979 })
14980 .or_else(|| {
14981 let offset = name_offsets.entry(name.to_string()).or_default();
14982 let candidate = export_signals
14983 .iter()
14984 .filter(|candidate| candidate.get("name").and_then(Value::as_str) == Some(name))
14985 .nth(*offset);
14986 if candidate.is_some() {
14987 *offset += 1;
14988 }
14989 candidate
14990 });
14991 let codec = event
14992 .payload
14993 .get("payload_codec")
14994 .and_then(Value::as_str)
14995 .or_else(|| {
14996 matched_export
14997 .and_then(|signal| signal.get("payload_codec"))
14998 .and_then(Value::as_str)
14999 })
15000 .unwrap_or(export_codec);
15001 let raw_arguments = signal_history_payload(&event.payload)
15002 .filter(|value| !value.is_null())
15003 .or_else(|| matched_export.and_then(|signal| signal.get("arguments")));
15004 let (arguments, avro_arguments) = decode_query_signal_arguments(raw_arguments, codec)?;
15005 let workflow_sequence = event
15006 .payload
15007 .get("workflow_sequence")
15008 .and_then(value_as_u64)
15009 .or_else(|| {
15010 matched_export
15011 .and_then(|signal| signal.get("workflow_sequence"))
15012 .and_then(value_as_u64)
15013 });
15014
15015 signals.push(QuerySignal {
15016 id: signal_id.map(str::to_string).or_else(|| {
15017 matched_export
15018 .and_then(|signal| signal.get("id"))
15019 .and_then(Value::as_str)
15020 .map(str::to_string)
15021 }),
15022 name: name.to_string(),
15023 arguments,
15024 avro_arguments,
15025 workflow_sequence,
15026 });
15027 }
15028
15029 if signals.is_empty() {
15030 for signal in export_signals {
15031 if signal.get("status").and_then(Value::as_str) == Some("rejected") {
15032 continue;
15033 }
15034 let Some(name) = signal.get("name").and_then(Value::as_str) else {
15035 continue;
15036 };
15037 let codec = signal
15038 .get("payload_codec")
15039 .and_then(Value::as_str)
15040 .unwrap_or(export_codec);
15041 let (arguments, avro_arguments) =
15042 decode_query_signal_arguments(signal.get("arguments"), codec)?;
15043 signals.push(QuerySignal {
15044 id: signal.get("id").and_then(Value::as_str).map(str::to_string),
15045 name: name.to_string(),
15046 arguments,
15047 avro_arguments,
15048 workflow_sequence: signal.get("workflow_sequence").and_then(value_as_u64),
15049 });
15050 }
15051 signals.sort_by_key(|signal| signal.workflow_sequence.unwrap_or(u64::MAX));
15052 }
15053
15054 Ok(signals)
15055}
15056
15057fn decode_query_signal_arguments(
15058 raw: Option<&Value>,
15059 codec: &str,
15060) -> Result<(Vec<Value>, Vec<AvroValue>)> {
15061 validate_payload_codec(codec)?;
15062 let decoded = match raw.filter(|value| !value.is_null()) {
15063 Some(value) => decode_wire_avro_value(value, codec)?,
15064 None => AvroValue::Array(Vec::new()),
15065 };
15066 let AvroValue::Array(avro_arguments) = normalize_avro_arguments(decoded) else {
15067 unreachable!("normalize_avro_arguments always returns an array");
15068 };
15069 let arguments = avro_arguments
15070 .iter()
15071 .cloned()
15072 .map(AvroValue::into_json)
15073 .collect::<Result<Vec<_>>>()?;
15074 Ok((arguments, avro_arguments))
15075}
15076
15077fn value_as_u64(value: &Value) -> Option<u64> {
15078 value
15079 .as_u64()
15080 .or_else(|| value.as_str().and_then(|value| value.parse().ok()))
15081}
15082
15083#[cfg(test)]
15084mod tests {
15085 use super::*;
15086 mod runtime_payloads;
15087 mod runtime_uploads;
15088 use std::{
15089 fs,
15090 io::{Read, Write},
15091 net::{SocketAddr, TcpListener, TcpStream},
15092 process::Command as ProcessCommand,
15093 sync::atomic::AtomicUsize,
15094 thread,
15095 };
15096
15097 #[derive(Clone, Copy, Debug)]
15098 enum InvalidTaskPayloadCodec {
15099 Missing,
15100 Null,
15101 NonString,
15102 }
15103
15104 impl InvalidTaskPayloadCodec {
15105 fn label(self) -> &'static str {
15106 match self {
15107 Self::Missing => "missing",
15108 Self::Null => "null",
15109 Self::NonString => "non-string",
15110 }
15111 }
15112
15113 fn apply(self, task: &mut Value) {
15114 let task = task.as_object_mut().expect("task fixture object");
15115 match self {
15116 Self::Missing => {
15117 task.remove("payload_codec");
15118 }
15119 Self::Null => {
15120 task.insert("payload_codec".to_string(), Value::Null);
15121 }
15122 Self::NonString => {
15123 task.insert("payload_codec".to_string(), json!(42));
15124 }
15125 }
15126 }
15127 }
15128
15129 fn fixture_envelope(value: Value) -> Value {
15130 encode_value_envelope(&value, DEFAULT_CODEC).expect("encode Avro test fixture")
15131 }
15132
15133 fn fixture_blob(value: Value) -> String {
15134 encode_payload(&value, DEFAULT_CODEC)
15135 .expect("encode Avro test fixture")
15136 .blob
15137 }
15138
15139 #[test]
15140 fn client_builder_rejects_the_sdk_owned_api_suffix() {
15141 for base_url in [
15142 "http://127.0.0.1:8080/api",
15143 "http://localhost:8080/api/",
15144 "https://runtime.example.test/namespaces/orders/api",
15145 ] {
15146 let error = Client::builder(base_url)
15147 .build()
15148 .expect_err("SDK-owned /api suffix must be rejected during build");
15149
15150 assert!(matches!(error, Error::InvalidBaseUrl), "{base_url}");
15151 assert!(
15152 error.to_string().contains("SDK appends /api automatically"),
15153 "the validation error must explain how to fix the endpoint"
15154 );
15155 }
15156 }
15157
15158 #[test]
15159 fn client_builder_preserves_self_hosted_and_managed_runtime_prefixes() {
15160 for (base_url, expected) in [
15161 ("http://127.0.0.1:8080", "http://127.0.0.1:8080"),
15162 (
15163 "http://localhost:8080/durable-workflow/",
15164 "http://localhost:8080/durable-workflow",
15165 ),
15166 (
15167 "https://runtime.example.test/namespaces/orders",
15168 "https://runtime.example.test/namespaces/orders",
15169 ),
15170 (
15171 "https://runtime.example.test/gateway/api/namespaces/orders",
15172 "https://runtime.example.test/gateway/api/namespaces/orders",
15173 ),
15174 (
15175 "https://api.example.test/runtime/orders/",
15176 "https://api.example.test/runtime/orders",
15177 ),
15178 ] {
15179 let client = Client::builder(base_url)
15180 .build()
15181 .expect("Server and Cloud runtime base URL must remain valid");
15182
15183 assert_eq!(client.base_url, expected);
15184 }
15185 }
15186
15187 #[test]
15188 fn workflow_completion_uses_the_additive_command_protocol_floor() {
15189 assert_eq!(
15190 workflow_completion_protocol_version(&[json!({"type": "complete_workflow"})]),
15191 WORKER_PROTOCOL_VERSION
15192 );
15193 assert_eq!(
15194 workflow_completion_protocol_version(&[json!({
15195 "type": "upsert_search_attributes",
15196 "attributes": {"OrderStatus": "waiting"},
15197 })]),
15198 SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION
15199 );
15200 assert_eq!(
15201 workflow_completion_protocol_version(&[json!({
15202 "type": "upsert_search_attributes",
15203 "attributes": {"OrderStatus": "waiting"},
15204 "attribute_types": {"OrderStatus": "keyword"},
15205 })]),
15206 TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION
15207 );
15208 assert_eq!(
15209 workflow_completion_protocol_version(&[
15210 json!({"type": "upsert_memo", "entries": {"status": "waiting"}}),
15211 json!({"type": "open_condition_wait", "condition_key": "ready"}),
15212 ]),
15213 MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION
15214 );
15215 assert_eq!(
15216 workflow_completion_protocol_version(&[
15217 json!({"type": "upsert_search_attributes", "attributes": {"State": "waiting"}}),
15218 json!({"type": "open_condition_wait", "condition_key": "ready"}),
15219 ]),
15220 CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION
15221 );
15222 assert_eq!(
15223 workflow_completion_protocol_version(&[json!({
15224 "type": "open_condition_wait",
15225 "condition_wait_occurrence_id": "rust:condition-wait:0",
15226 "condition_key": "ready",
15227 })]),
15228 CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
15229 );
15230 assert_eq!(
15231 workflow_completion_protocol_version_with_message_streams(
15232 &[json!({"type": "upsert_memo", "entries": {"status": "waiting"}})],
15233 true,
15234 ),
15235 MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION
15236 );
15237 assert_eq!(
15238 workflow_completion_protocol_version_with_message_streams(
15239 &[json!({
15240 "type": "open_condition_wait",
15241 "condition_wait_occurrence_id": "rust:condition-wait:0",
15242 "condition_key": "ready",
15243 })],
15244 true,
15245 ),
15246 CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
15247 );
15248 }
15249
15250 #[test]
15251 fn portable_worker_affinity_manifest_explicitly_refuses_unimplemented_features() {
15252 let manifest = portable_worker_affinity_capability_manifest();
15253
15254 for capability in ["local_activities", "worker_sessions", "sticky_execution"] {
15255 assert_eq!(manifest[capability]["supported"], json!(false));
15256 assert_eq!(
15257 manifest[capability]["minimum_protocol_version"],
15258 json!(PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION)
15259 );
15260 assert!(manifest[capability]["reason"]
15261 .as_str()
15262 .is_some_and(|reason| !reason.is_empty()));
15263 }
15264 }
15265
15266 fn typed_fidelity_probe() -> AvroValue {
15267 AvroValue::Map(BTreeMap::from([
15268 ("bytes".to_string(), AvroValue::Bytes(vec![0, 0xff])),
15269 ("empty".to_string(), AvroValue::Map(BTreeMap::new())),
15270 (
15271 "numeric".to_string(),
15272 AvroValue::Map(BTreeMap::from([
15273 ("0".to_string(), AvroValue::String("zero".to_string())),
15274 ("1".to_string(), AvroValue::String("one".to_string())),
15275 ])),
15276 ),
15277 (
15278 "nested".to_string(),
15279 AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([(
15280 "enabled".to_string(),
15281 AvroValue::Boolean(true),
15282 )]))]),
15283 ),
15284 (
15285 "projection_collisions".to_string(),
15286 AvroValue::Array(projection_collision_probe()),
15287 ),
15288 ]))
15289 }
15290
15291 fn projection_collision_probe() -> Vec<AvroValue> {
15292 vec![
15293 AvroValue::Map(BTreeMap::from([
15294 ("$type".to_string(), AvroValue::String("bytes".to_string())),
15295 (
15296 "base64".to_string(),
15297 AvroValue::String("ordinary user text".to_string()),
15298 ),
15299 ])),
15300 AvroValue::Map(BTreeMap::from([
15301 ("$type".to_string(), AvroValue::String("map".to_string())),
15302 (
15303 "entries".to_string(),
15304 AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([
15305 ("key".to_string(), AvroValue::String("ordinary".to_string())),
15306 (
15307 "value".to_string(),
15308 AvroValue::String("user map".to_string()),
15309 ),
15310 ]))]),
15311 ),
15312 ])),
15313 ]
15314 }
15315
15316 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
15317 struct TypedContract {
15318 nested: TypedNested,
15319 mode: TypedMode,
15320 optional: Option<String>,
15321 absent: Option<String>,
15322 items: Vec<i64>,
15323 labels: BTreeMap<String, String>,
15324 bytes: serde_bytes::ByteBuf,
15325 signed: i64,
15326 finite: f64,
15327 }
15328
15329 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
15330 struct TypedNested {
15331 enabled: bool,
15332 }
15333
15334 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
15335 enum TypedMode {
15336 Detailed { label: String },
15337 }
15338
15339 fn typed_contract() -> TypedContract {
15340 TypedContract {
15341 nested: TypedNested { enabled: true },
15342 mode: TypedMode::Detailed {
15343 label: "compiler-checked".to_string(),
15344 },
15345 optional: Some("present".to_string()),
15346 absent: None,
15347 items: vec![i64::MIN, 0, i64::MAX],
15348 labels: BTreeMap::from([
15349 ("language".to_string(), "rust".to_string()),
15350 ("wire".to_string(), "avro".to_string()),
15351 ]),
15352 bytes: serde_bytes::ByteBuf::from(vec![0, 0xff, 7]),
15353 signed: -9_223_372_036_854_775_000,
15354 finite: 12.5,
15355 }
15356 }
15357
15358 #[derive(Clone, Debug, Default, PartialEq)]
15359 struct ReplayCounterState {
15360 loaded: Option<String>,
15361 count: i64,
15362 finished: bool,
15363 }
15364
15365 fn replay_counter_worker() -> Worker {
15366 let client = Client::new("http://127.0.0.1:8080").expect("client");
15367 let mut worker = Worker::new(client, "rust-workers");
15368 worker.register_replayed_workflow(
15369 "replay-counter",
15370 ReplayCounterState::default,
15371 |ctx, _input, state| async move {
15372 let loaded = ctx.activity("load-counter", json!([])).await?;
15373 state.update(|current| {
15374 current.loaded = loaded.as_str().map(str::to_string);
15375 })?;
15376 for _ in 0..2 {
15377 let signal = ctx.wait_signal("increment").await?;
15378 let amount = signal.first().and_then(Value::as_i64).unwrap_or_default();
15379 state.update(|current| current.count += amount)?;
15380 }
15381 state.update(|current| current.finished = true)?;
15382 state.read(|current| Ok(json!(current.count)))?
15383 },
15384 );
15385 worker.register_replayed_query::<ReplayCounterState, _, _>(
15386 "replay-counter",
15387 "current",
15388 |_ctx, state, _args| async move {
15389 Ok(json!({
15390 "loaded": state.loaded,
15391 "count": state.count,
15392 "finished": state.finished,
15393 }))
15394 },
15395 );
15396 worker.register_replayed_query::<ReplayCounterState, _, _>(
15397 "replay-counter",
15398 "detached-mutation",
15399 |_ctx, state, _args| async move {
15400 let mut detached = (*state).clone();
15401 detached.count = 999;
15402 Ok(json!(detached.count))
15403 },
15404 );
15405 worker.register_replayed_query::<ReplayCounterState, _, _>(
15406 "replay-counter",
15407 "failed-mutation",
15408 |_ctx, state, _args| async move {
15409 let mut detached = (*state).clone();
15410 detached.count = 999;
15411 Err(Error::WorkerLoop("query refused".to_string()))
15412 },
15413 );
15414 worker
15415 }
15416
15417 fn replay_counter_query(
15418 query_name: &str,
15419 history_events: Value,
15420 run_status: &str,
15421 ) -> QueryTask {
15422 let arguments = fixture_envelope(json!([]));
15423 serde_json::from_value(json!({
15424 "query_task_id": format!("query-{query_name}"),
15425 "workflow_type": "replay-counter",
15426 "query_name": query_name,
15427 "payload_codec": DEFAULT_CODEC,
15428 "workflow_arguments": arguments.clone(),
15429 "query_arguments": arguments,
15430 "history_events": history_events,
15431 "run_status": run_status,
15432 }))
15433 .expect("query task")
15434 }
15435
15436 fn workflow_context(history: Vec<HistoryEvent>) -> WorkflowContext {
15437 workflow_context_with_codec(history, DEFAULT_CODEC)
15438 }
15439
15440 fn workflow_context_with_codec(
15441 history: Vec<HistoryEvent>,
15442 payload_codec: &str,
15443 ) -> WorkflowContext {
15444 WorkflowContext {
15445 state: Arc::new(Mutex::new(
15446 WorkflowState::new_with_identity(
15447 history,
15448 None,
15449 None,
15450 "rust-workers".to_string(),
15451 payload_codec.to_string(),
15452 None,
15453 )
15454 .expect("valid workflow history"),
15455 )),
15456 }
15457 }
15458
15459 fn history_event(event_type: &str, payload: Value) -> HistoryEvent {
15460 HistoryEvent {
15461 event_type: event_type.to_string(),
15462 payload,
15463 raw: HashMap::new(),
15464 }
15465 }
15466
15467 fn parallel_path_entry(
15468 kind: &str,
15469 base: u64,
15470 size: usize,
15471 index: usize,
15472 ) -> ParallelGroupMetadata {
15473 parallel_group_entry(base, size, index, kind)
15474 }
15475
15476 fn parallel_history_event(
15477 event_type: &str,
15478 sequence: u64,
15479 identity_field: &str,
15480 identity: &str,
15481 path: Vec<ParallelGroupMetadata>,
15482 result: Option<Value>,
15483 ) -> HistoryEvent {
15484 let mut payload = serde_json::Map::from_iter([
15485 ("sequence".to_string(), json!(sequence)),
15486 (identity_field.to_string(), json!(identity)),
15487 ]);
15488 let inner = path.last().expect("parallel history path");
15489 apply_parallel_group_path(&mut payload, std::slice::from_ref(inner));
15490 payload.insert("parallel_group_path".to_string(), json!(path));
15491 if let Some(result) = result {
15492 let field = if event_type == "ChildRunCompleted" {
15493 "result"
15494 } else {
15495 "result"
15496 };
15497 payload.insert(field.to_string(), fixture_envelope(result));
15498 payload.insert("payload_codec".to_string(), json!(DEFAULT_CODEC));
15499 }
15500 history_event(event_type, Value::Object(payload))
15501 }
15502
15503 fn nested_parallel_operations() -> Vec<ParallelOperation> {
15504 vec![
15505 ParallelOperation::activity("first", json!([])),
15506 ParallelOperation::group(vec![
15507 ParallelOperation::child_workflow(
15508 "second",
15509 ChildWorkflowOptions::new("child-workers"),
15510 json!([]),
15511 ),
15512 ParallelOperation::activity("third", json!([])),
15513 ]),
15514 ]
15515 }
15516
15517 fn nested_parallel_paths() -> [Vec<ParallelGroupMetadata>; 3] {
15518 let outer = [
15519 parallel_path_entry("mixed", 1, 3, 0),
15520 parallel_path_entry("mixed", 1, 3, 1),
15521 parallel_path_entry("mixed", 1, 3, 2),
15522 ];
15523 [
15524 vec![outer[0].clone()],
15525 vec![outer[1].clone(), parallel_path_entry("mixed", 2, 2, 0)],
15526 vec![outer[2].clone(), parallel_path_entry("mixed", 2, 2, 1)],
15527 ]
15528 }
15529
15530 #[test]
15531 fn parallel_schedules_every_nested_mixed_leaf_with_stable_metadata() {
15532 let ctx = workflow_context(Vec::new());
15533 let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15534 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15535
15536 assert!(matches!(
15537 call.as_mut().poll(&mut task_context),
15538 Poll::Pending
15539 ));
15540 let commands = ctx.take_commands().expect("parallel commands");
15541 assert_eq!(
15542 commands
15543 .iter()
15544 .map(|command| command["type"].as_str().unwrap_or_default())
15545 .collect::<Vec<_>>(),
15546 [
15547 "schedule_activity",
15548 "start_child_workflow",
15549 "schedule_activity"
15550 ]
15551 );
15552 let paths = nested_parallel_paths();
15553 for (command, path) in commands.iter().zip(paths) {
15554 assert_eq!(command["parallel_group_path"], json!(path));
15555 assert_eq!(
15556 command["parallel_group_id"],
15557 json!(path.last().expect("inner group").parallel_group_id)
15558 );
15559 }
15560 }
15561
15562 fn completed_nested_parallel_history() -> Vec<HistoryEvent> {
15563 let paths = nested_parallel_paths();
15564 let third = parallel_history_event(
15565 "ActivityCompleted",
15566 3,
15567 "activity_type",
15568 "third",
15569 paths[2].clone(),
15570 Some(json!("three")),
15571 );
15572 vec![
15573 parallel_history_event(
15574 "ActivityCompleted",
15575 1,
15576 "activity_type",
15577 "first",
15578 paths[0].clone(),
15579 Some(json!("one")),
15580 ),
15581 parallel_history_event(
15582 "ChildWorkflowScheduled",
15583 2,
15584 "child_workflow_type",
15585 "second",
15586 paths[1].clone(),
15587 None,
15588 ),
15589 parallel_history_event(
15590 "ChildRunCompleted",
15591 2,
15592 "child_workflow_type",
15593 "second",
15594 paths[1].clone(),
15595 Some(json!("two")),
15596 ),
15597 third.clone(),
15598 third,
15599 ]
15600 }
15601
15602 #[test]
15603 fn parallel_replay_rebuilds_input_order_and_tolerates_duplicate_delivery() {
15604 for _restart_or_completed_replay in 0..2 {
15605 let ctx = workflow_context(completed_nested_parallel_history());
15606 let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15607 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15608 let Poll::Ready(Ok(results)) = call.as_mut().poll(&mut task_context) else {
15609 panic!("completed nested parallel history must replay");
15610 };
15611 assert_eq!(
15612 results,
15613 vec![
15614 ParallelResult::Activity(json!("one")),
15615 ParallelResult::Group(vec![
15616 ParallelResult::ChildWorkflow(ChildWorkflowResult {
15617 parent: WorkflowIdentity {
15618 workflow_id: None,
15619 run_id: None,
15620 },
15621 child: WorkflowIdentity {
15622 workflow_id: None,
15623 run_id: None,
15624 },
15625 child_workflow_type: Some("second".to_string()),
15626 result: json!("two"),
15627 }),
15628 ParallelResult::Activity(json!("three")),
15629 ]),
15630 ]
15631 );
15632 assert!(ctx.take_commands().expect("commands").is_empty());
15633 ctx.ensure_history_consumed().expect("history consumed");
15634 }
15635 }
15636
15637 #[test]
15638 fn parallel_failure_keeps_typed_cause_path_and_late_completions() {
15639 let paths = nested_parallel_paths();
15640 let history = vec![
15641 parallel_history_event(
15642 "ActivityCompleted",
15643 1,
15644 "activity_type",
15645 "first",
15646 paths[0].clone(),
15647 Some(json!("one")),
15648 ),
15649 parallel_history_event(
15650 "ChildWorkflowScheduled",
15651 2,
15652 "child_workflow_type",
15653 "second",
15654 paths[1].clone(),
15655 None,
15656 ),
15657 parallel_history_event(
15658 "ChildRunFailed",
15659 2,
15660 "child_workflow_type",
15661 "second",
15662 paths[1].clone(),
15663 None,
15664 ),
15665 parallel_history_event(
15666 "ActivityCompleted",
15667 3,
15668 "activity_type",
15669 "third",
15670 paths[2].clone(),
15671 Some(json!("late")),
15672 ),
15673 ];
15674 let ctx = workflow_context(history);
15675 let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15676 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15677 let outcome = call.as_mut().poll(&mut task_context);
15678 let Poll::Ready(Err(Error::ParallelFailed(failure))) = outcome else {
15679 panic!("one failed child must return a typed partial failure: {outcome:?}");
15680 };
15681 assert_eq!(failure.member_path, [1, 0]);
15682 assert_eq!(failure.group_id, "parallel-calls:1:3");
15683 assert!(matches!(*failure.cause, Error::ChildWorkflowFailed(_)));
15684 assert_eq!(
15685 failure
15686 .completed
15687 .iter()
15688 .map(|completion| completion.member_path.clone())
15689 .collect::<Vec<_>>(),
15690 [vec![0], vec![1, 1]]
15691 );
15692 }
15693
15694 #[test]
15695 fn pending_parallel_history_restarts_without_rescheduling_any_leaf() {
15696 let paths = nested_parallel_paths();
15697 let history = vec![
15698 parallel_history_event(
15699 "ActivityScheduled",
15700 1,
15701 "activity_type",
15702 "first",
15703 paths[0].clone(),
15704 None,
15705 ),
15706 parallel_history_event(
15707 "ChildWorkflowScheduled",
15708 2,
15709 "child_workflow_type",
15710 "second",
15711 paths[1].clone(),
15712 None,
15713 ),
15714 parallel_history_event(
15715 "ActivityScheduled",
15716 3,
15717 "activity_type",
15718 "third",
15719 paths[2].clone(),
15720 None,
15721 ),
15722 ];
15723 for _restart in 0..2 {
15724 let ctx = workflow_context(history.clone());
15725 let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15726 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15727 let outcome = call.as_mut().poll(&mut task_context);
15728 assert!(matches!(outcome, Poll::Pending), "{outcome:?}");
15729 assert!(ctx.take_commands().expect("commands").is_empty());
15730 }
15731 }
15732
15733 fn selection_path(index: usize, key: &str) -> Vec<ParallelGroupMetadata> {
15734 vec![selection_group_entry(
15735 1,
15736 2,
15737 index,
15738 "activity",
15739 &SelectionMemberMetadata {
15740 key: SelectionKey::Name(key.to_string()),
15741 index,
15742 base_sequence: index as u64 + 1,
15743 size: 1,
15744 kind: "activity".to_string(),
15745 },
15746 )]
15747 }
15748
15749 fn selection_activity_event(
15750 event_type: &str,
15751 index: usize,
15752 key: &str,
15753 result: Option<Value>,
15754 ) -> HistoryEvent {
15755 let sequence = index as u64 + 1;
15756 let mut event = parallel_history_event(
15757 event_type,
15758 sequence,
15759 "activity_type",
15760 &format!("{key}-activity"),
15761 selection_path(index, key),
15762 result,
15763 );
15764 event.payload["activity_execution_id"] = json!(format!("activity-{key}"));
15765 event.raw.insert(
15766 "id".to_string(),
15767 json!(if event_type == "ActivityCompleted" {
15768 format!("event-{key}")
15769 } else {
15770 format!("{event_type}-{key}")
15771 }),
15772 );
15773 event
15774 }
15775
15776 fn selection_winner_marker() -> HistoryEvent {
15777 history_event(
15778 "SelectionResolved",
15779 json!({
15780 "selection_group_id": "select-calls:1:2",
15781 "selection_group_base_sequence": 1,
15782 "selection_group_size": 2,
15783 "member_key": "fast",
15784 "member_index": 1,
15785 "member_base_sequence": 2,
15786 "member_size": 1,
15787 "operation_kind": "activity",
15788 "operation_identity": "activity-fast",
15789 "outcome": "completed",
15790 "resolution_event_id": "event-fast",
15791 "resolution_event_type": "ActivityCompleted",
15792 }),
15793 )
15794 }
15795
15796 fn keyed_activity_selection(ctx: &WorkflowContext) -> SelectCall {
15797 ctx.select_keyed(vec![
15798 (
15799 "slow",
15800 ParallelOperation::activity_with_options(
15801 "slow-activity",
15802 ActivityOptions::new().task_queue("default"),
15803 json!([]),
15804 ),
15805 ),
15806 (
15807 "fast",
15808 ParallelOperation::activity_with_options(
15809 "fast-activity",
15810 ActivityOptions::new().task_queue("default"),
15811 json!([]),
15812 ),
15813 ),
15814 ])
15815 }
15816
15817 fn assert_persisted_selection_replay(history: Vec<HistoryEvent>) {
15818 let ctx = workflow_context(history);
15819 let mut call = Box::pin(keyed_activity_selection(&ctx));
15820 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15821 let selected = match call.as_mut().poll(&mut task_context) {
15822 Poll::Ready(Ok(selected)) => selected,
15823 Poll::Ready(Err(error)) => panic!("persisted selection winner must replay: {error:?}"),
15824 Poll::Pending => panic!("persisted selection winner must replay without pending"),
15825 };
15826 assert_eq!(selected.key, SelectionKey::Name("fast".to_string()));
15827 assert_eq!(
15828 selected.value,
15829 Some(ParallelResult::Activity(json!("winner-value")))
15830 );
15831 let slow = selected
15832 .handle(&SelectionKey::Name("slow".to_string()))
15833 .expect("slow handle")
15834 .clone();
15835 let mut await_slow = Box::pin(slow.await_result());
15836 assert!(matches!(
15837 await_slow.as_mut().poll(&mut task_context),
15838 Poll::Ready(Ok(ParallelResult::Activity(value))) if value == json!("loser-value")
15839 ));
15840 assert!(ctx.take_commands().expect("commands").is_empty());
15841 }
15842
15843 const SELECTION_COLD_REPLAY_HISTORY: &str = "DURABLE_WORKFLOW_SELECTION_COLD_REPLAY_HISTORY";
15844
15845 fn canonical_selection_history() -> Vec<HistoryEvent> {
15846 const FIXTURE: &[u8] =
15847 include_bytes!("../tests/fixtures/durable_selection_runtime_history.json");
15848 assert_eq!(
15849 format!("{:x}", Sha256::digest(FIXTURE)),
15850 "51fd8b9c16e978dcef536a5c727b9fdc0ae724d9afc17d9a7837d219f41ee3ba",
15851 );
15852 let fixture: Value = serde_json::from_slice(FIXTURE).expect("canonical selection fixture");
15853
15854 serde_json::from_value(fixture["history"].clone()).expect("canonical selection history")
15855 }
15856
15857 #[test]
15858 fn selection_fresh_process_entrypoint() {
15859 let Ok(path) = std::env::var(SELECTION_COLD_REPLAY_HISTORY) else {
15860 return;
15861 };
15862 let persisted = fs::read(path).expect("persisted selection history");
15863 assert_eq!(
15864 format!("{:x}", Sha256::digest(&persisted)),
15865 "51fd8b9c16e978dcef536a5c727b9fdc0ae724d9afc17d9a7837d219f41ee3ba",
15866 );
15867 let fixture: Value =
15868 serde_json::from_slice(&persisted).expect("valid persisted selection fixture");
15869 let history: Vec<HistoryEvent> = serde_json::from_value(fixture["history"].clone())
15870 .expect("valid persisted selection history");
15871
15872 assert_persisted_selection_replay(history);
15873 }
15874
15875 #[test]
15876 fn selection_starts_every_member_with_stable_keys_and_group_identity() {
15877 let ctx = workflow_context(Vec::new());
15878 let mut call = Box::pin(keyed_activity_selection(&ctx));
15879 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15880
15881 assert!(matches!(
15882 call.as_mut().poll(&mut task_context),
15883 Poll::Pending
15884 ));
15885 let commands = ctx.take_commands().expect("selection commands");
15886 assert_eq!(commands.len(), 2);
15887 assert_eq!(commands[0]["selection_member_key"], json!("slow"));
15888 assert_eq!(commands[1]["selection_member_key"], json!("fast"));
15889 assert!(commands.iter().all(|command| {
15890 command["parallel_group_id"] == json!("select-calls:1:2")
15891 && command["parallel_group_mode"] == json!("select")
15892 }));
15893 }
15894
15895 #[test]
15896 fn selection_key_domain_rejects_empty_authoring_and_malformed_history() {
15897 let ctx = workflow_context(Vec::new());
15898 let mut invalid = Box::pin(ctx.select_keyed(vec![(
15899 "",
15900 ParallelOperation::activity("invalid", json!([])),
15901 )]));
15902 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15903 assert!(matches!(
15904 invalid.as_mut().poll(&mut task_context),
15905 Poll::Ready(Err(Error::InvalidParallelGroup(ParallelGroupError {
15906 reason: "selection_key_invalid",
15907 ..
15908 })))
15909 ));
15910
15911 for invalid_key in [json!(""), json!(-1)] {
15912 let mut event = selection_activity_event("ActivityScheduled", 0, "slow", None);
15913 event.payload["selection_member_key"] = invalid_key.clone();
15914 event.payload["parallel_group_path"][0]["selection_member_key"] = invalid_key;
15915 assert!(matches!(
15916 WorkflowState::new_with_identity(
15917 vec![event],
15918 None,
15919 None,
15920 "rust-workers".to_string(),
15921 DEFAULT_CODEC.to_string(),
15922 None,
15923 ),
15924 Err(Error::NonDeterministicReplay(_))
15925 ));
15926 }
15927 }
15928
15929 #[test]
15930 fn selection_preserves_valid_named_and_numeric_keys() {
15931 let ctx = workflow_context(Vec::new());
15932 let mut selection = Box::pin(ctx.select_keyed(vec![
15933 (
15934 SelectionKey::Index(0),
15935 ParallelOperation::activity("numeric", json!([])),
15936 ),
15937 (
15938 SelectionKey::Name("named".to_string()),
15939 ParallelOperation::timer(Duration::from_secs(1)),
15940 ),
15941 ]));
15942 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15943
15944 assert!(matches!(
15945 selection.as_mut().poll(&mut task_context),
15946 Poll::Pending
15947 ));
15948 let commands = ctx.take_commands().expect("selection commands");
15949 assert_eq!(commands[0]["selection_member_key"], json!(0));
15950 assert_eq!(commands[1]["selection_member_key"], json!("named"));
15951 }
15952
15953 #[test]
15954 fn selection_replays_persisted_winner_and_loser_can_be_awaited_later() {
15955 let history = canonical_selection_history();
15956 assert_persisted_selection_replay(history.clone());
15957
15958 let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
15959 .join("tests/fixtures/durable_selection_runtime_history.json");
15960 let output =
15961 ProcessCommand::new(std::env::current_exe().expect("current Rust test binary"))
15962 .args([
15963 "--exact",
15964 "tests::selection_fresh_process_entrypoint",
15965 "--nocapture",
15966 ])
15967 .env(SELECTION_COLD_REPLAY_HISTORY, &path)
15968 .output()
15969 .expect("run fresh selection replay process");
15970
15971 assert!(
15972 output.status.success(),
15973 "fresh selection replay failed:\nstdout:\n{}\nstderr:\n{}",
15974 String::from_utf8_lossy(&output.stdout),
15975 String::from_utf8_lossy(&output.stderr),
15976 );
15977 }
15978
15979 #[test]
15980 fn selection_waits_durably_when_terminal_members_precede_the_winner_marker() {
15981 let mut history = canonical_selection_history();
15982 history.retain(|event| event.event_type != "SelectionResolved");
15983 let ctx = workflow_context(history);
15984 let mut selection = Box::pin(keyed_activity_selection(&ctx));
15985 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15986
15987 assert!(matches!(
15988 selection.as_mut().poll(&mut task_context),
15989 Poll::Pending
15990 ));
15991 assert!(ctx.take_commands().expect("commands").is_empty());
15992 assert!(
15993 ctx.matched_recorded_pending()
15994 .expect("selection pending state"),
15995 "terminal member history must keep the workflow durably pending until SelectionResolved commits"
15996 );
15997 }
15998
15999 #[test]
16000 fn selection_terminal_condition_history_waits_durably_for_its_winner_marker() {
16001 for (terminal_event, predicate_satisfied, timeout_seconds) in [
16002 ("ConditionWaitSatisfied", true, None),
16003 ("ConditionWaitTimedOut", false, Some(0)),
16004 ] {
16005 let member = SelectionMemberMetadata {
16006 key: SelectionKey::Name("condition".to_string()),
16007 index: 0,
16008 base_sequence: 1,
16009 size: 1,
16010 kind: "condition".to_string(),
16011 };
16012 let path = vec![selection_group_entry(1, 1, 0, "condition", &member)];
16013 let mut payload = json!({
16014 "sequence": 1,
16015 "condition_wait_id": "condition-1",
16016 "condition_wait_occurrence_id": "rust:condition-wait:0",
16017 "condition_key": "ready",
16018 "condition_definition_fingerprint": "sha256:ready-v1",
16019 "parallel_group_path": path,
16020 });
16021 payload
16022 .as_object_mut()
16023 .expect("condition history payload")
16024 .extend(
16025 serde_json::to_value(&path[0])
16026 .expect("condition selection metadata")
16027 .as_object()
16028 .expect("condition selection metadata object")
16029 .clone(),
16030 );
16031 if let Some(timeout_seconds) = timeout_seconds {
16032 payload["timeout_seconds"] = json!(timeout_seconds);
16033 }
16034 let history = vec![
16035 history_event("ConditionWaitOpened", payload.clone()),
16036 history_event(terminal_event, payload),
16037 ];
16038 let ctx = workflow_context(history);
16039 let mut options = ConditionWaitOptions::new("ready", "sha256:ready-v1");
16040 if timeout_seconds.is_some() {
16041 options = options.timeout(Duration::ZERO);
16042 }
16043 let mut selection = Box::pin(ctx.select_keyed(vec![(
16044 "condition",
16045 ParallelOperation::condition(options, move || Ok(predicate_satisfied)),
16046 )]));
16047 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16048
16049 assert!(matches!(
16050 selection.as_mut().poll(&mut task_context),
16051 Poll::Pending
16052 ));
16053 assert!(ctx.take_commands().expect("commands").is_empty());
16054 assert!(
16055 ctx.matched_recorded_pending()
16056 .expect("condition selection pending state"),
16057 "{terminal_event} must keep the workflow durably pending until SelectionResolved commits"
16058 );
16059 }
16060 }
16061
16062 #[test]
16063 fn selection_immediate_condition_members_open_a_durable_wait() {
16064 for predicate_satisfied in [true, false] {
16065 let ctx = workflow_context(Vec::new());
16066 let mut selection = Box::pin(ctx.select_keyed(vec![(
16067 "condition",
16068 ParallelOperation::condition(
16069 ConditionWaitOptions::new("ready", "sha256:ready-v1").timeout(Duration::ZERO),
16070 move || Ok(predicate_satisfied),
16071 ),
16072 )]));
16073 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16074
16075 assert!(matches!(
16076 selection.as_mut().poll(&mut task_context),
16077 Poll::Pending
16078 ));
16079 let commands = ctx.take_commands().expect("condition selection command");
16080 assert_eq!(commands.len(), 1);
16081 assert_eq!(commands[0]["type"], json!("open_condition_wait"));
16082 assert_eq!(commands[0]["timeout_seconds"], json!(0));
16083 assert_eq!(
16084 commands[0]["parallel_group_path"][0]["parallel_group_mode"],
16085 json!("select")
16086 );
16087 }
16088 }
16089
16090 #[test]
16091 fn selection_loser_cancellation_is_explicit_and_idempotent() {
16092 let history = vec![
16093 selection_activity_event("ActivityScheduled", 0, "slow", None),
16094 selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
16095 selection_winner_marker(),
16096 ];
16097 let ctx = workflow_context(history.clone());
16098 let mut call = Box::pin(keyed_activity_selection(&ctx));
16099 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16100 let Poll::Ready(Ok(selected)) = call.as_mut().poll(&mut task_context) else {
16101 panic!("winner must replay");
16102 };
16103 let slow = selected
16104 .handle(&SelectionKey::Name("slow".to_string()))
16105 .expect("slow handle")
16106 .clone();
16107 let mut cancel = Box::pin(slow.cancel());
16108 assert!(matches!(
16109 cancel.as_mut().poll(&mut task_context),
16110 Poll::Pending
16111 ));
16112 assert!(matches!(
16113 cancel.as_mut().poll(&mut task_context),
16114 Poll::Pending
16115 ));
16116 let commands = ctx.take_commands().expect("cancel command");
16117 assert_eq!(commands.len(), 1);
16118 assert_eq!(commands[0]["type"], json!("cancel_selection_operation"));
16119 assert_eq!(commands[0]["member_key"], json!("slow"));
16120
16121 let mut cancelled_history = history;
16122 cancelled_history.push(history_event(
16123 "SelectionOperationCancelled",
16124 json!({
16125 "selection_group_id": "select-calls:1:2",
16126 "member_key": "slow",
16127 "member_index": 0,
16128 "member_base_sequence": 1,
16129 "member_size": 1,
16130 "operation_kind": "activity",
16131 "operation_identity": "activity-slow",
16132 "cancelled_at": "2026-08-27T00:00:00Z",
16133 }),
16134 ));
16135 let replayed = workflow_context(cancelled_history);
16136 let mut call = Box::pin(keyed_activity_selection(&replayed));
16137 let Poll::Ready(Ok(selected)) = call.as_mut().poll(&mut task_context) else {
16138 panic!("winner must replay after cancellation");
16139 };
16140 let slow = selected
16141 .handle(&SelectionKey::Name("slow".to_string()))
16142 .expect("slow handle")
16143 .clone();
16144 let mut cancel = Box::pin(slow.cancel());
16145 assert!(matches!(
16146 cancel.as_mut().poll(&mut task_context),
16147 Poll::Ready(Ok(()))
16148 ));
16149 assert!(replayed.take_commands().expect("commands").is_empty());
16150 }
16151
16152 #[test]
16153 fn selection_cancellation_marker_is_bound_to_every_authored_handle_field() {
16154 let base_history = vec![
16155 selection_activity_event("ActivityScheduled", 0, "slow", None),
16156 selection_activity_event("ActivityScheduled", 1, "fast", None),
16157 selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
16158 selection_winner_marker(),
16159 ];
16160 for (field, corrupt) in [
16161 ("member_key", json!("fast")),
16162 ("member_index", json!(1)),
16163 ("member_base_sequence", json!(3)),
16164 ("member_size", json!(2)),
16165 ("operation_kind", json!("timer")),
16166 ("operation_identity", json!("forged")),
16167 ] {
16168 let mut cancellation = json!({
16169 "selection_group_id": "select-calls:1:2",
16170 "member_key": "slow",
16171 "member_index": 0,
16172 "member_base_sequence": 1,
16173 "member_size": 1,
16174 "operation_kind": "activity",
16175 "operation_identity": "activity-slow",
16176 });
16177 cancellation[field] = corrupt;
16178 let mut history = base_history.clone();
16179 history.push(history_event("SelectionOperationCancelled", cancellation));
16180 let ctx = workflow_context(history);
16181 let mut selection = Box::pin(keyed_activity_selection(&ctx));
16182 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16183
16184 assert!(matches!(
16185 selection.as_mut().poll(&mut task_context),
16186 Poll::Ready(Err(Error::NonDeterministicReplay(_)))
16187 ));
16188 }
16189 }
16190
16191 #[test]
16192 fn selection_child_identity_prefers_the_durable_run_id() {
16193 let ctx = workflow_context(vec![history_event(
16194 "ChildWorkflowScheduled",
16195 json!({
16196 "sequence": 1,
16197 "child_workflow_type": "child",
16198 "child_workflow_instance_id": "child-instance",
16199 "child_workflow_run_id": "child-run",
16200 }),
16201 )]);
16202 let state = ctx.state.lock().expect("workflow state");
16203
16204 assert_eq!(
16205 selection_operation_identity(&state, "child", 1, 1),
16206 "child-run"
16207 );
16208 }
16209
16210 #[test]
16211 fn selection_activity_identity_requires_canonical_execution_id() {
16212 let slow = selection_activity_event("ActivityScheduled", 0, "slow", None);
16213 let mut fast_open = selection_activity_event("ActivityScheduled", 1, "fast", None);
16214 let mut fast_completed =
16215 selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner")));
16216 for event in [&mut fast_open, &mut fast_completed] {
16217 event
16218 .payload
16219 .as_object_mut()
16220 .expect("activity payload")
16221 .remove("activity_execution_id");
16222 event.payload["activity_id"] = json!("forged-activity-id");
16223 }
16224 let mut marker = selection_winner_marker();
16225 marker.payload["operation_identity"] = json!("forged-activity-id");
16226 let ctx = workflow_context(vec![slow, fast_open, fast_completed, marker]);
16227 let mut selection = Box::pin(keyed_activity_selection(&ctx));
16228 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16229
16230 assert!(matches!(
16231 selection.as_mut().poll(&mut task_context),
16232 Poll::Ready(Err(Error::NonDeterministicReplay(_)))
16233 ));
16234 }
16235
16236 #[test]
16237 fn selection_completion_before_cancellation_remains_awaitable() {
16238 let history = vec![
16239 selection_activity_event("ActivityScheduled", 0, "slow", None),
16240 selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
16241 selection_winner_marker(),
16242 selection_activity_event(
16243 "ActivityCompleted",
16244 0,
16245 "slow",
16246 Some(json!("completed-first")),
16247 ),
16248 ];
16249 let ctx = workflow_context(history);
16250 let mut selection = Box::pin(keyed_activity_selection(&ctx));
16251 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16252 let Poll::Ready(Ok(selected)) = selection.as_mut().poll(&mut task_context) else {
16253 panic!("winner must replay");
16254 };
16255 let slow = selected
16256 .handle(&SelectionKey::Name("slow".to_string()))
16257 .expect("slow handle")
16258 .clone();
16259 let mut cancel = Box::pin(slow.cancel());
16260 assert!(matches!(
16261 cancel.as_mut().poll(&mut task_context),
16262 Poll::Ready(Ok(()))
16263 ));
16264 let mut await_slow = Box::pin(slow.await_result());
16265 assert!(matches!(
16266 await_slow.as_mut().poll(&mut task_context),
16267 Poll::Ready(Ok(ParallelResult::Activity(value))) if value == json!("completed-first")
16268 ));
16269 let commands = ctx.take_commands().expect("commands");
16270 assert!(commands.is_empty());
16271 }
16272
16273 #[test]
16274 fn selection_nested_later_failure_before_cancel_remains_the_awaited_failure() {
16275 let nested_member = SelectionMemberMetadata {
16276 key: SelectionKey::Name("nested".to_string()),
16277 index: 0,
16278 base_sequence: 1,
16279 size: 2,
16280 kind: "group".to_string(),
16281 };
16282 let deadline_member = SelectionMemberMetadata {
16283 key: SelectionKey::Name("deadline".to_string()),
16284 index: 1,
16285 base_sequence: 3,
16286 size: 1,
16287 kind: "timer".to_string(),
16288 };
16289 let nested_paths = [
16290 vec![
16291 selection_group_entry(1, 3, 0, "mixed", &nested_member),
16292 parallel_group_entry(1, 2, 0, "activity"),
16293 ],
16294 vec![
16295 selection_group_entry(1, 3, 1, "mixed", &nested_member),
16296 parallel_group_entry(1, 2, 1, "activity"),
16297 ],
16298 ];
16299 let deadline_path = vec![selection_group_entry(1, 3, 2, "mixed", &deadline_member)];
16300 let mut timer_fired = parallel_history_event(
16301 "TimerFired",
16302 3,
16303 "timer_id",
16304 "timer-3",
16305 deadline_path.clone(),
16306 None,
16307 );
16308 timer_fired.payload["delay_seconds"] = json!(0);
16309 timer_fired
16310 .raw
16311 .insert("id".to_string(), json!("timer-fired"));
16312 let mut timer_scheduled = parallel_history_event(
16313 "TimerScheduled",
16314 3,
16315 "timer_id",
16316 "timer-3",
16317 deadline_path,
16318 None,
16319 );
16320 timer_scheduled.payload["delay_seconds"] = json!(0);
16321 let history = vec![
16322 parallel_history_event(
16323 "ActivityScheduled",
16324 1,
16325 "activity_type",
16326 "nested-first",
16327 nested_paths[0].clone(),
16328 None,
16329 ),
16330 parallel_history_event(
16331 "ActivityScheduled",
16332 2,
16333 "activity_type",
16334 "nested-second",
16335 nested_paths[1].clone(),
16336 None,
16337 ),
16338 timer_scheduled,
16339 timer_fired,
16340 history_event(
16341 "SelectionResolved",
16342 json!({
16343 "selection_group_id": "select-calls:1:3",
16344 "selection_group_base_sequence": 1,
16345 "selection_group_size": 3,
16346 "member_key": "deadline",
16347 "member_index": 1,
16348 "member_base_sequence": 3,
16349 "member_size": 1,
16350 "operation_kind": "timer",
16351 "operation_identity": "timer-3",
16352 "outcome": "completed",
16353 "resolution_event_id": "timer-fired",
16354 "resolution_event_type": "TimerFired",
16355 }),
16356 ),
16357 parallel_history_event(
16358 "ActivityFailed",
16359 2,
16360 "activity_type",
16361 "nested-second",
16362 nested_paths[1].clone(),
16363 None,
16364 ),
16365 ];
16366 let ctx = workflow_context(history);
16367 let mut selection = Box::pin(ctx.select_keyed(vec![
16368 (
16369 "nested",
16370 ParallelOperation::group(vec![
16371 ParallelOperation::activity("nested-first", json!([])),
16372 ParallelOperation::activity("nested-second", json!([])),
16373 ]),
16374 ),
16375 ("deadline", ParallelOperation::timer(Duration::ZERO)),
16376 ]));
16377 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16378 let Poll::Ready(Ok(selected)) = selection.as_mut().poll(&mut task_context) else {
16379 panic!("deadline winner must replay");
16380 };
16381 let nested = selected
16382 .handle(&SelectionKey::Name("nested".to_string()))
16383 .expect("nested handle")
16384 .clone();
16385 let mut cancel = Box::pin(nested.cancel());
16386 assert!(matches!(
16387 cancel.as_mut().poll(&mut task_context),
16388 Poll::Ready(Ok(()))
16389 ));
16390 let mut await_nested = Box::pin(nested.await_result());
16391
16392 assert!(matches!(
16393 await_nested.as_mut().poll(&mut task_context),
16394 Poll::Ready(Err(Error::ActivityFailed(_)))
16395 ));
16396 assert!(ctx.take_commands().expect("commands").is_empty());
16397 }
16398
16399 #[test]
16400 fn selection_supports_child_timer_signal_condition_and_nested_groups() {
16401 let ctx = workflow_context(Vec::new());
16402 let mut call = Box::pin(ctx.select(vec![
16403 ParallelOperation::child_workflow(
16404 "child",
16405 ChildWorkflowOptions::new("children"),
16406 json!([]),
16407 ),
16408 ParallelOperation::timer(Duration::from_secs(30)),
16409 ParallelOperation::signal("approval"),
16410 ParallelOperation::condition(
16411 ConditionWaitOptions::new("ready", "sha256:ready"),
16412 || Ok(false),
16413 ),
16414 ParallelOperation::group(vec![
16415 ParallelOperation::activity("nested-one", json!([])),
16416 ParallelOperation::activity("nested-two", json!([])),
16417 ]),
16418 ]));
16419 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16420 assert!(matches!(
16421 call.as_mut().poll(&mut task_context),
16422 Poll::Pending
16423 ));
16424 let commands = ctx.take_commands().expect("selection commands");
16425 assert_eq!(
16426 commands
16427 .iter()
16428 .map(|command| command["type"].as_str().unwrap_or_default())
16429 .collect::<Vec<_>>(),
16430 [
16431 "start_child_workflow",
16432 "start_timer",
16433 "open_signal_wait",
16434 "open_condition_wait",
16435 "schedule_activity",
16436 "schedule_activity",
16437 ]
16438 );
16439 assert!(commands.iter().all(|command| {
16440 command["parallel_group_path"][0]["parallel_group_mode"] == json!("select")
16441 }));
16442 assert_eq!(
16443 commands[4]["parallel_group_path"].as_array().map(Vec::len),
16444 Some(2)
16445 );
16446 assert_eq!(
16447 commands[4]["parallel_group_path"][0]["selection_member_kind"],
16448 json!("group")
16449 );
16450 assert_eq!(
16451 commands[5]["parallel_group_path"][0]["selection_member_kind"],
16452 json!("group")
16453 );
16454
16455 let one_leaf_ctx = workflow_context(Vec::new());
16456 let mut one_leaf = Box::pin(one_leaf_ctx.select(vec![ParallelOperation::group(vec![
16457 ParallelOperation::activity("nested-only", json!([])),
16458 ])]));
16459 assert!(matches!(
16460 one_leaf.as_mut().poll(&mut task_context),
16461 Poll::Pending
16462 ));
16463 let one_leaf_commands = one_leaf_ctx.take_commands().expect("one-leaf commands");
16464 assert_eq!(one_leaf_commands.len(), 1);
16465 assert_eq!(
16466 one_leaf_commands[0]["parallel_group_path"][0]["selection_member_kind"],
16467 json!("group")
16468 );
16469 assert_eq!(
16470 one_leaf_commands[0]["parallel_group_path"][0]["selection_member_size"],
16471 json!(1)
16472 );
16473 }
16474
16475 async fn trip_saga(ctx: WorkflowContext) -> Result<Value> {
16476 let mut saga = ctx.saga();
16477 let outcome = async {
16478 let flight = ctx.activity("trip.reserve-flight", json!([])).await?;
16479 saga.add_compensation("trip.cancel-flight", json!([flight]))?;
16480 let hotel = ctx.activity("trip.reserve-hotel", json!([])).await?;
16481 saga.add_compensation("trip.cancel-hotel", json!([hotel]))?;
16482 ctx.activity("trip.charge", json!([])).await?;
16483 Ok(json!({"status": "booked"}))
16484 }
16485 .await;
16486 saga.finish(outcome).await
16487 }
16488
16489 fn saga_activity(
16490 event_type: &str,
16491 sequence: u64,
16492 activity_type: &str,
16493 result: Option<Value>,
16494 ) -> HistoryEvent {
16495 let mut payload = json!({
16496 "sequence": sequence,
16497 "activity_type": activity_type,
16498 "message": format!("{activity_type} failed"),
16499 "exception_type": "PlannedFailure",
16500 "non_retryable": true,
16501 });
16502 if let Some(result) = result {
16503 payload["result"] = fixture_envelope(result);
16504 }
16505 history_event(event_type, payload)
16506 }
16507
16508 #[test]
16509 fn saga_replays_reverse_compensation_across_restart_and_duplicate_delivery() {
16510 let completed_hotel_compensation = saga_activity(
16511 "ActivityCompleted",
16512 4,
16513 "trip.cancel-hotel",
16514 Some(Value::Null),
16515 );
16516 let history = vec![
16517 saga_activity(
16518 "ActivityCompleted",
16519 1,
16520 "trip.reserve-flight",
16521 Some(json!("flight-1")),
16522 ),
16523 saga_activity(
16524 "ActivityCompleted",
16525 2,
16526 "trip.reserve-hotel",
16527 Some(json!("hotel-1")),
16528 ),
16529 saga_activity("ActivityFailed", 3, "trip.charge", None),
16530 completed_hotel_compensation.clone(),
16531 completed_hotel_compensation,
16532 ];
16533
16534 for _restart in 0..2 {
16535 let ctx = workflow_context(history.clone());
16536 let mut future = Box::pin(trip_saga(ctx.clone()));
16537 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16538 assert!(matches!(
16539 future.as_mut().poll(&mut task_context),
16540 Poll::Pending
16541 ));
16542 let commands = ctx.take_commands().expect("compensation command");
16543 assert_eq!(commands.len(), 1);
16544 assert_eq!(commands[0]["activity_type"], "trip.cancel-flight");
16545 }
16546 }
16547
16548 #[test]
16549 fn saga_compensation_failure_preserves_both_typed_failures() {
16550 let history = vec![
16551 saga_activity(
16552 "ActivityCompleted",
16553 1,
16554 "trip.reserve-flight",
16555 Some(json!("flight-1")),
16556 ),
16557 saga_activity(
16558 "ActivityCompleted",
16559 2,
16560 "trip.reserve-hotel",
16561 Some(json!("hotel-1")),
16562 ),
16563 saga_activity("ActivityFailed", 3, "trip.charge", None),
16564 saga_activity("ActivityFailed", 4, "trip.cancel-hotel", None),
16565 ];
16566 let ctx = workflow_context(history);
16567 let mut future = Box::pin(trip_saga(ctx));
16568 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16569 let Poll::Ready(Err(Error::SagaCompensationFailed(failure))) =
16570 future.as_mut().poll(&mut task_context)
16571 else {
16572 panic!("compensation failure must remain structured");
16573 };
16574 assert!(matches!(
16575 *failure.initiating_failure,
16576 Error::ActivityFailed(_)
16577 ));
16578 assert!(matches!(
16579 *failure.compensation_failure,
16580 Error::ActivityFailed(_)
16581 ));
16582 assert_eq!(failure.compensation_activity_type, "trip.cancel-hotel");
16583 assert_eq!(failure.compensation_registration_order, 2);
16584 }
16585
16586 #[test]
16587 fn saga_compensates_cooperative_cancellation() {
16588 let ctx = workflow_context(vec![saga_activity(
16589 "ActivityCompleted",
16590 1,
16591 "trip.reserve-flight",
16592 Some(json!("flight-1")),
16593 )]);
16594 ctx.state.lock().expect("state").cancel_requested = true;
16595 let run = {
16596 let ctx = ctx.clone();
16597 async move {
16598 let mut saga = ctx.saga();
16599 let outcome = async {
16600 let flight = ctx.activity("trip.reserve-flight", json!([])).await?;
16601 saga.add_compensation("trip.cancel-flight", json!([flight]))?;
16602 ctx.throw_if_cancellation_requested()?;
16603 Ok(json!("unexpected"))
16604 }
16605 .await;
16606 saga.finish(outcome).await
16607 }
16608 };
16609 let mut future = Box::pin(run);
16610 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16611 assert!(matches!(
16612 future.as_mut().poll(&mut task_context),
16613 Poll::Pending
16614 ));
16615 let commands = ctx.take_commands().expect("cancellation compensation");
16616 assert_eq!(commands[0]["activity_type"], "trip.cancel-flight");
16617 }
16618
16619 fn workflow_task(
16620 workflow_type: &str,
16621 history_events: Vec<HistoryEvent>,
16622 payload_codec: &str,
16623 ) -> WorkflowTask {
16624 WorkflowTask {
16625 task_id: format!("wft-{workflow_type}"),
16626 workflow_command_id: None,
16627 workflow_id: Some(format!("wf-{workflow_type}")),
16628 run_id: Some(format!("run-{workflow_type}")),
16629 workflow_type: workflow_type.to_string(),
16630 cancel_requested: false,
16631 payload_codec: payload_codec.to_string(),
16632 arguments: Some(
16633 encode_value_envelope(&json!([]), payload_codec).expect("workflow arguments"),
16634 ),
16635 total_history_events: Some(history_events.len() as u64),
16636 history_size_bytes: None,
16637 continue_as_new_recommended: None,
16638 history_budget_pressure: None,
16639 history_events,
16640 next_history_page_token: None,
16641 workflow_task_attempt: 1,
16642 workflow_signal_id: None,
16643 signal_name: None,
16644 signal_arguments: None,
16645 workflow_update_id: None,
16646 update_name: None,
16647 lease_owner: Some("rust-worker".to_string()),
16648 }
16649 }
16650
16651 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
16652 struct SideEffectProbe {
16653 request_id: String,
16654 attempt: u32,
16655 }
16656
16657 #[test]
16658 fn typed_side_effect_runs_callback_once_and_replay_skips_it() {
16659 let calls = AtomicUsize::new(0);
16660 let ctx = workflow_context(Vec::new());
16661 let value = ctx
16662 .side_effect(|| {
16663 calls.fetch_add(1, Ordering::SeqCst);
16664 SideEffectProbe {
16665 request_id: "request-42".to_string(),
16666 attempt: 3,
16667 }
16668 })
16669 .expect("first side effect");
16670 assert_eq!(value.attempt, 3);
16671 assert_eq!(calls.load(Ordering::SeqCst), 1);
16672 let commands = ctx.take_commands().expect("commands");
16673 assert_eq!(commands.len(), 1);
16674 assert_eq!(commands[0]["type"], "record_side_effect");
16675 assert_eq!(
16676 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
16677 serde_json::to_value(&value).expect("value")
16678 );
16679
16680 let replay = workflow_context(vec![history_event(
16681 "SideEffectRecorded",
16682 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16683 )]);
16684 let replayed: SideEffectProbe = replay
16685 .side_effect(|| {
16686 calls.fetch_add(1, Ordering::SeqCst);
16687 panic!("committed side-effect callbacks must not run during replay")
16688 })
16689 .expect("replayed side effect");
16690 assert_eq!(replayed, value);
16691 assert_eq!(calls.load(Ordering::SeqCst), 1);
16692 assert!(replay.take_commands().expect("commands").is_empty());
16693 replay.ensure_history_consumed().expect("history consumed");
16694 }
16695
16696 #[test]
16697 fn side_effect_uses_avro_envelope_and_uuid_is_replay_stable() {
16698 let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
16699 let value = ctx
16700 .side_effect(|| SideEffectProbe {
16701 request_id: "avro-request".to_string(),
16702 attempt: 1,
16703 })
16704 .expect("Avro side effect");
16705 let uuid = ctx.uuid_v4().expect("deterministic UUID");
16706 let commands = ctx.take_commands().expect("commands");
16707 assert_eq!(commands.len(), 2);
16708 assert_eq!(commands[0]["result"]["codec"], DEFAULT_CODEC);
16709 assert_eq!(commands[1]["result"]["codec"], DEFAULT_CODEC);
16710 assert_eq!(
16711 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
16712 serde_json::to_value(&value).expect("value")
16713 );
16714
16715 let replay = workflow_context_with_codec(
16716 vec![
16717 history_event(
16718 "SideEffectRecorded",
16719 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16720 ),
16721 history_event(
16722 "SideEffectRecorded",
16723 json!({"sequence": 2, "result": commands[1]["result"].clone()}),
16724 ),
16725 ],
16726 DEFAULT_CODEC,
16727 );
16728 let replayed: SideEffectProbe = replay
16729 .side_effect(|| panic!("Avro callback must not run"))
16730 .expect("replayed Avro value");
16731 let replayed_uuid = replay.uuid_v4().expect("replayed UUID");
16732 assert_eq!(replayed, value);
16733 assert_eq!(replayed_uuid, uuid);
16734 assert!(replay.take_commands().expect("commands").is_empty());
16735 }
16736
16737 #[test]
16738 fn typed_side_effect_replay_preserves_bytes_and_maps() {
16739 let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
16740 let value = ctx
16741 .side_effect_avro_value(typed_fidelity_probe)
16742 .expect("typed side effect");
16743 let commands = ctx.take_commands().expect("side-effect command");
16744 assert_eq!(
16745 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
16746 .expect("recorded side effect"),
16747 value
16748 );
16749
16750 let replay = workflow_context_with_codec(
16751 vec![history_event(
16752 "SideEffectRecorded",
16753 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16754 )],
16755 DEFAULT_CODEC,
16756 );
16757 assert_eq!(
16758 replay
16759 .side_effect_avro_value(|| panic!("replay must not invoke callback"))
16760 .expect("replayed typed side effect"),
16761 value
16762 );
16763 }
16764
16765 #[test]
16766 fn ordered_side_effects_share_the_durable_command_stream() {
16767 let first = encode_value_envelope(&json!("first"), DEFAULT_CODEC).expect("first");
16768 let second = encode_value_envelope(&json!(29), DEFAULT_CODEC).expect("second");
16769 let ctx = workflow_context(vec![
16770 history_event(
16771 "SideEffectRecorded",
16772 json!({"sequence": 1, "result": first}),
16773 ),
16774 history_event(
16775 "SideEffectRecorded",
16776 json!({"sequence": 2, "result": second}),
16777 ),
16778 ]);
16779 let first: String = ctx
16780 .side_effect(|| panic!("first callback must not run"))
16781 .expect("first replay");
16782 let second: i32 = ctx
16783 .side_effect(|| panic!("second callback must not run"))
16784 .expect("second replay");
16785 assert_eq!(first, "first");
16786 assert_eq!(second, 29);
16787 ctx.ensure_history_consumed().expect("ordered history");
16788
16789 let reordered = workflow_context(vec![history_event(
16790 "VersionMarkerRecorded",
16791 json!({
16792 "sequence": 1,
16793 "change_id": "before-side-effect",
16794 "version": 1,
16795 "min_supported": 1,
16796 "max_supported": 1,
16797 }),
16798 )]);
16799 let error = reordered
16800 .side_effect(|| "new".to_string())
16801 .expect_err("command reordering must fail");
16802 assert!(matches!(
16803 error,
16804 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16805 if reason == "recorded_command_mismatch"
16806 ));
16807 }
16808
16809 #[test]
16810 fn version_markers_replay_across_upgrades_and_do_not_duplicate() {
16811 let ctx = workflow_context(Vec::new());
16812 assert_eq!(ctx.get_version("checkout-v2", 1, 2).expect("version"), 2);
16813 assert_eq!(ctx.get_version("checkout-v2", 1, 3).expect("cached"), 2);
16814 assert!(ctx.patched("new-search").expect("patch"));
16815 ctx.deprecate_patch("new-search").expect("deprecate patch");
16816 let commands = ctx.take_commands().expect("commands");
16817 assert_eq!(commands.len(), 2);
16818 assert_eq!(commands[0]["type"], "record_version_marker");
16819 assert_eq!(commands[0]["version"], 2);
16820 assert_eq!(commands[1]["change_id"], "new-search");
16821
16822 let replay = workflow_context(vec![history_event(
16823 "VersionMarkerRecorded",
16824 json!({
16825 "sequence": 1,
16826 "change_id": "checkout-v2",
16827 "version": 2,
16828 "min_supported": 1,
16829 "max_supported": 2,
16830 }),
16831 )]);
16832 assert_eq!(replay.get_version("checkout-v2", 1, 4).expect("upgrade"), 2);
16833 assert_eq!(replay.get_version("checkout-v2", 2, 5).expect("repeat"), 2);
16834 assert!(replay.take_commands().expect("commands").is_empty());
16835 replay.ensure_history_consumed().expect("history consumed");
16836 }
16837
16838 #[test]
16839 fn version_markers_reject_incompatible_or_malformed_history() {
16840 let incompatible = workflow_context(vec![history_event(
16841 "VersionMarkerRecorded",
16842 json!({
16843 "sequence": 1,
16844 "change_id": "checkout-v2",
16845 "version": 1,
16846 "min_supported": 1,
16847 "max_supported": 2,
16848 }),
16849 )]);
16850 let error = incompatible
16851 .get_version("checkout-v2", 2, 3)
16852 .expect_err("old version is unsupported");
16853 assert!(matches!(
16854 error,
16855 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16856 if reason == "version_marker_incompatible_range"
16857 ));
16858
16859 for (history, reason) in [
16860 (
16861 vec![history_event("SideEffectRecorded", json!({"sequence": 1}))],
16862 "side_effect_result_missing",
16863 ),
16864 (
16865 vec![history_event(
16866 "SideEffectRecorded",
16867 json!({
16868 "sequence": 1,
16869 "result": {"codec": "avro", "blob": "not-base64"},
16870 }),
16871 )],
16872 "side_effect_payload_incompatible",
16873 ),
16874 (
16875 vec![history_event(
16876 "SideEffectRecorded",
16877 json!({"sequence": 1, "result": {"unwrapped": true}}),
16878 )],
16879 "side_effect_payload_malformed",
16880 ),
16881 (
16882 vec![history_event(
16883 "VersionMarkerRecorded",
16884 json!({
16885 "sequence": 1,
16886 "change_id": "change",
16887 "version": 1,
16888 "min_supported": 2,
16889 "max_supported": 1,
16890 }),
16891 )],
16892 "version_marker_history_range_invalid",
16893 ),
16894 ] {
16895 let error = WorkflowState::new(
16896 history,
16897 "rust-workers".to_string(),
16898 DEFAULT_CODEC.to_string(),
16899 None,
16900 )
16901 .expect_err("malformed history must fail");
16902 assert!(matches!(
16903 error,
16904 Error::NonDeterministicReplay(ReplayFailure { reason: actual, .. })
16905 if actual == reason
16906 ));
16907 }
16908 }
16909
16910 #[test]
16911 fn typed_search_attributes_replay_value_and_type_identity_after_restart() {
16912 let history = vec![history_event(
16913 "SearchAttributesUpserted",
16914 json!({
16915 "sequence": 1,
16916 "attributes": {"customer_tier": "gold"},
16917 "attribute_types": {"customer_tier": "keyword"},
16918 "merged": {"customer_tier": "gold"}
16919 }),
16920 )];
16921
16922 let matching = workflow_context(history.clone());
16923 matching
16924 .upsert_search_attributes(
16925 SearchAttributeUpdate::new()
16926 .keyword("customer_tier", "gold")
16927 .expect("keyword update"),
16928 )
16929 .expect("matching typed update must replay");
16930 matching
16931 .ensure_history_consumed()
16932 .expect("history consumed");
16933
16934 let changed_type = workflow_context(history.clone());
16935 let error = changed_type
16936 .upsert_search_attributes(
16937 SearchAttributeUpdate::new()
16938 .string("customer_tier", "gold")
16939 .expect("string update"),
16940 )
16941 .expect_err("same JSON value with a different declaration must be nondeterministic");
16942 let Error::NonDeterministicReplay(failure) = error else {
16943 panic!("typed identity drift must be a replay failure");
16944 };
16945 assert_eq!(failure.reason, "search_attribute_type_mismatch");
16946 assert_eq!(failure.sequence, Some(1));
16947
16948 let changed_value = workflow_context(history);
16949 let error = changed_value
16950 .upsert_search_attributes(
16951 SearchAttributeUpdate::new()
16952 .keyword("customer_tier", "platinum")
16953 .expect("keyword update"),
16954 )
16955 .expect_err("changed values must be nondeterministic");
16956 let Error::NonDeterministicReplay(failure) = error else {
16957 panic!("value drift must be a replay failure");
16958 };
16959 assert_eq!(failure.reason, "search_attribute_value_mismatch");
16960 }
16961
16962 #[test]
16963 fn legacy_search_attribute_history_keeps_type_identity_unknown() {
16964 let history = vec![history_event(
16965 "SearchAttributesUpserted",
16966 json!({
16967 "sequence": 1,
16968 "attributes": {"customer_tier": "gold"},
16969 "merged": {"customer_tier": "gold"}
16970 }),
16971 )];
16972
16973 for update in [
16974 SearchAttributeUpdate::new()
16975 .keyword("customer_tier", "gold")
16976 .expect("keyword update"),
16977 SearchAttributeUpdate::new()
16978 .string("customer_tier", "gold")
16979 .expect("string update"),
16980 ] {
16981 let restarted = workflow_context(history.clone());
16982 restarted
16983 .upsert_search_attributes(update)
16984 .expect("legacy history constrains values but has unknown type identity");
16985 restarted
16986 .ensure_history_consumed()
16987 .expect("history consumed");
16988 }
16989 }
16990
16991 #[test]
16992 fn search_attribute_command_emits_canonical_types() {
16993 let ctx = workflow_context(Vec::new());
16994 ctx.upsert_search_attributes(
16995 SearchAttributeUpdate::new()
16996 .keyword("customer_tier", "gold")
16997 .expect("keyword update")
16998 .int("attempts", 3)
16999 .expect("int update")
17000 .delete("obsolete")
17001 .expect("delete update"),
17002 )
17003 .expect("valid search attributes");
17004
17005 assert_eq!(
17006 ctx.take_commands().expect("commands"),
17007 vec![json!({
17008 "type": "upsert_search_attributes",
17009 "attributes": {
17010 "attempts": 3,
17011 "customer_tier": "gold",
17012 "obsolete": null
17013 },
17014 "attribute_types": {
17015 "attempts": "int",
17016 "customer_tier": "keyword"
17017 }
17018 })]
17019 );
17020 }
17021
17022 #[test]
17023 fn duplicate_side_effects_and_version_markers_are_rejected() {
17024 let duplicate_side_effect = WorkflowState::new(
17025 vec![
17026 history_event(
17027 "SideEffectRecorded",
17028 json!({"sequence": 1, "result": fixture_envelope(json!(1))}),
17029 ),
17030 history_event(
17031 "SideEffectRecorded",
17032 json!({"sequence": 1, "result": fixture_envelope(json!(2))}),
17033 ),
17034 ],
17035 "rust-workers".to_string(),
17036 DEFAULT_CODEC.to_string(),
17037 None,
17038 )
17039 .expect_err("duplicate side effect");
17040 assert!(matches!(
17041 duplicate_side_effect,
17042 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
17043 if reason == "duplicate_side_effect_record"
17044 ));
17045
17046 let marker = |sequence| {
17047 history_event(
17048 "VersionMarkerRecorded",
17049 json!({
17050 "sequence": sequence,
17051 "change_id": "same-change",
17052 "version": 1,
17053 "min_supported": 1,
17054 "max_supported": 1,
17055 }),
17056 )
17057 };
17058 let duplicate_marker = WorkflowState::new(
17059 vec![marker(1), marker(3)],
17060 "rust-workers".to_string(),
17061 DEFAULT_CODEC.to_string(),
17062 None,
17063 )
17064 .expect_err("duplicate marker");
17065 assert!(matches!(
17066 duplicate_marker,
17067 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
17068 if reason == "duplicate_version_marker"
17069 ));
17070 }
17071
17072 #[test]
17073 fn workflow_stream_authoring_derives_identity_and_replay_skips_duplicate_append() {
17074 let mut state = WorkflowState::new(
17075 Vec::new(),
17076 "rust-workers".to_string(),
17077 DEFAULT_CODEC.to_string(),
17078 None,
17079 )
17080 .expect("workflow state");
17081 state.workflow_command_identity = "command-7".to_string();
17082 let context = WorkflowContext {
17083 state: Arc::new(Mutex::new(state)),
17084 };
17085 let item =
17086 WorkflowStreamAppendItem::from_reference("s3://bucket/item.avro").item_type("receipt");
17087
17088 context
17089 .append_workflow_stream("output", &[item], Some(10))
17090 .expect("append command");
17091 context
17092 .error_workflow_stream("output", "producer failed", None)
17093 .expect("error command");
17094 let commands = context.take_commands().expect("commands");
17095
17096 assert_eq!(commands[0]["type"], "record_side_effect");
17097 assert_eq!(
17098 commands[0]["workflow_stream"]["command_identity"],
17099 "command-7"
17100 );
17101 assert_eq!(commands[0]["workflow_stream"]["command_ordinal"], 0);
17102 assert_eq!(
17103 commands[0]["workflow_stream"]["items"][0]["idempotency_key"],
17104 "dw-stream:command-7:0:0"
17105 );
17106 assert_eq!(commands[1]["workflow_stream"]["operation"], "error");
17107
17108 let recorded = history_event(
17109 "SideEffectRecorded",
17110 json!({"sequence": 1, "result": fixture_envelope(Value::Null)}),
17111 );
17112 let mut replay_state = WorkflowState::new(
17113 vec![recorded],
17114 "rust-workers".to_string(),
17115 DEFAULT_CODEC.to_string(),
17116 None,
17117 )
17118 .expect("replay state");
17119 replay_state.workflow_command_identity = "command-7".to_string();
17120 let replay_context = WorkflowContext {
17121 state: Arc::new(Mutex::new(replay_state)),
17122 };
17123 replay_context
17124 .append_workflow_stream(
17125 "output",
17126 &[WorkflowStreamAppendItem::from_reference(
17127 "s3://bucket/item.avro",
17128 )],
17129 Some(10),
17130 )
17131 .expect("replayed append");
17132 assert!(replay_context
17133 .take_commands()
17134 .expect("replayed commands")
17135 .is_empty());
17136 }
17137
17138 #[test]
17139 fn workflow_stream_authoring_requires_server_durable_command_identity() {
17140 let context = workflow_context(Vec::new());
17141 let error = context
17142 .append_workflow_stream(
17143 "output",
17144 &[WorkflowStreamAppendItem::from_reference(
17145 "s3://bucket/item.avro",
17146 )],
17147 None,
17148 )
17149 .expect_err("stream append without durable command identity must fail closed");
17150
17151 assert!(matches!(error, Error::MissingWorkflowCommandIdentity));
17152 assert!(context.take_commands().expect("commands").is_empty());
17153 }
17154
17155 #[test]
17156 fn cold_worker_replay_does_not_repeat_committed_side_effects_or_markers() {
17157 fn worker(calls: Arc<AtomicUsize>) -> Worker {
17158 let client = Client::new("http://127.0.0.1:8080").expect("client");
17159 let mut worker = Worker::new(client, "rust-workers");
17160 worker.register_workflow("rust.side-effect-version", move |ctx, _input| {
17161 let calls = Arc::clone(&calls);
17162 async move {
17163 let captured = ctx.side_effect(|| {
17164 calls.fetch_add(1, Ordering::SeqCst);
17165 "captured-once".to_string()
17166 })?;
17167 let version = ctx.get_version("cold-restart", 1, 2)?;
17168 Ok(json!({"captured": captured, "version": version}))
17169 }
17170 });
17171 worker
17172 }
17173
17174 fn task(history_events: Vec<HistoryEvent>) -> WorkflowTask {
17175 WorkflowTask {
17176 task_id: "wft-side-effect-version".to_string(),
17177 workflow_command_id: None,
17178 workflow_id: Some("wf-side-effect-version".to_string()),
17179 run_id: Some("run-side-effect-version".to_string()),
17180 workflow_type: "rust.side-effect-version".to_string(),
17181 cancel_requested: false,
17182 payload_codec: DEFAULT_CODEC.to_string(),
17183 arguments: Some(
17184 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("arguments"),
17185 ),
17186 history_events,
17187 total_history_events: None,
17188 history_size_bytes: None,
17189 continue_as_new_recommended: None,
17190 history_budget_pressure: None,
17191 next_history_page_token: None,
17192 workflow_task_attempt: 1,
17193 workflow_signal_id: None,
17194 signal_name: None,
17195 signal_arguments: None,
17196 workflow_update_id: None,
17197 update_name: None,
17198 lease_owner: Some("rust-worker".to_string()),
17199 }
17200 }
17201
17202 let calls = Arc::new(AtomicUsize::new(0));
17203 let initial = worker(Arc::clone(&calls))
17204 .execute_workflow_task(task(Vec::new()))
17205 .expect("initial execution");
17206 assert_eq!(
17207 initial
17208 .iter()
17209 .map(|command| &command["type"])
17210 .collect::<Vec<_>>(),
17211 vec![
17212 "record_side_effect",
17213 "record_version_marker",
17214 "complete_workflow"
17215 ]
17216 );
17217 assert_eq!(calls.load(Ordering::SeqCst), 1);
17218
17219 let restarted = worker(Arc::clone(&calls));
17220 let replayed = restarted
17221 .execute_workflow_task(task(vec![
17222 history_event(
17223 "SideEffectRecorded",
17224 json!({"sequence": 1, "result": initial[0]["result"].clone()}),
17225 ),
17226 history_event(
17227 "VersionMarkerRecorded",
17228 json!({
17229 "sequence": 2,
17230 "change_id": "cold-restart",
17231 "version": 2,
17232 "min_supported": 1,
17233 "max_supported": 2,
17234 }),
17235 ),
17236 ]))
17237 .expect("cold replay");
17238 assert_eq!(replayed.len(), 1);
17239 assert_eq!(replayed[0]["type"], "complete_workflow");
17240 assert_eq!(calls.load(Ordering::SeqCst), 1);
17241 }
17242
17243 #[test]
17244 fn side_effect_replay_rejects_changed_rust_value_type() {
17245 let result = encode_value_envelope(&json!({"value": 42}), DEFAULT_CODEC).expect("result");
17246 let ctx = workflow_context(vec![history_event(
17247 "SideEffectRecorded",
17248 json!({"sequence": 1, "result": result}),
17249 )]);
17250 let error = ctx
17251 .side_effect::<Vec<String>, _>(|| panic!("callback must not run"))
17252 .expect_err("changed type must fail replay");
17253 assert!(matches!(
17254 error,
17255 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
17256 if reason == "side_effect_type_mismatch"
17257 ));
17258 }
17259
17260 fn completed_retry_activity_history() -> Vec<HistoryEvent> {
17261 vec![
17262 history_event(
17263 "ActivityScheduled",
17264 json!({
17265 "sequence": 1,
17266 "activity_type": "flaky",
17267 "activity_execution_id": "act-1",
17268 "activity": {
17269 "id": "act-1",
17270 "sequence": 1,
17271 "type": "flaky",
17272 "queue": "critical-activities",
17273 "execution_mode": null,
17274 "retry_policy": {
17275 "snapshot_version": 1,
17276 "max_attempts": 3,
17277 "backoff_seconds": [2, 4],
17278 "start_to_close_timeout": 30,
17279 "schedule_to_start_timeout": 5,
17280 "schedule_to_close_timeout": 90,
17281 "heartbeat_timeout": 10,
17282 "non_retryable_error_types": ["PermanentError"]
17283 }
17284 }
17285 }),
17286 ),
17287 history_event(
17288 "ActivityStarted",
17289 json!({
17290 "sequence": 1,
17291 "activity_type": "flaky",
17292 "activity_execution_id": "act-1",
17293 "activity_attempt_id": "attempt-1",
17294 "attempt_number": 1
17295 }),
17296 ),
17297 history_event(
17298 "ActivityRetryScheduled",
17299 json!({
17300 "sequence": 1,
17301 "activity_type": "flaky",
17302 "activity_execution_id": "act-1",
17303 "activity_attempt_id": "attempt-1",
17304 "attempt_number": 1,
17305 "retry_after_attempt": 1,
17306 "retry_backoff_seconds": 2,
17307 "failure_category": "activity",
17308 "exception_type": "TransientError"
17309 }),
17310 ),
17311 history_event(
17312 "ActivityStarted",
17313 json!({
17314 "sequence": 1,
17315 "activity_type": "flaky",
17316 "activity_execution_id": "act-1",
17317 "activity_attempt_id": "attempt-2",
17318 "attempt_number": 2
17319 }),
17320 ),
17321 history_event(
17322 "ActivityCompleted",
17323 json!({
17324 "sequence": 1,
17325 "activity_type": "flaky",
17326 "activity_execution_id": "act-1",
17327 "activity_attempt_id": "attempt-2",
17328 "attempt_number": 2,
17329 "payload_codec": DEFAULT_CODEC,
17330 "result": fixture_envelope(json!({"status":"recovered"}))
17331 }),
17332 ),
17333 ]
17334 }
17335
17336 fn retry_activity_options() -> ActivityOptions {
17337 ActivityOptions::new()
17338 .task_queue("critical-activities")
17339 .retry_policy(
17340 ActivityRetryPolicy::new(3)
17341 .backoff_intervals([Duration::from_secs(2), Duration::from_secs(4)])
17342 .non_retryable_error_type("PermanentError"),
17343 )
17344 .start_to_close_timeout(Duration::from_secs(30))
17345 .schedule_to_start_timeout(Duration::from_secs(5))
17346 .schedule_to_close_timeout(Duration::from_secs(90))
17347 .heartbeat_timeout(Duration::from_secs(10))
17348 }
17349
17350 #[test]
17351 fn fixed_avro_value_round_trips_json_values() {
17352 let value = json!({"greeting": "hello", "count": 3, "ok": true});
17353 let envelope = PayloadEnvelope::avro(&value).expect("encode");
17354 assert_eq!(envelope.codec, DEFAULT_CODEC);
17355 assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
17356 }
17357
17358 #[tokio::test]
17359 async fn typed_handler_adapters_round_trip_serde_contracts_on_the_fixed_wire() {
17360 let client = Client::new("http://127.0.0.1:8080").expect("client");
17361 let mut worker = Worker::new(client, "rust-workers");
17362 worker.register_typed_workflow(
17363 "typed.contract.workflow",
17364 |_ctx, input: TypedContract| async move { Ok(input) },
17365 );
17366 worker.register_typed_activity(
17367 "typed.contract.activity",
17368 |_ctx, input: TypedContract| async move { Ok(input) },
17369 );
17370
17371 let expected = typed_contract();
17372 let arguments = AvroValue::Array(vec![
17373 AvroValue::from_serialize(&expected).expect("typed request")
17374 ]);
17375 let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("arguments");
17376 let mut workflow = workflow_task("typed.contract.workflow", Vec::new(), DEFAULT_CODEC);
17377 workflow.arguments = Some(envelope.clone());
17378 let commands = worker
17379 .execute_workflow_task(workflow)
17380 .expect("typed workflow task");
17381 let workflow_result: TypedContract =
17382 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17383 .expect("workflow result envelope")
17384 .deserialize()
17385 .expect("workflow result type");
17386 assert_eq!(workflow_result, expected);
17387
17388 let activity = ActivityTask {
17389 task_id: "typed-contract-activity".to_string(),
17390 activity_attempt_id: Some("typed-contract-attempt".to_string()),
17391 attempt_id: None,
17392 activity_type: "typed.contract.activity".to_string(),
17393 payload_codec: DEFAULT_CODEC.to_string(),
17394 arguments: Some(envelope),
17395 attempt_number: 1,
17396 lease_owner: Some("rust-worker".to_string()),
17397 };
17398 let activity_result: TypedContract = worker
17399 .execute_activity_task(activity)
17400 .await
17401 .expect("typed activity task")
17402 .deserialize()
17403 .expect("activity result type");
17404 assert_eq!(activity_result, expected);
17405 }
17406
17407 #[tokio::test]
17408 async fn typed_handler_errors_include_handler_name_direction_and_rust_type() {
17409 let client = Client::new("http://127.0.0.1:8080").expect("client");
17410 let mut worker = Worker::new(client, "rust-workers");
17411 worker.register_typed_workflow(
17412 "typed.shape.workflow",
17413 |_ctx, input: TypedContract| async move { Ok(input) },
17414 );
17415 worker.register_typed_activity("typed.unsupported.activity", |_ctx, (): ()| async move {
17416 Ok(f64::NAN)
17417 });
17418
17419 let mut workflow = workflow_task("typed.shape.workflow", Vec::new(), DEFAULT_CODEC);
17420 workflow.arguments = Some(
17421 encode_typed_envelope(
17422 &AvroValue::Array(vec![
17423 AvroValue::String("first".to_string()),
17424 AvroValue::String("second".to_string()),
17425 ]),
17426 DEFAULT_CODEC,
17427 )
17428 .expect("malformed typed arguments"),
17429 );
17430 let commands = worker
17431 .execute_workflow_task(workflow)
17432 .expect("shape mismatch becomes a workflow failure");
17433 let message = commands[0]["message"].as_str().expect("failure message");
17434 assert!(message.contains("workflow handler \"typed.shape.workflow\" input type"));
17435 assert!(message.contains(type_name::<TypedContract>()));
17436 assert!(message.contains("task carried 2 arguments"));
17437
17438 let activity = ActivityTask {
17439 task_id: "typed-unsupported-activity".to_string(),
17440 activity_attempt_id: Some("typed-unsupported-attempt".to_string()),
17441 attempt_id: None,
17442 activity_type: "typed.unsupported.activity".to_string(),
17443 payload_codec: DEFAULT_CODEC.to_string(),
17444 arguments: Some(
17445 encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
17446 .expect("unit arguments"),
17447 ),
17448 attempt_number: 1,
17449 lease_owner: Some("rust-worker".to_string()),
17450 };
17451 let Error::HandlerType {
17452 handler_kind,
17453 handler_name,
17454 value_kind,
17455 rust_type,
17456 message,
17457 } = worker
17458 .execute_activity_task(activity)
17459 .await
17460 .expect_err("non-finite handler output must fail")
17461 else {
17462 panic!("expected contextual handler type failure");
17463 };
17464 assert_eq!(handler_kind, HandlerKind::Activity);
17465 assert_eq!(handler_name, "typed.unsupported.activity");
17466 assert_eq!(value_kind, HandlerValueKind::Result);
17467 assert_eq!(rust_type, type_name::<f64>());
17468 assert!(message.contains("non_finite_float"));
17469 }
17470
17471 #[tokio::test]
17472 async fn typed_replayed_workflow_decodes_input_and_activity_result_losslessly() {
17473 #[derive(Clone, Default)]
17474 struct State {
17475 observed: Option<TypedContract>,
17476 }
17477
17478 let client = Client::new("http://127.0.0.1:8080").expect("client");
17479 let mut worker = Worker::new(client, "rust-workers");
17480 worker.register_typed_replayed_workflow(
17481 "typed.contract.replayed",
17482 State::default,
17483 |ctx, input: TypedContract, state| async move {
17484 let result: TypedContract =
17485 ctx.activity_typed("typed.contract.activity", input).await?;
17486 state.update(|current| current.observed = Some(result.clone()))?;
17487 Ok(result)
17488 },
17489 );
17490 worker.register_replayed_query::<State, _, _>(
17491 "typed.contract.replayed",
17492 "observed",
17493 |_ctx, state, _args| async move {
17494 Ok(json!(state.observed.as_ref().map(|value| value.signed)))
17495 },
17496 );
17497
17498 let expected = typed_contract();
17499 let typed_value = AvroValue::from_serialize(&expected).expect("typed value");
17500 let workflow_arguments =
17501 encode_typed_envelope(&AvroValue::Array(vec![typed_value.clone()]), DEFAULT_CODEC)
17502 .expect("workflow arguments");
17503 let result = encode_typed_envelope(&typed_value, DEFAULT_CODEC).expect("activity result");
17504 let task = QueryTask {
17505 query_task_id: "typed-replay-query".to_string(),
17506 query_task_attempt: 1,
17507 lease_owner: Some("rust-worker".to_string()),
17508 workflow_id: Some("typed-replay".to_string()),
17509 run_id: Some("typed-replay-run".to_string()),
17510 workflow_type: "typed.contract.replayed".to_string(),
17511 query_name: "observed".to_string(),
17512 payload_codec: DEFAULT_CODEC.to_string(),
17513 workflow_arguments: Some(workflow_arguments),
17514 query_arguments: Some(
17515 encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
17516 .expect("query arguments"),
17517 ),
17518 history_events: vec![
17519 history_event(
17520 "ActivityScheduled",
17521 json!({
17522 "sequence": 1,
17523 "activity_type": "typed.contract.activity"
17524 }),
17525 ),
17526 history_event(
17527 "ActivityCompleted",
17528 json!({
17529 "sequence": 1,
17530 "activity_type": "typed.contract.activity",
17531 "payload_codec": DEFAULT_CODEC,
17532 "result": result
17533 }),
17534 ),
17535 ],
17536 history_export: None,
17537 run_status: Some("completed".to_string()),
17538 };
17539
17540 assert_eq!(
17541 worker
17542 .execute_query_task(task)
17543 .await
17544 .expect("typed replay query")
17545 .deserialize::<i64>()
17546 .expect("query result"),
17547 expected.signed
17548 );
17549 }
17550
17551 #[tokio::test]
17552 async fn typed_worker_surfaces_preserve_bytes_and_map_list_identity() {
17553 let client = Client::new("http://127.0.0.1:8080").expect("client");
17554 let mut worker = Worker::new(client, "rust-workers");
17555 worker.register_workflow_avro_value("typed.echo", |_ctx, input| async move { Ok(input) });
17556 worker
17557 .register_activity_avro_value("typed.activity", |_ctx, input| async move { Ok(input) });
17558 worker.register_query_avro_value("typed.echo", "inspect", |_ctx, input| async move {
17559 Ok(input)
17560 });
17561 worker.register_update_avro_value("typed.echo", "replace", |_ctx, input| async move {
17562 Ok(input)
17563 });
17564 worker.register_workflow_avro_value("typed.signal", |ctx, _input| async move {
17565 Ok(AvroValue::Array(
17566 ctx.wait_signal_avro_value("changed").await?,
17567 ))
17568 });
17569
17570 let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
17571 let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed envelope");
17572
17573 let mut workflow = workflow_task("typed.echo", Vec::new(), DEFAULT_CODEC);
17574 workflow.arguments = Some(envelope.clone());
17575 let commands = worker
17576 .execute_workflow_task(workflow)
17577 .expect("typed workflow task");
17578 assert_eq!(commands[0]["type"], "complete_workflow");
17579 assert_eq!(
17580 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17581 .expect("typed workflow result"),
17582 arguments
17583 );
17584
17585 let activity = ActivityTask {
17586 task_id: "activity-typed".to_string(),
17587 activity_attempt_id: Some("attempt-typed".to_string()),
17588 attempt_id: None,
17589 activity_type: "typed.activity".to_string(),
17590 payload_codec: DEFAULT_CODEC.to_string(),
17591 arguments: Some(envelope.clone()),
17592 attempt_number: 1,
17593 lease_owner: Some("rust-worker".to_string()),
17594 };
17595 assert_eq!(
17596 worker
17597 .execute_activity_task(activity)
17598 .await
17599 .expect("typed activity result"),
17600 arguments
17601 );
17602
17603 let query = QueryTask {
17604 query_task_id: "query-typed".to_string(),
17605 query_task_attempt: 1,
17606 lease_owner: Some("rust-worker".to_string()),
17607 workflow_id: Some("typed-1".to_string()),
17608 run_id: Some("run-typed".to_string()),
17609 workflow_type: "typed.echo".to_string(),
17610 query_name: "inspect".to_string(),
17611 payload_codec: DEFAULT_CODEC.to_string(),
17612 workflow_arguments: Some(
17613 encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
17614 .expect("workflow input"),
17615 ),
17616 query_arguments: Some(envelope.clone()),
17617 history_events: Vec::new(),
17618 history_export: None,
17619 run_status: Some("running".to_string()),
17620 };
17621 assert_eq!(
17622 worker
17623 .execute_query_task(query)
17624 .await
17625 .expect("typed query result"),
17626 arguments
17627 );
17628
17629 let mut update = workflow_task(
17630 "typed.echo",
17631 vec![history_event(
17632 "UpdateAccepted",
17633 json!({
17634 "update_id": "update-typed",
17635 "update_name": "replace",
17636 "arguments": envelope.clone(),
17637 }),
17638 )],
17639 DEFAULT_CODEC,
17640 );
17641 update.workflow_update_id = Some("update-typed".to_string());
17642 update.update_name = Some("replace".to_string());
17643 let commands = worker
17644 .execute_workflow_task(update)
17645 .expect("typed update task");
17646 assert_eq!(commands[0]["type"], "complete_update");
17647 assert_eq!(
17648 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17649 .expect("typed update result"),
17650 arguments
17651 );
17652
17653 let mut signal = workflow_task(
17654 "typed.signal",
17655 vec![history_event(
17656 "SignalReceived",
17657 json!({
17658 "signal_id": "signal-typed",
17659 "signal_name": "changed",
17660 "arguments": envelope.clone(),
17661 }),
17662 )],
17663 DEFAULT_CODEC,
17664 );
17665 signal.workflow_signal_id = Some("signal-typed".to_string());
17666 signal.signal_name = Some("changed".to_string());
17667 signal.signal_arguments = Some(envelope);
17668 let commands = worker
17669 .execute_workflow_task(signal)
17670 .expect("typed signal resume");
17671 assert_eq!(
17672 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17673 .expect("typed signal result"),
17674 arguments
17675 );
17676 }
17677
17678 #[tokio::test]
17679 async fn typed_helpers_never_parse_json_inspection_projection() {
17680 let collision_values = projection_collision_probe();
17681 let expected = AvroValue::Array(collision_values.clone());
17682 let envelope = encode_typed_envelope(&expected, DEFAULT_CODEC).expect("collision envelope");
17683
17684 let activity_context = workflow_context_with_codec(
17685 vec![history_event(
17686 "ActivityCompleted",
17687 json!({
17688 "sequence": 1,
17689 "activity_type": "collision.activity",
17690 "payload_codec": DEFAULT_CODEC,
17691 "result": envelope.clone(),
17692 }),
17693 )],
17694 DEFAULT_CODEC,
17695 );
17696 assert_eq!(
17697 activity_context
17698 .activity_avro_value("collision.activity", AvroValue::Array(Vec::new()))
17699 .await
17700 .expect("typed activity collision result"),
17701 expected
17702 );
17703
17704 let signal_context = workflow_context_with_codec(
17705 vec![
17706 history_event(
17707 "SignalWaitOpened",
17708 json!({"sequence": 1, "signal_name": "collision"}),
17709 ),
17710 history_event(
17711 "SignalApplied",
17712 json!({
17713 "sequence": 1,
17714 "signal_name": "collision",
17715 "payload_codec": DEFAULT_CODEC,
17716 "value": envelope.clone(),
17717 }),
17718 ),
17719 ],
17720 DEFAULT_CODEC,
17721 );
17722 assert_eq!(
17723 signal_context
17724 .wait_signal_avro_value("collision")
17725 .await
17726 .expect("typed signal collision arguments"),
17727 collision_values
17728 );
17729
17730 let child_context = workflow_context_with_codec(
17731 vec![
17732 history_event(
17733 "ChildWorkflowScheduled",
17734 json!({
17735 "sequence": 1,
17736 "child_workflow_instance_id": "collision-child",
17737 "child_workflow_run_id": "collision-run",
17738 "child_workflow_type": "collision.child",
17739 }),
17740 ),
17741 history_event(
17742 "ChildRunCompleted",
17743 json!({
17744 "sequence": 1,
17745 "child_workflow_instance_id": "collision-child",
17746 "child_workflow_run_id": "collision-run",
17747 "child_workflow_type": "collision.child",
17748 "payload_codec": DEFAULT_CODEC,
17749 "result": envelope,
17750 }),
17751 ),
17752 ],
17753 DEFAULT_CODEC,
17754 );
17755 let child = child_context
17756 .start_child_workflow_avro_value(
17757 "collision.child",
17758 ChildWorkflowOptions::new("collision-workers"),
17759 AvroValue::Array(Vec::new()),
17760 )
17761 .await
17762 .expect("typed child collision result");
17763 assert_eq!(child.result, expected);
17764 }
17765
17766 #[tokio::test]
17767 async fn replayed_typed_query_keeps_lossless_workflow_and_query_inputs() {
17768 let client = Client::new("http://127.0.0.1:8080").expect("client");
17769 let mut worker = Worker::new(client, "rust-workers");
17770 worker.register_replayed_workflow_avro_value(
17771 "typed.replayed",
17772 || (),
17773 |_ctx, input, _state| async move { Ok(input) },
17774 );
17775 worker.register_replayed_query_avro_value::<(), _, _>(
17776 "typed.replayed",
17777 "inspect",
17778 |ctx, _state, args| async move {
17779 let mut signals = ctx.signals_avro_value("collision");
17780 let signal = signals
17781 .pop()
17782 .map(AvroValue::Array)
17783 .unwrap_or_else(|| AvroValue::Array(Vec::new()));
17784 Ok(AvroValue::Array(vec![
17785 ctx.workflow_input_avro_value().clone(),
17786 signal,
17787 args,
17788 ]))
17789 },
17790 );
17791 let arguments = AvroValue::Array(projection_collision_probe());
17792 let signal_arguments =
17793 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed query signal arguments");
17794 let task = QueryTask {
17795 query_task_id: "query-typed-replay".to_string(),
17796 query_task_attempt: 1,
17797 lease_owner: Some("rust-worker".to_string()),
17798 workflow_id: Some("typed-replay".to_string()),
17799 run_id: Some("run-typed-replay".to_string()),
17800 workflow_type: "typed.replayed".to_string(),
17801 query_name: "inspect".to_string(),
17802 payload_codec: DEFAULT_CODEC.to_string(),
17803 workflow_arguments: Some(
17804 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("workflow arguments"),
17805 ),
17806 query_arguments: Some(
17807 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("query arguments"),
17808 ),
17809 history_events: vec![history_event(
17810 "SignalReceived",
17811 json!({
17812 "signal_id": "collision-signal",
17813 "signal_name": "collision",
17814 "workflow_sequence": 1,
17815 "payload_codec": DEFAULT_CODEC,
17816 "arguments": signal_arguments,
17817 }),
17818 )],
17819 history_export: None,
17820 run_status: Some("completed".to_string()),
17821 };
17822
17823 assert_eq!(
17824 worker
17825 .execute_query_task(task)
17826 .await
17827 .expect("typed replay query"),
17828 AvroValue::Array(vec![arguments.clone(), arguments.clone(), arguments])
17829 );
17830 }
17831
17832 #[test]
17833 fn public_avro_adapter_rejects_non_string_map_keys_before_json_conversion() {
17834 let value = BTreeMap::from([(1_i32, "integer key")]);
17835 let error = PayloadEnvelope::avro(&value)
17836 .expect_err("integer map keys must fail")
17837 .to_string();
17838
17839 assert!(error.contains("invalid_map_key"));
17840 }
17841
17842 #[test]
17843 fn json_tagged_payload_fails_closed_with_actionable_diagnostic() {
17844 let envelope = PayloadEnvelope {
17845 codec: "json".to_string(),
17846 blob: r#"{"greeting":"hello"}"#.to_string(),
17847 };
17848
17849 let error = decode_payload::<Value>(&envelope).expect_err("JSON payload must fail");
17850 let diagnostic = error.to_string();
17851 assert!(diagnostic.contains("unsupported_payload_codec"));
17852 assert!(diagnostic.contains("codec=\"avro\""));
17853 assert!(diagnostic.contains("HTTP document transport"));
17854 }
17855
17856 #[test]
17857 fn untagged_json_payload_value_fails_closed() {
17858 let error = decode_wire_value(&json!({"stale": true}), DEFAULT_CODEC)
17859 .expect_err("untagged JSON payload values must fail");
17860 let diagnostic = error.to_string();
17861 assert!(diagnostic.contains("unsupported_payload_codec"));
17862 assert!(diagnostic.contains("untagged durable payload"));
17863 assert!(diagnostic.contains("HTTP document transport"));
17864 }
17865
17866 #[test]
17867 fn prerelease_avro_payload_without_single_object_frame_is_rejected() {
17868 let envelope = PayloadEnvelope {
17869 codec: DEFAULT_CODEC.to_string(),
17870 blob: BASE64.encode([0x01]),
17871 };
17872
17873 let error = decode_payload::<Value>(&envelope).expect_err("prerelease payload must fail");
17874 assert!(error.to_string().contains("invalid_payload_framing"));
17875 }
17876
17877 #[tokio::test]
17878 async fn workflow_completion_rejects_invalid_payload_slots_without_transport() {
17879 let server = MockWorkerServer::start();
17880 let client = Client::builder(server.base_url())
17881 .timeout(Duration::from_secs(2))
17882 .build()
17883 .expect("client");
17884 let invalid_commands = [
17885 json!({
17886 "type": "complete_workflow",
17887 "result": {"codec": "json", "blob": null}
17888 }),
17889 json!({
17890 "type": "schedule_activity",
17891 "arguments": {"codec": "yaml", "blob": "ignored"}
17892 }),
17893 json!({
17894 "type": "start_child_workflow",
17895 "arguments": {"codec": DEFAULT_CODEC, "blob": null}
17896 }),
17897 json!({"type": "continue_as_new", "arguments": []}),
17898 json!({"type": "complete_update"}),
17899 json!({"type": "record_side_effect", "result": null}),
17900 json!({
17901 "type": "start_service_operation",
17902 "payload_codec": DEFAULT_CODEC,
17903 "request_payload": "raw-avro-bytes"
17904 }),
17905 ];
17906
17907 for command in invalid_commands {
17908 let error = client
17909 .complete_workflow_task("invalid-codec", "rust-worker", 1, vec![command])
17910 .await
17911 .expect_err("invalid durable payload must fail locally");
17912 let diagnostic = error.to_string();
17913 assert!(
17914 diagnostic.contains("unsupported_payload_codec")
17915 || diagnostic.contains("invalid_payload_envelope")
17916 || diagnostic.contains("untagged durable payload"),
17917 "unexpected validation diagnostic: {diagnostic}"
17918 );
17919 }
17920
17921 assert_eq!(
17922 server.request_count("/api/worker/workflow-tasks/invalid-codec/complete"),
17923 0,
17924 "invalid command payloads must not reach HTTP transport"
17925 );
17926 }
17927
17928 #[test]
17929 fn workflow_completion_validates_only_protocol_owned_payload_slots() {
17930 let envelope = fixture_envelope(json!({"codec": "customer-value"}));
17931 let commands = [
17932 json!({"type": "complete_workflow", "result": envelope.clone()}),
17933 json!({"type": "schedule_activity", "arguments": envelope.clone()}),
17934 json!({"type": "start_child_workflow", "arguments": envelope.clone()}),
17935 json!({"type": "continue_as_new", "arguments": envelope.clone()}),
17936 json!({"type": "complete_update", "result": envelope.clone()}),
17937 json!({"type": "record_side_effect", "result": envelope.clone()}),
17938 json!({
17939 "type": "start_service_operation",
17940 "payload_codec": DEFAULT_CODEC,
17941 "request_payload": envelope.clone()
17942 }),
17943 json!({
17944 "type": "complete_workflow",
17945 "result": envelope,
17946 "metadata": {
17947 "codec": "json",
17948 "payload_codec": "customer-codec",
17949 "result": {"codec": "yaml", "blob": null}
17950 }
17951 }),
17952 ];
17953
17954 validate_workflow_task_commands(&commands)
17955 .expect("customer metadata must not become a protocol codec declaration");
17956 }
17957
17958 #[test]
17959 fn valid_avro_tasks_normalize_absent_and_null_arguments_to_empty_lists() {
17960 assert_eq!(
17961 decode_task_avro_arguments(None, DEFAULT_CODEC).expect("absent arguments"),
17962 AvroValue::Array(Vec::new())
17963 );
17964 assert_eq!(
17965 decode_task_avro_arguments(Some(&Value::Null), DEFAULT_CODEC).expect("null arguments"),
17966 AvroValue::Array(Vec::new())
17967 );
17968
17969 let mut signal = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
17970 signal.signal_name = Some("empty-signal".to_string());
17971 signal.signal_arguments = None;
17972 let decoded = decode_resume_signal(&signal)
17973 .expect("valid Avro signal")
17974 .expect("named signal resumes the workflow");
17975 assert!(decoded.arguments.is_empty());
17976 }
17977
17978 #[tokio::test]
17979 async fn malformed_task_level_codecs_become_pre_handler_failures() {
17980 let client = Client::new("http://127.0.0.1:8080").expect("client");
17981 let mut worker = Worker::new(client, "rust-workers");
17982 let handler_calls = Arc::new(AtomicUsize::new(0));
17983
17984 let calls = Arc::clone(&handler_calls);
17985 worker.register_workflow("codec.workflow", move |_ctx, _args| {
17986 calls.fetch_add(1, Ordering::SeqCst);
17987 async move { Ok(Value::Null) }
17988 });
17989 let calls = Arc::clone(&handler_calls);
17990 worker.register_activity("codec.activity", move |_ctx, _args| {
17991 calls.fetch_add(1, Ordering::SeqCst);
17992 async move { Ok(Value::Null) }
17993 });
17994 let calls = Arc::clone(&handler_calls);
17995 worker.register_query("codec.workflow", "known", move |_ctx, _args| {
17996 calls.fetch_add(1, Ordering::SeqCst);
17997 async move { Ok(Value::Null) }
17998 });
17999
18000 let mut failures = Vec::new();
18001 for codec_case in [
18002 InvalidTaskPayloadCodec::Missing,
18003 InvalidTaskPayloadCodec::Null,
18004 InvalidTaskPayloadCodec::NonString,
18005 ] {
18006 let mut workflow = json!({
18007 "task_id": format!("workflow-{}", codec_case.label()),
18008 "workflow_type": "codec.workflow"
18009 });
18010 codec_case.apply(&mut workflow);
18011 match serde_json::from_value::<WorkflowTask>(workflow) {
18012 Ok(task) => match worker.execute_workflow_task(task) {
18013 Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
18014 outcome => failures.push(format!(
18015 "workflow {} codec returned {outcome:?}",
18016 codec_case.label()
18017 )),
18018 },
18019 Err(error) => failures.push(format!(
18020 "workflow {} codec failed transport deserialization: {error}",
18021 codec_case.label()
18022 )),
18023 }
18024
18025 let mut activity = json!({
18026 "task_id": format!("activity-{}", codec_case.label()),
18027 "activity_attempt_id": format!("attempt-{}", codec_case.label()),
18028 "activity_type": "codec.activity",
18029 "attempt_number": 1
18030 });
18031 codec_case.apply(&mut activity);
18032 match serde_json::from_value::<ActivityTask>(activity) {
18033 Ok(task) => match worker.execute_activity_task(task).await {
18034 Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
18035 outcome => failures.push(format!(
18036 "activity {} codec returned {outcome:?}",
18037 codec_case.label()
18038 )),
18039 },
18040 Err(error) => failures.push(format!(
18041 "activity {} codec failed transport deserialization: {error}",
18042 codec_case.label()
18043 )),
18044 }
18045
18046 let mut query = json!({
18047 "query_task_id": format!("query-{}", codec_case.label()),
18048 "workflow_type": "codec.workflow",
18049 "query_name": "known"
18050 });
18051 codec_case.apply(&mut query);
18052 match serde_json::from_value::<QueryTask>(query) {
18053 Ok(task) => match worker.execute_query_task(task).await {
18054 Err(failure) if failure.message.contains("unsupported_payload_codec") => {}
18055 outcome => failures.push(format!(
18056 "query {} codec returned {outcome:?}",
18057 codec_case.label()
18058 )),
18059 },
18060 Err(error) => failures.push(format!(
18061 "query {} codec failed transport deserialization: {error}",
18062 codec_case.label()
18063 )),
18064 }
18065 }
18066
18067 assert!(failures.is_empty(), "{}", failures.join("\n"));
18068 assert_eq!(
18069 handler_calls.load(Ordering::SeqCst),
18070 0,
18071 "invalid task codecs must not invoke a handler"
18072 );
18073 }
18074
18075 #[tokio::test]
18076 async fn polled_malformed_task_codecs_are_settled_without_handler_execution() {
18077 for codec_case in [
18078 InvalidTaskPayloadCodec::Missing,
18079 InvalidTaskPayloadCodec::Null,
18080 InvalidTaskPayloadCodec::NonString,
18081 ] {
18082 let server = MockWorkerServer::invalid_task_payload_codec(codec_case);
18083 let client = Client::builder(server.base_url())
18084 .timeout(Duration::from_secs(2))
18085 .build()
18086 .expect("client");
18087 let mut worker = Worker::new(client, "rust-workers")
18088 .worker_id("codec-worker")
18089 .poll_timeout(Duration::from_millis(10));
18090 let handler_calls = Arc::new(AtomicUsize::new(0));
18091
18092 let calls = Arc::clone(&handler_calls);
18093 worker.register_workflow("codec.workflow", move |_ctx, _args| {
18094 calls.fetch_add(1, Ordering::SeqCst);
18095 async move { Ok(Value::Null) }
18096 });
18097 let calls = Arc::clone(&handler_calls);
18098 worker.register_activity("codec.activity", move |_ctx, _args| {
18099 calls.fetch_add(1, Ordering::SeqCst);
18100 async move { Ok(Value::Null) }
18101 });
18102 let calls = Arc::clone(&handler_calls);
18103 worker.register_query("codec.workflow", "known", move |_ctx, _args| {
18104 calls.fetch_add(1, Ordering::SeqCst);
18105 async move { Ok(Value::Null) }
18106 });
18107
18108 assert_eq!(
18109 worker.run_once().await.expect("invalid tasks are settled"),
18110 3,
18111 "all {} codec tasks must be handled",
18112 codec_case.label()
18113 );
18114 assert_eq!(
18115 handler_calls.load(Ordering::SeqCst),
18116 0,
18117 "{} task codecs must fail before every handler",
18118 codec_case.label()
18119 );
18120
18121 for path in [
18122 "/api/worker/workflow-tasks/codec-workflow/fail",
18123 "/api/worker/activity-tasks/codec-activity/fail",
18124 "/api/worker/query-tasks/codec-query/fail",
18125 ] {
18126 let body = server.request_body(path);
18127 assert!(
18128 body["failure"]["message"]
18129 .as_str()
18130 .is_some_and(|message| message.contains("unsupported_payload_codec")),
18131 "{path} must receive the stable codec diagnostic for the {} case: {body}",
18132 codec_case.label()
18133 );
18134 }
18135 assert_eq!(
18136 server.request_body("/api/worker/query-tasks/codec-query/fail")["failure"]
18137 ["reason"],
18138 "query_payload_decode_failed"
18139 );
18140 for path in [
18141 "/api/worker/workflow-tasks/codec-workflow/complete",
18142 "/api/worker/activity-tasks/codec-activity/complete",
18143 "/api/worker/query-tasks/codec-query/complete",
18144 ] {
18145 assert_eq!(
18146 server.request_count(path),
18147 0,
18148 "invalid {} codec task reached {path}",
18149 codec_case.label()
18150 );
18151 }
18152 }
18153 }
18154
18155 #[tokio::test]
18156 async fn invalid_inbound_codecs_precede_handlers_and_unrelated_outcomes() {
18157 let client = Client::new("http://127.0.0.1:8080").expect("client");
18158 let mut worker = Worker::new(client, "rust-workers");
18159 let handler_calls = Arc::new(AtomicUsize::new(0));
18160
18161 let calls = Arc::clone(&handler_calls);
18162 worker.register_workflow("codec.workflow", move |_ctx, _args| {
18163 calls.fetch_add(1, Ordering::SeqCst);
18164 async move { Ok(Value::Null) }
18165 });
18166 let calls = Arc::clone(&handler_calls);
18167 worker.register_activity("codec.activity", move |_ctx, _args| {
18168 calls.fetch_add(1, Ordering::SeqCst);
18169 async move { Ok(Value::Null) }
18170 });
18171 let calls = Arc::clone(&handler_calls);
18172 worker.register_update("codec.workflow", "known", move |_ctx, _args| {
18173 calls.fetch_add(1, Ordering::SeqCst);
18174 async move { Ok(Value::Null) }
18175 });
18176 let calls = Arc::clone(&handler_calls);
18177 worker.register_query("codec.workflow", "known", move |_ctx, _args| {
18178 calls.fetch_add(1, Ordering::SeqCst);
18179 async move { Ok(Value::Null) }
18180 });
18181
18182 let mut workflow = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
18183 workflow.payload_codec = "json".to_string();
18184 workflow.arguments = None;
18185 let error = worker
18186 .execute_workflow_task(workflow)
18187 .expect_err("task codec must be checked before workflow invocation");
18188 assert!(error.to_string().contains("unsupported_payload_codec"));
18189
18190 let activity = ActivityTask {
18191 task_id: "activity-invalid-codec".to_string(),
18192 activity_attempt_id: None,
18193 attempt_id: None,
18194 activity_type: "codec.activity".to_string(),
18195 payload_codec: "unknown".to_string(),
18196 arguments: None,
18197 attempt_number: 1,
18198 lease_owner: None,
18199 };
18200 let error = worker
18201 .execute_activity_task(activity)
18202 .await
18203 .expect_err("task codec must be checked before activity invocation");
18204 assert!(error.to_string().contains("unsupported_payload_codec"));
18205
18206 let mut update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
18207 update.workflow_update_id = Some("update-invalid-codec".to_string());
18208 update.update_name = Some("known".to_string());
18209 update.history_events.push(history_event(
18210 "UpdateAccepted",
18211 json!({
18212 "update_id": "update-invalid-codec",
18213 "update_name": "known",
18214 "arguments": {"codec": "json", "blob": null}
18215 }),
18216 ));
18217 let error = worker
18218 .execute_workflow_task(update)
18219 .expect_err("nested update codec must be checked before handler lookup");
18220 assert!(error.to_string().contains("unsupported_payload_codec"));
18221
18222 let query: QueryTask = serde_json::from_value(json!({
18223 "query_task_id": "query-invalid-codec",
18224 "workflow_type": "codec.workflow",
18225 "query_name": "known",
18226 "payload_codec": DEFAULT_CODEC,
18227 "workflow_arguments": null,
18228 "query_arguments": null,
18229 "history_export": {
18230 "payloads": {"codec": DEFAULT_CODEC},
18231 "signals": [{
18232 "name": "empty",
18233 "payload_codec": "json",
18234 "arguments": null
18235 }]
18236 }
18237 }))
18238 .expect("query task");
18239 let failure = worker
18240 .execute_query_task(query)
18241 .await
18242 .expect_err("exported signal codec must be checked before query invocation");
18243 assert_eq!(failure.reason, "query_payload_decode_failed");
18244 assert!(failure.message.contains("unsupported_payload_codec"));
18245
18246 let exported_history: QueryTask = serde_json::from_value(json!({
18247 "query_task_id": "query-invalid-history-codec",
18248 "workflow_type": "codec.workflow",
18249 "query_name": "known",
18250 "payload_codec": DEFAULT_CODEC,
18251 "history_export": {
18252 "payloads": {"codec": DEFAULT_CODEC},
18253 "history_events": [{
18254 "type": "ActivityCompleted",
18255 "payload": {"payload_codec": "unknown", "result": null}
18256 }]
18257 }
18258 }))
18259 .expect("query task");
18260 let failure = worker
18261 .execute_query_task(exported_history)
18262 .await
18263 .expect_err("exported history codec must be checked before query invocation");
18264 assert_eq!(failure.reason, "query_payload_decode_failed");
18265 assert!(failure.message.contains("unsupported_payload_codec"));
18266 assert_eq!(handler_calls.load(Ordering::SeqCst), 0);
18267
18268 let mut unknown_workflow = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
18269 unknown_workflow.arguments = None;
18270 unknown_workflow.history_events.push(history_event(
18271 "SignalReceived",
18272 json!({
18273 "signal_name": "empty",
18274 "payload_codec": "json",
18275 "arguments": null
18276 }),
18277 ));
18278 let error = worker
18279 .execute_workflow_task(unknown_workflow)
18280 .expect_err("history codec must precede unknown workflow outcome");
18281 assert!(error.to_string().contains("unsupported_payload_codec"));
18282
18283 let unknown_activity = ActivityTask {
18284 task_id: "activity-unknown".to_string(),
18285 activity_attempt_id: None,
18286 attempt_id: None,
18287 activity_type: "missing".to_string(),
18288 payload_codec: "json".to_string(),
18289 arguments: None,
18290 attempt_number: 1,
18291 lease_owner: None,
18292 };
18293 let error = worker
18294 .execute_activity_task(unknown_activity)
18295 .await
18296 .expect_err("codec must precede unknown activity outcome");
18297 assert!(error.to_string().contains("unsupported_payload_codec"));
18298
18299 let mut unknown_update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
18300 unknown_update.payload_codec = "json".to_string();
18301 unknown_update.arguments = None;
18302 unknown_update.workflow_update_id = Some("update-unknown".to_string());
18303 unknown_update.update_name = Some("missing".to_string());
18304 let error = worker
18305 .execute_workflow_task(unknown_update)
18306 .expect_err("codec must precede fail_update shortcut");
18307 assert!(error.to_string().contains("unsupported_payload_codec"));
18308
18309 let unknown_query: QueryTask = serde_json::from_value(json!({
18310 "query_task_id": "query-unknown",
18311 "workflow_type": "missing",
18312 "query_name": "missing",
18313 "payload_codec": "json",
18314 "workflow_arguments": null,
18315 "query_arguments": null
18316 }))
18317 .expect("query task");
18318 let failure = worker
18319 .execute_query_task(unknown_query)
18320 .await
18321 .expect_err("codec must precede unknown query outcome");
18322 assert_eq!(failure.reason, "query_payload_decode_failed");
18323 assert!(failure.message.contains("unsupported_payload_codec"));
18324 }
18325
18326 #[tokio::test]
18327 async fn invalid_signal_history_payload_aliases_precede_shortcuts() {
18328 let client = Client::new("http://127.0.0.1:8080").expect("client");
18329 let worker = Worker::new(client, "rust-workers");
18330
18331 for event_type in ["SignalReceived", "SignalApplied"] {
18332 for (payload_field, codec) in [
18333 ("value", "json"),
18334 ("input", "unknown"),
18335 ("arguments", "json"),
18336 ] {
18337 let payload = json!({
18338 "signal_name": "empty",
18339 payload_field: {"codec": codec, "blob": null}
18340 });
18341 let workflow = workflow_task(
18342 "missing",
18343 vec![history_event(event_type, payload.clone())],
18344 DEFAULT_CODEC,
18345 );
18346 let error = worker
18347 .execute_workflow_task(workflow)
18348 .expect_err("signal payload codec must precede unknown workflow outcome");
18349 assert!(
18350 error.to_string().contains("unsupported_payload_codec"),
18351 "{event_type}.{payload_field} returned an unrelated workflow error: {error}"
18352 );
18353
18354 let query: QueryTask = serde_json::from_value(json!({
18355 "query_task_id": format!("query-{event_type}-{payload_field}"),
18356 "workflow_type": "missing",
18357 "query_name": "missing",
18358 "payload_codec": DEFAULT_CODEC,
18359 "workflow_arguments": null,
18360 "query_arguments": null,
18361 "history_events": [{
18362 "event_type": event_type,
18363 "payload": payload
18364 }]
18365 }))
18366 .expect("query task");
18367 let failure = worker
18368 .execute_query_task(query)
18369 .await
18370 .expect_err("signal payload codec must precede unknown query outcome");
18371 assert_eq!(
18372 failure.reason, "query_payload_decode_failed",
18373 "{event_type}.{payload_field} returned an unrelated query outcome"
18374 );
18375 assert!(
18376 failure.message.contains("unsupported_payload_codec"),
18377 "{event_type}.{payload_field} returned an unrelated query error: {}",
18378 failure.message
18379 );
18380 }
18381 }
18382 }
18383
18384 #[test]
18385 fn workflow_context_schedules_activity_until_completion_is_in_history() {
18386 let ctx = WorkflowContext {
18387 state: Arc::new(Mutex::new(
18388 WorkflowState::new_with_identity(
18389 Vec::new(),
18390 Some("wf-parent".to_string()),
18391 Some("run-parent".to_string()),
18392 "rust-workers".to_string(),
18393 DEFAULT_CODEC.to_string(),
18394 None,
18395 )
18396 .expect("workflow state"),
18397 )),
18398 };
18399
18400 let mut call = Box::pin(ctx.activity("hello.activity", json!(["Ada"])));
18401 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18402 assert!(matches!(
18403 call.as_mut().poll(&mut task_context),
18404 Poll::Pending
18405 ));
18406
18407 let commands = ctx.take_commands().expect("commands");
18408 assert_eq!(commands[0]["type"], "schedule_activity");
18409 assert_eq!(commands[0]["activity_type"], "hello.activity");
18410 }
18411
18412 #[test]
18413 fn activity_options_encode_retry_policy_queue_and_every_timeout() {
18414 let ctx = workflow_context(Vec::new());
18415 let options = ActivityOptions::new()
18416 .task_queue("payments")
18417 .retry_policy(
18418 ActivityRetryPolicy::new(4)
18419 .exponential_backoff(Duration::from_secs(1), 3, Some(Duration::from_secs(10)))
18420 .non_retryable_error_type("ValidationError"),
18421 )
18422 .start_to_close_timeout(Duration::from_secs(120))
18423 .schedule_to_start_timeout(Duration::from_secs(10))
18424 .schedule_to_close_timeout(Duration::from_secs(300))
18425 .heartbeat_timeout(Duration::from_secs(15));
18426 let mut call = Box::pin(ctx.activity_with_options(
18427 "charge-card",
18428 options,
18429 json!([{"order_id": "o-1"}]),
18430 ));
18431 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18432
18433 assert!(matches!(
18434 call.as_mut().poll(&mut task_context),
18435 Poll::Pending
18436 ));
18437 assert!(matches!(
18438 call.as_mut().poll(&mut task_context),
18439 Poll::Pending
18440 ));
18441
18442 let commands = ctx.take_commands().expect("activity command");
18443 assert_eq!(commands.len(), 1, "one future emits one logical schedule");
18444 assert_eq!(commands[0]["queue"], "payments");
18445 assert_eq!(
18446 commands[0]["retry_policy"],
18447 json!({
18448 "max_attempts": 4,
18449 "backoff_seconds": [1, 3, 9],
18450 "non_retryable_error_types": ["ValidationError"],
18451 })
18452 );
18453 assert_eq!(commands[0]["start_to_close_timeout"], 120);
18454 assert_eq!(commands[0]["schedule_to_start_timeout"], 10);
18455 assert_eq!(commands[0]["schedule_to_close_timeout"], 300);
18456 assert_eq!(commands[0]["heartbeat_timeout"], 15);
18457 }
18458
18459 #[test]
18460 fn activity_options_encode_explicit_and_rounded_backoff_intervals() {
18461 let ctx = workflow_context(Vec::new());
18462 let options = ActivityOptions::new().retry_policy(
18463 ActivityRetryPolicy::new(3)
18464 .backoff_intervals([Duration::from_millis(1), Duration::from_millis(1_001)]),
18465 );
18466 let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
18467 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18468
18469 assert!(matches!(
18470 call.as_mut().poll(&mut task_context),
18471 Poll::Pending
18472 ));
18473 assert_eq!(
18474 ctx.take_commands().expect("command")[0]["retry_policy"]["backoff_seconds"],
18475 json!([1, 2])
18476 );
18477 }
18478
18479 #[test]
18480 fn invalid_activity_options_return_typed_errors_before_emitting_commands() {
18481 let cases = [
18482 (
18483 ActivityOptions::new().task_queue(" "),
18484 ActivityOptionsErrorKind::EmptyTaskQueue,
18485 ),
18486 (
18487 ActivityOptions::new().retry_policy(ActivityRetryPolicy::default()),
18488 ActivityOptionsErrorKind::EmptyRetryPolicy,
18489 ),
18490 (
18491 ActivityOptions::new().retry_policy(ActivityRetryPolicy::new(0)),
18492 ActivityOptionsErrorKind::InvalidMaxAttempts,
18493 ),
18494 (
18495 ActivityOptions::new().retry_policy(ActivityRetryPolicy {
18496 max_attempts: None,
18497 backoff: Some(ActivityBackoff::Explicit(vec![Duration::from_secs(1)])),
18498 non_retryable_error_types: Vec::new(),
18499 }),
18500 ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
18501 ),
18502 (
18503 ActivityOptions::new().retry_policy(
18504 ActivityRetryPolicy::new(2)
18505 .backoff_intervals([Duration::from_secs(1), Duration::from_secs(2)]),
18506 ),
18507 ActivityOptionsErrorKind::TooManyBackoffIntervals,
18508 ),
18509 (
18510 ActivityOptions::new().retry_policy(
18511 ActivityRetryPolicy::new(2).exponential_backoff(
18512 Duration::from_secs(1),
18513 0,
18514 None,
18515 ),
18516 ),
18517 ActivityOptionsErrorKind::InvalidBackoffCoefficient,
18518 ),
18519 (
18520 ActivityOptions::new()
18521 .retry_policy(ActivityRetryPolicy::new(2).non_retryable_error_type(" ")),
18522 ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
18523 ),
18524 (
18525 ActivityOptions::new().retry_policy(
18526 ActivityRetryPolicy::new(10_002).exponential_backoff(
18527 Duration::from_secs(1),
18528 1,
18529 None,
18530 ),
18531 ),
18532 ActivityOptionsErrorKind::BackoffGenerationTooLarge,
18533 ),
18534 (
18535 ActivityOptions::new().retry_policy(
18536 ActivityRetryPolicy::new(2)
18537 .backoff_intervals([Duration::from_secs(i64::MAX as u64 + 1)]),
18538 ),
18539 ActivityOptionsErrorKind::BackoffOverflow,
18540 ),
18541 ];
18542
18543 for (options, expected_kind) in cases {
18544 let ctx = workflow_context(Vec::new());
18545 let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
18546 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18547 let Poll::Ready(Err(Error::InvalidActivityOptions(error))) =
18548 call.as_mut().poll(&mut task_context)
18549 else {
18550 panic!("expected typed activity validation error");
18551 };
18552 assert_eq!(error.kind, expected_kind);
18553 assert!(ctx.take_commands().expect("commands").is_empty());
18554 }
18555 }
18556
18557 #[test]
18558 fn activity_options_validate_positive_and_ordered_timeouts() {
18559 let zero_timeout_cases = [
18560 ActivityOptions::new().start_to_close_timeout(Duration::ZERO),
18561 ActivityOptions::new().schedule_to_start_timeout(Duration::ZERO),
18562 ActivityOptions::new().schedule_to_close_timeout(Duration::ZERO),
18563 ActivityOptions::new().heartbeat_timeout(Duration::ZERO),
18564 ];
18565 for options in zero_timeout_cases {
18566 assert_eq!(
18567 options.validate().expect_err("zero timeout").kind,
18568 ActivityOptionsErrorKind::TimeoutNotPositive
18569 );
18570 }
18571
18572 let ordering_cases = [
18573 ActivityOptions::new()
18574 .heartbeat_timeout(Duration::from_secs(11))
18575 .start_to_close_timeout(Duration::from_secs(10)),
18576 ActivityOptions::new()
18577 .start_to_close_timeout(Duration::from_secs(31))
18578 .schedule_to_close_timeout(Duration::from_secs(30)),
18579 ActivityOptions::new()
18580 .schedule_to_start_timeout(Duration::from_secs(31))
18581 .schedule_to_close_timeout(Duration::from_secs(30)),
18582 ];
18583 for options in ordering_cases {
18584 assert_eq!(
18585 options.validate().expect_err("timeout order").kind,
18586 ActivityOptionsErrorKind::TimeoutOrder
18587 );
18588 }
18589
18590 assert_eq!(
18591 ActivityOptions::new()
18592 .start_to_close_timeout(Duration::from_secs(i64::MAX as u64 + 1))
18593 .validate()
18594 .expect_err("protocol integer overflow")
18595 .kind,
18596 ActivityOptionsErrorKind::TimeoutOverflow
18597 );
18598 }
18599
18600 #[test]
18601 fn replayed_activity_retry_history_completes_without_duplicate_schedule() {
18602 let ctx = workflow_context(completed_retry_activity_history());
18603 let mut call =
18604 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18605 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18606
18607 assert!(matches!(
18608 call.as_mut().poll(&mut task_context),
18609 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18610 ));
18611 assert!(ctx.take_commands().expect("commands").is_empty());
18612 ctx.ensure_history_consumed().expect("history consumed");
18613 }
18614
18615 #[test]
18616 fn duplicate_non_retryable_types_use_one_command_and_replay_representation() {
18617 let mut options = retry_activity_options();
18618 options
18619 .retry_policy
18620 .as_mut()
18621 .expect("retry policy")
18622 .non_retryable_error_types
18623 .extend([" PermanentError ".to_string(), "PermanentError".to_string()]);
18624
18625 let new_ctx = workflow_context(Vec::new());
18626 let mut new_call =
18627 Box::pin(new_ctx.activity_with_options("flaky", options.clone(), json!([])));
18628 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18629 assert!(matches!(
18630 new_call.as_mut().poll(&mut task_context),
18631 Poll::Pending
18632 ));
18633 let commands = new_ctx.take_commands().expect("commands");
18634 assert_eq!(commands.len(), 1);
18635 assert_eq!(
18636 commands[0]["retry_policy"]["non_retryable_error_types"],
18637 json!(["PermanentError"])
18638 );
18639
18640 let replay_ctx = workflow_context(completed_retry_activity_history());
18641 let mut replay_call =
18642 Box::pin(replay_ctx.activity_with_options("flaky", options, json!([])));
18643 assert!(matches!(
18644 replay_call.as_mut().poll(&mut task_context),
18645 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18646 ));
18647 assert!(replay_ctx.take_commands().expect("commands").is_empty());
18648 replay_ctx
18649 .ensure_history_consumed()
18650 .expect("history consumed");
18651 }
18652
18653 #[test]
18654 fn replayed_intermediate_retry_remains_pending_across_restarts() {
18655 let history = completed_retry_activity_history()
18656 .into_iter()
18657 .take(3)
18658 .collect::<Vec<_>>();
18659
18660 for _restart in 0..2 {
18661 let ctx = workflow_context(history.clone());
18662 let mut call =
18663 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18664 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18665 assert!(matches!(
18666 call.as_mut().poll(&mut task_context),
18667 Poll::Pending
18668 ));
18669 assert!(ctx.take_commands().expect("commands").is_empty());
18670 }
18671 }
18672
18673 #[test]
18674 fn replayed_activity_rejects_changed_queue_retry_and_every_timeout_field() {
18675 let mut changed_queue = retry_activity_options();
18676 changed_queue.task_queue = Some("different-queue".to_string());
18677
18678 let mut changed_max_attempts = retry_activity_options();
18679 let retry_policy = changed_max_attempts
18680 .retry_policy
18681 .as_mut()
18682 .expect("retry policy");
18683 retry_policy.max_attempts = Some(4);
18684
18685 let mut changed_backoff = retry_activity_options();
18686 let retry_policy = changed_backoff.retry_policy.as_mut().expect("retry policy");
18687 retry_policy.backoff = Some(ActivityBackoff::Explicit(vec![
18688 Duration::from_secs(3),
18689 Duration::from_secs(4),
18690 ]));
18691
18692 let mut changed_non_retryable_types = retry_activity_options();
18693 let retry_policy = changed_non_retryable_types
18694 .retry_policy
18695 .as_mut()
18696 .expect("retry policy");
18697 retry_policy.non_retryable_error_types = vec!["AnotherPermanentError".to_string()];
18698
18699 let mut changed_start_to_close = retry_activity_options();
18700 changed_start_to_close.start_to_close_timeout = Some(Duration::from_secs(31));
18701 let mut changed_schedule_to_start = retry_activity_options();
18702 changed_schedule_to_start.schedule_to_start_timeout = Some(Duration::from_secs(6));
18703 let mut changed_schedule_to_close = retry_activity_options();
18704 changed_schedule_to_close.schedule_to_close_timeout = Some(Duration::from_secs(91));
18705 let mut changed_heartbeat = retry_activity_options();
18706 changed_heartbeat.heartbeat_timeout = Some(Duration::from_secs(11));
18707
18708 let cases = [
18709 (changed_queue, "activity_task_queue_mismatch"),
18710 (changed_max_attempts, "activity_retry_policy_mismatch"),
18711 (changed_backoff, "activity_retry_policy_mismatch"),
18712 (
18713 changed_non_retryable_types,
18714 "activity_retry_policy_mismatch",
18715 ),
18716 (changed_start_to_close, "activity_retry_policy_mismatch"),
18717 (changed_schedule_to_start, "activity_retry_policy_mismatch"),
18718 (changed_schedule_to_close, "activity_retry_policy_mismatch"),
18719 (changed_heartbeat, "activity_retry_policy_mismatch"),
18720 ];
18721
18722 for (options, expected_reason) in cases {
18723 let ctx = workflow_context(completed_retry_activity_history());
18724 let mut call = Box::pin(ctx.activity_with_options("flaky", options, json!([])));
18725 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18726 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18727 call.as_mut().poll(&mut task_context)
18728 else {
18729 panic!("changed activity options must fail replay");
18730 };
18731 assert_eq!(failure.reason, expected_reason);
18732 assert_eq!(failure.sequence, Some(1));
18733 assert!(ctx.take_commands().expect("commands").is_empty());
18734 }
18735 }
18736
18737 #[test]
18738 fn replayed_activity_rejects_changed_execution_mode_and_snapshot_version() {
18739 let cases = [
18740 (
18741 "execution_mode",
18742 json!("local"),
18743 "activity_execution_mode_mismatch",
18744 ),
18745 (
18746 "snapshot_version",
18747 json!(2),
18748 "activity_retry_policy_mismatch",
18749 ),
18750 ];
18751
18752 for (field, value, expected_reason) in cases {
18753 let mut history = completed_retry_activity_history();
18754 let activity = history[0].payload["activity"]
18755 .as_object_mut()
18756 .expect("activity snapshot");
18757 if field == "execution_mode" {
18758 activity.insert(field.to_string(), value);
18759 } else {
18760 activity["retry_policy"]
18761 .as_object_mut()
18762 .expect("retry snapshot")
18763 .insert(field.to_string(), value);
18764 }
18765
18766 let ctx = workflow_context(history);
18767 let mut call =
18768 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18769 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18770 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18771 call.as_mut().poll(&mut task_context)
18772 else {
18773 panic!("changed {field} must fail replay");
18774 };
18775 assert_eq!(failure.reason, expected_reason);
18776 assert_eq!(failure.sequence, Some(1));
18777 assert!(ctx.take_commands().expect("commands").is_empty());
18778 }
18779 }
18780
18781 #[test]
18782 fn replayed_legacy_activity_treats_missing_option_snapshot_as_unknown() {
18783 let mut history = completed_retry_activity_history();
18784 let activity = history[0].payload["activity"]
18785 .as_object_mut()
18786 .expect("activity snapshot");
18787 activity.remove("execution_mode");
18788 activity.remove("retry_policy");
18789
18790 let mut current = retry_activity_options();
18791 current.start_to_close_timeout = Some(Duration::from_secs(45));
18792 current.schedule_to_start_timeout = Some(Duration::from_secs(8));
18793 current.schedule_to_close_timeout = Some(Duration::from_secs(120));
18794 current.heartbeat_timeout = Some(Duration::from_secs(12));
18795
18796 let ctx = workflow_context(history);
18797 let mut call = Box::pin(ctx.activity_with_options("flaky", current, json!([])));
18798 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18799 assert!(matches!(
18800 call.as_mut().poll(&mut task_context),
18801 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18802 ));
18803 assert!(ctx.take_commands().expect("commands").is_empty());
18804 ctx.ensure_history_consumed().expect("history consumed");
18805 }
18806
18807 #[test]
18808 fn terminal_activity_failed_after_start_returns_typed_failure() {
18809 let history = vec![
18810 history_event(
18811 "ActivityScheduled",
18812 json!({
18813 "sequence": 1,
18814 "activity_type": "flaky",
18815 "activity_execution_id": "act-terminal",
18816 "activity": {
18817 "id": "act-terminal",
18818 "sequence": 1,
18819 "type": "flaky",
18820 "queue": "critical-activities",
18821 "retry_policy": {
18822 "snapshot_version": 1,
18823 "max_attempts": 3,
18824 "backoff_seconds": [2, 4],
18825 "non_retryable_error_types": ["PermanentError"]
18826 }
18827 }
18828 }),
18829 ),
18830 history_event(
18831 "ActivityStarted",
18832 json!({
18833 "sequence": 1,
18834 "activity_type": "flaky",
18835 "activity_execution_id": "act-terminal",
18836 "activity_attempt_id": "attempt-1",
18837 "attempt_number": 1
18838 }),
18839 ),
18840 history_event(
18841 "ActivityFailed",
18842 json!({
18843 "sequence": 1,
18844 "activity_type": "flaky",
18845 "activity_execution_id": "act-terminal",
18846 "activity_attempt_id": "attempt-1",
18847 "attempt_number": 1,
18848 "failure_id": "failure-terminal",
18849 "failure_category": "activity",
18850 "exception_type": "PermanentError",
18851 "message": "cannot retry",
18852 "non_retryable": true
18853 }),
18854 ),
18855 ];
18856 let ctx = workflow_context(history);
18857 let mut call =
18858 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18859 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18860
18861 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18862 call.as_mut().poll(&mut task_context)
18863 else {
18864 panic!("terminal ActivityFailed must settle the activity future");
18865 };
18866 assert_eq!(failure.kind, ActivityFailureKind::Failed);
18867 assert_eq!(
18868 failure.activity_execution_id.as_deref(),
18869 Some("act-terminal")
18870 );
18871 assert_eq!(failure.exception_type.as_deref(), Some("PermanentError"));
18872 assert!(failure.non_retryable);
18873 assert!(ctx.take_commands().expect("commands").is_empty());
18874 ctx.ensure_history_consumed().expect("history consumed");
18875 }
18876
18877 #[test]
18878 fn activity_terminal_events_return_machine_readable_failures() {
18879 let cases = [
18880 (
18881 "ActivityFailed",
18882 json!({
18883 "sequence": 1,
18884 "activity_type": "charge-card",
18885 "activity_execution_id": "act-1",
18886 "activity_attempt_id": "attempt-2",
18887 "attempt_number": 2,
18888 "failure_id": "failure-1",
18889 "failure_category": "activity",
18890 "exception_type": "PaymentDeclined",
18891 "exception_class": "payments.PaymentDeclined",
18892 "message": "card declined",
18893 "non_retryable": true
18894 }),
18895 ActivityFailureKind::Failed,
18896 "activity",
18897 ),
18898 (
18899 "ActivityCancelled",
18900 json!({
18901 "sequence": 1,
18902 "activity_type": "charge-card",
18903 "activity_execution_id": "act-1",
18904 "activity_attempt_id": "attempt-1"
18905 }),
18906 ActivityFailureKind::Cancelled,
18907 "cancelled",
18908 ),
18909 ];
18910
18911 for (event_type, payload, expected_kind, expected_reason) in cases {
18912 let ctx = workflow_context(vec![history_event(event_type, payload)]);
18913 let mut call = Box::pin(ctx.activity("charge-card", json!([])));
18914 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18915 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18916 call.as_mut().poll(&mut task_context)
18917 else {
18918 panic!("expected terminal activity failure");
18919 };
18920 assert_eq!(failure.kind, expected_kind);
18921 assert_eq!(failure.reason, expected_reason);
18922 assert_eq!(failure.activity_execution_id.as_deref(), Some("act-1"));
18923 assert_eq!(failure.activity_type.as_deref(), Some("charge-card"));
18924 }
18925 }
18926
18927 #[test]
18928 fn every_activity_timeout_class_is_typed() {
18929 for timeout_kind in [
18930 "start_to_close",
18931 "schedule_to_start",
18932 "schedule_to_close",
18933 "heartbeat",
18934 ] {
18935 let ctx = workflow_context(vec![history_event(
18936 "ActivityTimedOut",
18937 json!({
18938 "sequence": 1,
18939 "activity_type": "slow",
18940 "activity_execution_id": "act-timeout",
18941 "activity_attempt_id": "attempt-timeout",
18942 "failure_category": "timeout",
18943 "timeout_kind": timeout_kind,
18944 "message": "deadline expired"
18945 }),
18946 )]);
18947 let mut call = Box::pin(ctx.activity("slow", json!([])));
18948 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18949 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18950 call.as_mut().poll(&mut task_context)
18951 else {
18952 panic!("expected timeout failure");
18953 };
18954 assert_eq!(failure.kind, ActivityFailureKind::TimedOut);
18955 assert_eq!(failure.reason, timeout_kind);
18956 assert_eq!(failure.timeout_kind.as_deref(), Some(timeout_kind));
18957 assert_eq!(failure.failure_category.as_deref(), Some("timeout"));
18958 }
18959 }
18960
18961 #[test]
18962 fn workflow_sleep_emits_one_durable_timer_and_rounds_up() {
18963 let ctx = workflow_context(Vec::new());
18964 let mut sleep = Box::pin(ctx.sleep(Duration::from_millis(1_001)));
18965 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18966
18967 assert!(matches!(
18968 sleep.as_mut().poll(&mut task_context),
18969 Poll::Pending
18970 ));
18971 assert!(matches!(
18972 sleep.as_mut().poll(&mut task_context),
18973 Poll::Pending
18974 ));
18975
18976 let commands = ctx.take_commands().expect("timer command");
18977 assert_eq!(
18978 commands,
18979 vec![json!({
18980 "type": "start_timer",
18981 "delay_seconds": 2,
18982 })]
18983 );
18984 }
18985
18986 #[test]
18987 fn workflow_sleep_replays_matching_schedule_and_fire_without_a_command() {
18988 let history = vec![
18989 history_event(
18990 "TimerScheduled",
18991 json!({
18992 "sequence": 1,
18993 "timer_id": "timer-1",
18994 "delay_seconds": 5,
18995 "fire_at": "2026-07-11T12:00:05Z",
18996 }),
18997 ),
18998 history_event(
18999 "TimerFired",
19000 json!({
19001 "sequence": 1,
19002 "timer_id": "timer-1",
19003 "delay_seconds": 5,
19004 "fire_at": "2026-07-11T12:00:05Z",
19005 "fired_at": "2026-07-11T12:00:05Z",
19006 }),
19007 ),
19008 ];
19009
19010 for _restart in 0..2 {
19011 let ctx = workflow_context(history.clone());
19012 let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(5)));
19013 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19014 assert!(matches!(
19015 sleep.as_mut().poll(&mut task_context),
19016 Poll::Ready(Ok(()))
19017 ));
19018 assert!(ctx.take_commands().expect("commands").is_empty());
19019 ctx.ensure_history_consumed().expect("history consumed");
19020 }
19021 }
19022
19023 #[test]
19024 fn workflow_sleep_rejects_changed_delay_during_replay() {
19025 let ctx = workflow_context(vec![
19026 history_event(
19027 "TimerScheduled",
19028 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19029 ),
19030 history_event(
19031 "TimerFired",
19032 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19033 ),
19034 ]);
19035 let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(500)));
19036 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19037
19038 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
19039 sleep.as_mut().poll(&mut task_context)
19040 else {
19041 panic!("changed timer delay must be rejected");
19042 };
19043 assert_eq!(failure.reason, "timer_delay_mismatch");
19044 assert_eq!(failure.sequence, Some(1));
19045 }
19046
19047 #[test]
19048 fn workflow_condition_wait_emits_published_identity_and_timeout_contract() {
19049 let ctx = workflow_context(Vec::new());
19050 let mut wait = Box::pin(
19051 ctx.wait_condition(
19052 ConditionWaitOptions::new("approval.ready", "sha256:approval-v1")
19053 .timeout(Duration::from_millis(60_001)),
19054 || Ok(false),
19055 ),
19056 );
19057 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19058
19059 assert!(matches!(
19060 wait.as_mut().poll(&mut task_context),
19061 Poll::Pending
19062 ));
19063 assert!(matches!(
19064 wait.as_mut().poll(&mut task_context),
19065 Poll::Pending
19066 ));
19067 assert_eq!(
19068 ctx.take_commands().expect("condition command"),
19069 vec![json!({
19070 "type": "open_condition_wait",
19071 "condition_wait_occurrence_id": "rust:condition-wait:0",
19072 "condition_key": "approval.ready",
19073 "condition_definition_fingerprint": "sha256:approval-v1",
19074 "timeout_seconds": 61,
19075 })]
19076 );
19077 }
19078
19079 #[test]
19080 fn workflow_condition_wait_returns_explicit_immediate_results_without_commands() {
19081 let ctx = workflow_context(Vec::new());
19082 let mut satisfied = Box::pin(wait_condition!(ctx, "already-ready", || Ok(true)));
19083 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19084 assert!(matches!(
19085 satisfied.as_mut().poll(&mut task_context),
19086 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19087 ));
19088
19089 let mut timed_out = Box::pin(wait_condition!(
19090 ctx,
19091 "no-wait",
19092 timeout: Duration::ZERO,
19093 || Ok(false),
19094 ));
19095 assert!(matches!(
19096 timed_out.as_mut().poll(&mut task_context),
19097 Poll::Ready(Ok(ConditionWaitResult::TimedOut))
19098 ));
19099 assert!(ctx.take_commands().expect("commands").is_empty());
19100 }
19101
19102 #[test]
19103 fn signal_and_update_history_reevaluate_open_conditions_after_restart() {
19104 let signal_history = vec![
19105 history_event(
19106 "ConditionWaitOpened",
19107 json!({
19108 "sequence": 4,
19109 "condition_wait_id": "condition:4",
19110 "condition_wait_occurrence_id": "rust:condition-wait:0",
19111 "condition_key": "approval",
19112 "condition_definition_fingerprint": "sha256:approval-v1",
19113 "timeout_seconds": 30,
19114 }),
19115 ),
19116 history_event(
19117 "SignalReceived",
19118 json!({
19119 "workflow_sequence": 4,
19120 "signal_name": "approve",
19121 "arguments": fixture_envelope(json!(["Ada"])),
19122 }),
19123 ),
19124 ];
19125 for _worker_before_or_after_restart in 0..2 {
19126 let ctx = workflow_context(signal_history.clone());
19127 let predicate_ctx = ctx.clone();
19128 let mut wait = Box::pin(
19129 ctx.wait_condition(
19130 ConditionWaitOptions::new("approval", "sha256:approval-v1")
19131 .timeout(Duration::from_secs(30)),
19132 move || Ok(!predicate_ctx.signals("approve")?.is_empty()),
19133 ),
19134 );
19135 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19136 assert!(matches!(
19137 wait.as_mut().poll(&mut task_context),
19138 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19139 ));
19140 assert!(ctx.take_commands().expect("commands").is_empty());
19141 ctx.ensure_history_consumed().expect("condition consumed");
19142 }
19143
19144 let update_history = vec![
19145 history_event(
19146 "ConditionWaitOpened",
19147 json!({
19148 "sequence": 7,
19149 "condition_wait_id": "condition:7",
19150 "condition_wait_occurrence_id": "rust:condition-wait:0",
19151 "condition_key": "update-approval",
19152 "condition_definition_fingerprint": "sha256:update-approval-v1",
19153 }),
19154 ),
19155 history_event(
19156 "UpdateApplied",
19157 json!({
19158 "sequence": 7,
19159 "update_id": "update-1",
19160 "update_name": "approve",
19161 "arguments": fixture_envelope(json!([true])),
19162 }),
19163 ),
19164 ];
19165 let ctx = workflow_context(update_history);
19166 let predicate_ctx = ctx.clone();
19167 let mut wait = Box::pin(ctx.wait_condition(
19168 ConditionWaitOptions::new("update-approval", "sha256:update-approval-v1"),
19169 move || {
19170 Ok(predicate_ctx
19171 .updates("approve")?
19172 .first()
19173 .and_then(|arguments| arguments.first())
19174 .and_then(Value::as_bool)
19175 == Some(true))
19176 },
19177 ));
19178 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19179 assert!(matches!(
19180 wait.as_mut().poll(&mut task_context),
19181 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19182 ));
19183 assert!(ctx.take_commands().expect("commands").is_empty());
19184 ctx.ensure_history_consumed().expect("condition consumed");
19185 }
19186
19187 #[test]
19188 fn condition_wait_preserves_open_satisfied_and_timed_out_replay_states() {
19189 let open_history = vec![
19190 history_event(
19191 "ConditionWaitOpened",
19192 json!({
19193 "sequence": 3,
19194 "condition_wait_id": "condition:3",
19195 "condition_wait_occurrence_id": "rust:condition-wait:0",
19196 "condition_key": "two-votes",
19197 "condition_definition_fingerprint": "sha256:two-votes-v1",
19198 "timeout_seconds": 120,
19199 }),
19200 ),
19201 history_event(
19202 "SignalReceived",
19203 json!({
19204 "workflow_sequence": 3,
19205 "signal_name": "vote",
19206 "arguments": fixture_envelope(json!(["first"])),
19207 }),
19208 ),
19209 ];
19210 for _worker_before_or_after_restart in 0..2 {
19211 let ctx = workflow_context(open_history.clone());
19212 let predicate_ctx = ctx.clone();
19213 let mut wait = Box::pin(
19214 ctx.wait_condition(
19215 ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1")
19216 .timeout(Duration::from_secs(120)),
19217 move || Ok(predicate_ctx.signals("vote")?.len() >= 2),
19218 ),
19219 );
19220 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19221 assert!(matches!(
19222 wait.as_mut().poll(&mut task_context),
19223 Poll::Pending
19224 ));
19225 assert_eq!(
19226 ctx.take_commands().expect("reopened condition"),
19227 vec![json!({
19228 "type": "open_condition_wait",
19229 "condition_wait_occurrence_id": "rust:condition-wait:0",
19230 "condition_key": "two-votes",
19231 "condition_definition_fingerprint": "sha256:two-votes-v1",
19232 "timeout_seconds": 120,
19233 })]
19234 );
19235 }
19236
19237 let satisfied_ctx = workflow_context(vec![
19238 history_event(
19239 "ConditionWaitOpened",
19240 json!({
19241 "sequence": 5,
19242 "condition_wait_id": "condition:5",
19243 "condition_wait_occurrence_id": "rust:condition-wait:0",
19244 "condition_key": "approval",
19245 "condition_definition_fingerprint": "sha256:approval-v1",
19246 }),
19247 ),
19248 history_event(
19249 "ConditionWaitSatisfied",
19250 json!({
19251 "sequence": 5,
19252 "condition_wait_id": "condition:5",
19253 "condition_wait_occurrence_id": "rust:condition-wait:0",
19254 "condition_key": "approval",
19255 "condition_definition_fingerprint": "sha256:approval-v1",
19256 }),
19257 ),
19258 ]);
19259 let mut satisfied = Box::pin(satisfied_ctx.wait_condition(
19260 ConditionWaitOptions::new("approval", "sha256:approval-v1"),
19261 || Ok(false),
19262 ));
19263 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19264 assert!(matches!(
19265 satisfied.as_mut().poll(&mut task_context),
19266 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19267 ));
19268
19269 let timed_out_ctx = workflow_context(vec![
19270 history_event(
19271 "ConditionWaitOpened",
19272 json!({
19273 "sequence": 8,
19274 "condition_wait_id": "condition:8",
19275 "condition_wait_occurrence_id": "rust:condition-wait:0",
19276 "condition_key": "approval-timeout",
19277 "condition_definition_fingerprint": "sha256:approval-timeout-v1",
19278 "timeout_seconds": 5,
19279 }),
19280 ),
19281 history_event(
19282 "TimerScheduled",
19283 json!({
19284 "sequence": 9,
19285 "timer_id": "condition-timer:9",
19286 "timer_kind": "condition_timeout",
19287 "condition_wait_id": "condition:8",
19288 "delay_seconds": 5,
19289 }),
19290 ),
19291 history_event(
19292 "TimerFired",
19293 json!({
19294 "sequence": 9,
19295 "timer_id": "condition-timer:9",
19296 "timer_kind": "condition_timeout",
19297 "condition_wait_id": "condition:8",
19298 "delay_seconds": 5,
19299 }),
19300 ),
19301 ]);
19302 let mut timed_out = Box::pin(
19303 timed_out_ctx.wait_condition(
19304 ConditionWaitOptions::new("approval-timeout", "sha256:approval-timeout-v1")
19305 .timeout(Duration::from_secs(5)),
19306 || Ok(true),
19307 ),
19308 );
19309 assert!(matches!(
19310 timed_out.as_mut().poll(&mut task_context),
19311 Poll::Ready(Ok(ConditionWaitResult::TimedOut))
19312 ));
19313 }
19314
19315 #[test]
19316 fn condition_wait_replays_repeated_physical_opens_as_one_logical_wait() {
19317 let history = vec![
19318 history_event(
19319 "ConditionWaitOpened",
19320 json!({
19321 "sequence": 3,
19322 "condition_wait_id": "condition:3",
19323 "condition_wait_occurrence_id": "rust:condition-wait:0",
19324 "condition_key": "two-votes",
19325 "condition_definition_fingerprint": "sha256:two-votes-v1",
19326 }),
19327 ),
19328 history_event(
19329 "SignalReceived",
19330 json!({
19331 "workflow_sequence": 3,
19332 "signal_name": "vote",
19333 "arguments": fixture_envelope(json!(["first"])),
19334 }),
19335 ),
19336 history_event(
19337 "ConditionWaitSatisfied",
19338 json!({
19339 "sequence": 3,
19340 "condition_wait_id": "condition:3",
19341 "condition_wait_occurrence_id": "rust:condition-wait:0",
19342 "condition_key": "two-votes",
19343 "condition_definition_fingerprint": "sha256:two-votes-v1",
19344 }),
19345 ),
19346 history_event(
19347 "ConditionWaitOpened",
19348 json!({
19349 "sequence": 5,
19350 "condition_wait_id": "condition:5",
19351 "condition_wait_occurrence_id": "rust:condition-wait:0",
19352 "condition_key": "two-votes",
19353 "condition_definition_fingerprint": "sha256:two-votes-v1",
19354 }),
19355 ),
19356 history_event(
19357 "SignalReceived",
19358 json!({
19359 "workflow_sequence": 5,
19360 "signal_name": "vote",
19361 "arguments": fixture_envelope(json!(["second"])),
19362 }),
19363 ),
19364 history_event(
19365 "ConditionWaitSatisfied",
19366 json!({
19367 "sequence": 5,
19368 "condition_wait_id": "condition:5",
19369 "condition_wait_occurrence_id": "rust:condition-wait:0",
19370 "condition_key": "two-votes",
19371 "condition_definition_fingerprint": "sha256:two-votes-v1",
19372 }),
19373 ),
19374 ];
19375 for _cold_worker_or_restart in 0..2 {
19376 let ctx = workflow_context(history.clone());
19377 let predicate_ctx = ctx.clone();
19378 let mut wait = Box::pin(ctx.wait_condition(
19379 ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1"),
19380 move || Ok(predicate_ctx.signals("vote")?.len() >= 2),
19381 ));
19382 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19383
19384 assert!(matches!(
19385 wait.as_mut().poll(&mut task_context),
19386 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19387 ));
19388 assert!(ctx.take_commands().expect("commands").is_empty());
19389 ctx.ensure_history_consumed()
19390 .expect("every physical wait-open is consumed");
19391 }
19392 }
19393
19394 #[test]
19395 fn condition_wait_replays_update_driven_physical_opens_as_one_occurrence() {
19396 let history = vec![
19397 history_event(
19398 "ConditionWaitOpened",
19399 json!({
19400 "sequence": 3,
19401 "condition_wait_id": "condition:3",
19402 "condition_wait_occurrence_id": "rust:condition-wait:0",
19403 "condition_key": "approved",
19404 "condition_definition_fingerprint": "sha256:approved-v1",
19405 }),
19406 ),
19407 history_event(
19408 "UpdateApplied",
19409 json!({
19410 "sequence": 3,
19411 "update_id": "update-1",
19412 "update_name": "approve",
19413 "arguments": fixture_envelope(json!([false])),
19414 }),
19415 ),
19416 history_event(
19417 "ConditionWaitOpened",
19418 json!({
19419 "sequence": 5,
19420 "condition_wait_id": "condition:5",
19421 "condition_wait_occurrence_id": "rust:condition-wait:0",
19422 "condition_key": "approved",
19423 "condition_definition_fingerprint": "sha256:approved-v1",
19424 }),
19425 ),
19426 history_event(
19427 "UpdateApplied",
19428 json!({
19429 "sequence": 5,
19430 "update_id": "update-2",
19431 "update_name": "approve",
19432 "arguments": fixture_envelope(json!([true])),
19433 }),
19434 ),
19435 ];
19436
19437 for _cold_worker_or_restart in 0..2 {
19438 let ctx = workflow_context(history.clone());
19439 let predicate_ctx = ctx.clone();
19440 let mut wait = Box::pin(ctx.wait_condition(
19441 ConditionWaitOptions::new("approved", "sha256:approved-v1"),
19442 move || {
19443 Ok(predicate_ctx
19444 .updates("approve")?
19445 .last()
19446 .and_then(|arguments| arguments.first())
19447 .and_then(Value::as_bool)
19448 == Some(true))
19449 },
19450 ));
19451 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19452
19453 assert!(matches!(
19454 wait.as_mut().poll(&mut task_context),
19455 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19456 ));
19457 assert!(ctx.take_commands().expect("commands").is_empty());
19458 ctx.ensure_history_consumed()
19459 .expect("every update-driven reopen is consumed");
19460 }
19461 }
19462
19463 #[test]
19464 fn condition_wait_replay_keeps_every_adjacent_authored_occurrence_distinct() {
19465 for (first_key, first_fingerprint, second_key, second_fingerprint) in [
19466 ("shared", "sha256:first", "shared", "sha256:second"),
19467 ("first", "sha256:shared", "second", "sha256:shared"),
19468 ("shared", "sha256:shared", "shared", "sha256:shared"),
19469 ("first", "sha256:first", "second", "sha256:second"),
19470 ] {
19471 let history = vec![
19472 history_event(
19473 "ConditionWaitOpened",
19474 json!({
19475 "sequence": 3,
19476 "condition_wait_id": "condition:3",
19477 "condition_wait_occurrence_id": "rust:condition-wait:0",
19478 "condition_key": first_key,
19479 "condition_definition_fingerprint": first_fingerprint,
19480 }),
19481 ),
19482 history_event(
19483 "ConditionWaitSatisfied",
19484 json!({
19485 "sequence": 3,
19486 "condition_wait_id": "condition:3",
19487 "condition_wait_occurrence_id": "rust:condition-wait:0",
19488 "condition_key": first_key,
19489 "condition_definition_fingerprint": first_fingerprint,
19490 }),
19491 ),
19492 history_event(
19493 "ConditionWaitOpened",
19494 json!({
19495 "sequence": 4,
19496 "condition_wait_id": "condition:4",
19497 "condition_wait_occurrence_id": "rust:condition-wait:1",
19498 "condition_key": second_key,
19499 "condition_definition_fingerprint": second_fingerprint,
19500 }),
19501 ),
19502 history_event(
19503 "ConditionWaitSatisfied",
19504 json!({
19505 "sequence": 4,
19506 "condition_wait_id": "condition:4",
19507 "condition_wait_occurrence_id": "rust:condition-wait:1",
19508 "condition_key": second_key,
19509 "condition_definition_fingerprint": second_fingerprint,
19510 }),
19511 ),
19512 ];
19513 for _cold_worker_or_restart in 0..2 {
19514 let ctx = workflow_context(history.clone());
19515 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19516 let mut first = Box::pin(ctx.wait_condition(
19517 ConditionWaitOptions::new(first_key, first_fingerprint),
19518 || Ok(false),
19519 ));
19520 assert!(matches!(
19521 first.as_mut().poll(&mut task_context),
19522 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19523 ));
19524
19525 let mut second = Box::pin(ctx.wait_condition(
19526 ConditionWaitOptions::new(second_key, second_fingerprint),
19527 || Ok(false),
19528 ));
19529 assert!(matches!(
19530 second.as_mut().poll(&mut task_context),
19531 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19532 ));
19533 assert!(ctx.take_commands().expect("commands").is_empty());
19534 ctx.ensure_history_consumed()
19535 .expect("each authored wait consumes one occurrence");
19536 }
19537 }
19538 }
19539
19540 #[test]
19541 fn cold_workers_replay_adjacent_condition_waits_from_one_loop_call_site() {
19542 fn worker() -> Worker {
19543 let client = Client::new("http://127.0.0.1:8080").expect("client");
19544 let mut worker = Worker::new(client, "rust-workers");
19545 worker.register_workflow("rust.condition-loop", |ctx, _input| async move {
19546 let mut outcomes = Vec::new();
19547 for _ in 0..2 {
19548 outcomes.push(
19549 ctx.wait_condition(
19550 ConditionWaitOptions::new("shared", "sha256:shared"),
19551 || Ok(false),
19552 )
19553 .await?,
19554 );
19555 }
19556 Ok(json!(outcomes))
19557 });
19558 worker
19559 }
19560
19561 let task = workflow_task(
19562 "rust.condition-loop",
19563 vec![
19564 history_event(
19565 "ConditionWaitOpened",
19566 json!({
19567 "sequence": 1,
19568 "condition_wait_id": "condition:1",
19569 "condition_wait_occurrence_id": "rust:condition-wait:0",
19570 "condition_key": "shared",
19571 "condition_definition_fingerprint": "sha256:shared",
19572 }),
19573 ),
19574 history_event(
19575 "ConditionWaitSatisfied",
19576 json!({
19577 "sequence": 1,
19578 "condition_wait_id": "condition:1",
19579 "condition_wait_occurrence_id": "rust:condition-wait:0",
19580 "condition_key": "shared",
19581 "condition_definition_fingerprint": "sha256:shared",
19582 }),
19583 ),
19584 history_event(
19585 "ConditionWaitOpened",
19586 json!({
19587 "sequence": 2,
19588 "condition_wait_id": "condition:2",
19589 "condition_wait_occurrence_id": "rust:condition-wait:1",
19590 "condition_key": "shared",
19591 "condition_definition_fingerprint": "sha256:shared",
19592 }),
19593 ),
19594 history_event(
19595 "ConditionWaitSatisfied",
19596 json!({
19597 "sequence": 2,
19598 "condition_wait_id": "condition:2",
19599 "condition_wait_occurrence_id": "rust:condition-wait:1",
19600 "condition_key": "shared",
19601 "condition_definition_fingerprint": "sha256:shared",
19602 }),
19603 ),
19604 ],
19605 DEFAULT_CODEC,
19606 );
19607
19608 for _cold_worker_or_restart in 0..2 {
19609 let commands = worker()
19610 .execute_workflow_task(task.clone())
19611 .expect("adjacent loop waits replay deterministically");
19612 assert_eq!(commands.len(), 1);
19613 assert_eq!(commands[0]["type"], "complete_workflow");
19614 assert_eq!(
19615 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
19616 json!(["satisfied", "satisfied"])
19617 );
19618 }
19619 }
19620
19621 #[test]
19622 fn condition_wait_replay_rejects_identity_predicate_and_timeout_changes() {
19623 let history = vec![history_event(
19624 "ConditionWaitOpened",
19625 json!({
19626 "sequence": 12,
19627 "condition_wait_id": "condition:12",
19628 "condition_wait_occurrence_id": "rust:condition-wait:0",
19629 "condition_key": "approval",
19630 "condition_definition_fingerprint": "sha256:approval-v1",
19631 "timeout_seconds": 30,
19632 }),
19633 )];
19634 for (options, expected_reason) in [
19635 (
19636 ConditionWaitOptions::new("changed", "sha256:approval-v1")
19637 .timeout(Duration::from_secs(30)),
19638 "condition_wait_key_mismatch",
19639 ),
19640 (
19641 ConditionWaitOptions::new("approval", "sha256:approval-v2")
19642 .timeout(Duration::from_secs(30)),
19643 "condition_wait_predicate_mismatch",
19644 ),
19645 (
19646 ConditionWaitOptions::new("approval", "sha256:approval-v1")
19647 .timeout(Duration::from_secs(29)),
19648 "condition_wait_timeout_mismatch",
19649 ),
19650 ] {
19651 let ctx = workflow_context(history.clone());
19652 let mut wait = Box::pin(ctx.wait_condition(options, || Ok(false)));
19653 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19654 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
19655 wait.as_mut().poll(&mut task_context)
19656 else {
19657 panic!("changed condition definition must fail replay");
19658 };
19659 assert_eq!(failure.reason, expected_reason);
19660 assert_eq!(failure.sequence, Some(12));
19661 }
19662 }
19663
19664 #[test]
19665 fn condition_wait_history_requires_the_canonical_predicate_fingerprint() {
19666 let error = WorkflowState::new(
19667 vec![history_event(
19668 "ConditionWaitOpened",
19669 json!({
19670 "sequence": 12,
19671 "condition_wait_id": "condition:12",
19672 "condition_wait_occurrence_id": "rust:condition-wait:0",
19673 "condition_key": "approval",
19674 }),
19675 )],
19676 "rust-workers".to_string(),
19677 DEFAULT_CODEC.to_string(),
19678 None,
19679 )
19680 .expect_err("condition history without a predicate fingerprint must fail");
19681
19682 assert!(matches!(
19683 error,
19684 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19685 if reason == "condition_wait_predicate_fingerprint_missing"
19686 ));
19687 }
19688
19689 #[test]
19690 fn condition_wait_history_requires_authored_occurrence_identity() {
19691 let error = WorkflowState::new(
19692 vec![history_event(
19693 "ConditionWaitOpened",
19694 json!({
19695 "sequence": 12,
19696 "condition_wait_id": "condition:12",
19697 "condition_key": "approval",
19698 "condition_definition_fingerprint": "sha256:approval-v1",
19699 }),
19700 )],
19701 "rust-workers".to_string(),
19702 DEFAULT_CODEC.to_string(),
19703 None,
19704 )
19705 .expect_err("condition history without occurrence identity must fail");
19706
19707 assert!(matches!(
19708 error,
19709 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19710 if reason == "condition_wait_occurrence_id_missing"
19711 ));
19712 }
19713
19714 #[test]
19715 fn typed_search_attribute_updates_validate_emit_and_replay() {
19716 let update = SearchAttributeUpdate::new()
19717 .keyword("OrderStatus", " waiting ")
19718 .expect("keyword")
19719 .int("Attempt", 3)
19720 .expect("int")
19721 .bool("Escalated", false)
19722 .expect("bool")
19723 .keyword_list("Regions", ["us-east", "eu-west"])
19724 .expect("list")
19725 .datetime("UpdatedAt", "2026-08-22T04:00:00Z")
19726 .expect("datetime")
19727 .delete("LegacyStatus")
19728 .expect("delete");
19729 let ctx = workflow_context(Vec::new());
19730 ctx.upsert_search_attributes(update.clone())
19731 .expect("typed update");
19732 assert_eq!(
19733 ctx.take_commands().expect("search-attribute command"),
19734 vec![json!({
19735 "type": "upsert_search_attributes",
19736 "attributes": {
19737 "Attempt": 3,
19738 "Escalated": false,
19739 "LegacyStatus": null,
19740 "OrderStatus": "waiting",
19741 "Regions": ["us-east", "eu-west"],
19742 "UpdatedAt": "2026-08-22T04:00:00Z",
19743 },
19744 "attribute_types": {
19745 "Attempt": "int",
19746 "Escalated": "bool",
19747 "OrderStatus": "keyword",
19748 "Regions": "keyword_list",
19749 "UpdatedAt": "datetime",
19750 },
19751 })]
19752 );
19753
19754 let replay = workflow_context(vec![history_event(
19755 "SearchAttributesUpserted",
19756 json!({
19757 "sequence": 6,
19758 "attributes": {
19759 "Attempt": 3,
19760 "Escalated": false,
19761 "LegacyStatus": null,
19762 "OrderStatus": "waiting",
19763 "Regions": ["us-east", "eu-west"],
19764 "UpdatedAt": "2026-08-22T04:00:00Z",
19765 },
19766 "attribute_types": {
19767 "Attempt": "int",
19768 "Escalated": "bool",
19769 "OrderStatus": "keyword",
19770 "Regions": "keyword_list",
19771 "UpdatedAt": "datetime",
19772 },
19773 "merged": {},
19774 }),
19775 )]);
19776 replay
19777 .upsert_search_attributes(update)
19778 .expect("matching update replays");
19779 assert!(replay.take_commands().expect("commands").is_empty());
19780 replay.ensure_history_consumed().expect("history consumed");
19781
19782 let type_drift = workflow_context(vec![history_event(
19783 "SearchAttributesUpserted",
19784 json!({
19785 "sequence": 7,
19786 "attributes": {"OrderStatus": "waiting"},
19787 "attribute_types": {"OrderStatus": "keyword"},
19788 "merged": {"OrderStatus": "waiting"},
19789 }),
19790 )]);
19791 let error = type_drift
19792 .upsert_search_attributes(
19793 SearchAttributeUpdate::new()
19794 .string("OrderStatus", "waiting")
19795 .expect("string update"),
19796 )
19797 .expect_err("same JSON value with a changed type must fail replay");
19798 assert!(matches!(
19799 error,
19800 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19801 if reason == "search_attribute_type_mismatch"
19802 ));
19803
19804 let malformed_types = WorkflowState::new(
19805 vec![history_event(
19806 "SearchAttributesUpserted",
19807 json!({
19808 "sequence": 8,
19809 "attributes": {"OrderStatus": "waiting"},
19810 "attribute_types": {"OrderStatus": "unsupported"},
19811 "merged": {"OrderStatus": "waiting"},
19812 }),
19813 )],
19814 "rust-workers".to_string(),
19815 DEFAULT_CODEC.to_string(),
19816 None,
19817 )
19818 .expect_err("unsupported search-attribute type metadata must fail");
19819 assert!(matches!(
19820 malformed_types,
19821 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19822 if reason == "search_attribute_types_malformed"
19823 ));
19824
19825 assert!(matches!(
19826 SearchAttributeUpdate::new().keyword("bad key", "value"),
19827 Err(SearchAttributeUpdateError::InvalidKey(_))
19828 ));
19829 assert!(matches!(
19830 SearchAttributeUpdate::new().float("Ratio", f64::NAN),
19831 Err(SearchAttributeUpdateError::NonFiniteFloat(_))
19832 ));
19833 assert!(matches!(
19834 SearchAttributeUpdate::new().keyword("UnicodeKeyword", "é".repeat(128)),
19835 Err(SearchAttributeUpdateError::ValueTooLong { .. })
19836 ));
19837 assert!(matches!(
19838 SearchAttributeUpdate::new().datetime("UpdatedAt", "2026-02-30T04:00:00Z"),
19839 Err(SearchAttributeUpdateError::InvalidDateTime(_))
19840 ));
19841 assert!(matches!(
19842 workflow_context(Vec::new()).upsert_search_attributes(SearchAttributeUpdate::new()),
19843 Err(Error::InvalidSearchAttributeUpdate(
19844 SearchAttributeUpdateError::Empty
19845 ))
19846 ));
19847 }
19848
19849 #[test]
19850 fn typed_search_attribute_text_uses_the_runtime_byte_limit() {
19851 let ascii = "a".repeat(MAX_SEARCH_ATTRIBUTE_STRING_LENGTH);
19852 let utf8 = "é".repeat(MAX_SEARCH_ATTRIBUTE_STRING_LENGTH / 2);
19853
19854 assert!(SearchAttributeUpdate::new()
19855 .string("AsciiDescription", ascii)
19856 .is_ok());
19857 assert!(SearchAttributeUpdate::new()
19858 .string("Utf8Description", utf8)
19859 .is_ok());
19860 assert!(matches!(
19861 SearchAttributeUpdate::new().string(
19862 "TooLongDescription",
19863 "é".repeat((MAX_SEARCH_ATTRIBUTE_STRING_LENGTH / 2) + 1),
19864 ),
19865 Err(SearchAttributeUpdateError::ValueTooLong {
19866 kind: "string",
19867 limit: MAX_SEARCH_ATTRIBUTE_STRING_LENGTH,
19868 ..
19869 })
19870 ));
19871 }
19872
19873 #[test]
19874 fn workflow_history_rejects_unpaired_or_mismatched_timer_events() {
19875 let lone_fire = WorkflowState::new(
19876 vec![history_event(
19877 "TimerFired",
19878 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19879 )],
19880 "rust-workers".to_string(),
19881 DEFAULT_CODEC.to_string(),
19882 None,
19883 )
19884 .expect_err("TimerFired requires TimerScheduled");
19885 assert!(matches!(
19886 lone_fire,
19887 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19888 if reason == "timer_schedule_missing_or_duplicate"
19889 ));
19890
19891 let wrong_identity = WorkflowState::new(
19892 vec![
19893 history_event(
19894 "TimerScheduled",
19895 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19896 ),
19897 history_event(
19898 "TimerFired",
19899 json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
19900 ),
19901 ],
19902 "rust-workers".to_string(),
19903 DEFAULT_CODEC.to_string(),
19904 None,
19905 )
19906 .expect_err("fire must match scheduled timer identity");
19907 assert!(matches!(
19908 wrong_identity,
19909 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19910 if reason == "timer_identity_mismatch"
19911 ));
19912
19913 let duplicate_fire = WorkflowState::new(
19914 vec![
19915 history_event(
19916 "TimerScheduled",
19917 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19918 ),
19919 history_event(
19920 "TimerFired",
19921 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19922 ),
19923 history_event(
19924 "TimerFired",
19925 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19926 ),
19927 ],
19928 "rust-workers".to_string(),
19929 DEFAULT_CODEC.to_string(),
19930 None,
19931 )
19932 .expect_err("a durable timer cannot fire twice");
19933 assert!(matches!(
19934 duplicate_fire,
19935 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19936 if reason == "duplicate_timer_fire"
19937 ));
19938
19939 let wrong_fired_delay = WorkflowState::new(
19940 vec![
19941 history_event(
19942 "TimerScheduled",
19943 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19944 ),
19945 history_event(
19946 "TimerFired",
19947 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 6}),
19948 ),
19949 ],
19950 "rust-workers".to_string(),
19951 DEFAULT_CODEC.to_string(),
19952 None,
19953 )
19954 .expect_err("timer schedule and fire delays must agree");
19955 assert!(matches!(
19956 wrong_fired_delay,
19957 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19958 if reason == "timer_history_delay_mismatch"
19959 ));
19960 }
19961
19962 #[test]
19963 fn replay_rejects_activity_moved_before_recorded_timer() {
19964 let ctx = workflow_context(vec![
19965 history_event(
19966 "TimerScheduled",
19967 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19968 ),
19969 history_event(
19970 "TimerFired",
19971 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19972 ),
19973 history_event(
19974 "ActivityCompleted",
19975 json!({
19976 "sequence": 2,
19977 "activity_type": "after-timer",
19978 "payload_codec": DEFAULT_CODEC,
19979 "result": fixture_envelope(json!("done")),
19980 }),
19981 ),
19982 ]);
19983 let mut activity = Box::pin(ctx.activity("after-timer", json!([])));
19984 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19985
19986 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
19987 activity.as_mut().poll(&mut task_context)
19988 else {
19989 panic!("reordered durable command must be rejected");
19990 };
19991 assert_eq!(failure.reason, "recorded_command_mismatch");
19992 assert_eq!(failure.sequence, Some(1));
19993 assert_eq!(failure.expected.as_deref(), Some("timer"));
19994 assert_eq!(failure.actual.as_deref(), Some("activity:after-timer"));
19995 }
19996
19997 #[test]
19998 fn workflow_context_emits_a_typed_named_signal_wait() {
19999 let ctx = workflow_context(Vec::new());
20000 let mut signal = Box::pin(ctx.wait_signal("finish"));
20001 let mut task_context = TaskContext::from_waker(noop_waker_ref());
20002
20003 assert!(matches!(
20004 signal.as_mut().poll(&mut task_context),
20005 Poll::Pending
20006 ));
20007 assert_eq!(
20008 ctx.take_commands().expect("signal-wait command"),
20009 vec![json!({
20010 "type": "open_signal_wait",
20011 "signal_name": "finish",
20012 })]
20013 );
20014 }
20015
20016 #[test]
20017 fn runtime_message_stream_transport_cannot_be_opened_as_a_user_signal() {
20018 let ctx = workflow_context(Vec::new());
20019 let mut signal = Box::pin(ctx.wait_signal(MESSAGE_STREAM_SIGNAL));
20020 let mut task_context = TaskContext::from_waker(noop_waker_ref());
20021
20022 let Poll::Ready(Err(Error::Codec(message))) = signal.as_mut().poll(&mut task_context)
20023 else {
20024 panic!("runtime-reserved signal should be rejected");
20025 };
20026 assert!(message.contains("reserved by the workflow runtime"));
20027 assert!(ctx.take_commands().expect("commands").is_empty());
20028 }
20029
20030 #[tokio::test]
20031 async fn runtime_message_stream_transport_cannot_be_sent_as_a_user_signal() {
20032 let client = Client::builder("http://127.0.0.1:9")
20033 .build()
20034 .expect("client");
20035 let error = client
20036 .signal_workflow("workflow-1", MESSAGE_STREAM_SIGNAL, json!(["forged"]))
20037 .await
20038 .expect_err("runtime-reserved signal should be rejected before transport");
20039
20040 assert!(
20041 matches!(error, Error::Codec(ref message) if message.contains("reserved by the workflow runtime"))
20042 );
20043 }
20044
20045 #[test]
20046 fn message_stream_worker_task_consumes_current_contiguous_bounded_batch() {
20047 fn delivery(message_id: &str, position: u64, value: &str) -> Value {
20048 let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
20049 value.to_string(),
20050 )]))
20051 .expect("message payload");
20052 json!({
20053 "schema": MESSAGE_STREAM_SCHEMA,
20054 "stream_name": "orders",
20055 "message_id": message_id,
20056 "position": position,
20057 "payload_envelope": payload,
20058 })
20059 }
20060
20061 fn opened(sequence: u64) -> HistoryEvent {
20062 history_event(
20063 "SignalWaitOpened",
20064 json!({
20065 "sequence": sequence,
20066 "signal_name": MESSAGE_STREAM_SIGNAL,
20067 }),
20068 )
20069 }
20070
20071 fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
20072 history_event(
20073 "SignalApplied",
20074 json!({
20075 "sequence": sequence,
20076 "signal_name": MESSAGE_STREAM_SIGNAL,
20077 "value": fixture_envelope(json!([delivery])),
20078 }),
20079 )
20080 }
20081
20082 fn received(delivery: Value) -> HistoryEvent {
20083 history_event(
20084 "SignalReceived",
20085 json!({
20086 "signal_name": MESSAGE_STREAM_SIGNAL,
20087 "arguments": fixture_envelope(json!([delivery])),
20088 "payload_codec": DEFAULT_CODEC,
20089 }),
20090 )
20091 }
20092
20093 let client = Client::new("http://127.0.0.1:8080").expect("client");
20094 let mut worker = Worker::new(client, "rust-workers");
20095 worker.register_workflow("rust.message-stream-batch", |ctx, _input| async move {
20096 let messages = ctx.message_stream("orders")?.receive(2).await?;
20097 Ok(json!(messages
20098 .into_iter()
20099 .map(|message| message.message_id)
20100 .collect::<Vec<_>>()))
20101 });
20102
20103 let first = delivery("message-1", 1, "one");
20104 let second = delivery("message-2", 2, "two");
20105 let batch = worker
20106 .execute_workflow_task_decision(workflow_task(
20107 "rust.message-stream-batch",
20108 vec![
20109 opened(1),
20110 received(first.clone()),
20111 applied(1, first.clone()),
20112 received(first.clone()),
20113 received(second),
20114 ],
20115 DEFAULT_CODEC,
20116 ))
20117 .expect("worker task consumes the available batch");
20118
20119 assert_eq!(batch.commands.len(), 1);
20120 assert_eq!(batch.commands[0]["type"], "complete_workflow");
20121 assert_eq!(
20122 decode_wire_value(&batch.commands[0]["result"], DEFAULT_CODEC)
20123 .expect("workflow result"),
20124 json!(["message-1", "message-2"])
20125 );
20126 assert_eq!(
20127 batch.message_stream_cursors,
20128 vec![json!({"stream_name": "orders", "through_position": 2})]
20129 );
20130 assert!(batch.message_stream_waits.is_empty());
20131
20132 let partial = worker
20133 .execute_workflow_task_decision(workflow_task(
20134 "rust.message-stream-batch",
20135 vec![opened(1), received(first.clone()), applied(1, first)],
20136 DEFAULT_CODEC,
20137 ))
20138 .expect("worker task returns without waiting for a missing second item");
20139 assert_eq!(partial.commands.len(), 1);
20140 assert_eq!(partial.commands[0]["type"], "complete_workflow");
20141 assert_eq!(
20142 decode_wire_value(&partial.commands[0]["result"], DEFAULT_CODEC)
20143 .expect("workflow result"),
20144 json!(["message-1"])
20145 );
20146 assert_eq!(
20147 partial.message_stream_cursors,
20148 vec![json!({"stream_name": "orders", "through_position": 1})]
20149 );
20150 assert!(partial.message_stream_waits.is_empty());
20151 }
20152
20153 #[test]
20154 fn message_stream_replay_preserves_partial_batch_boundary_before_later_wait() {
20155 fn delivery(message_id: &str, position: u64, value: &str) -> Value {
20156 let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
20157 value.to_string(),
20158 )]))
20159 .expect("message payload");
20160 json!({
20161 "schema": MESSAGE_STREAM_SCHEMA,
20162 "stream_name": "orders",
20163 "message_id": message_id,
20164 "position": position,
20165 "payload_envelope": payload,
20166 })
20167 }
20168
20169 fn opened(sequence: u64) -> HistoryEvent {
20170 history_event(
20171 "SignalWaitOpened",
20172 json!({
20173 "sequence": sequence,
20174 "signal_name": MESSAGE_STREAM_SIGNAL,
20175 }),
20176 )
20177 }
20178
20179 fn received(delivery: Value) -> HistoryEvent {
20180 history_event(
20181 "SignalReceived",
20182 json!({
20183 "signal_name": MESSAGE_STREAM_SIGNAL,
20184 "arguments": fixture_envelope(json!([delivery])),
20185 "payload_codec": DEFAULT_CODEC,
20186 }),
20187 )
20188 }
20189
20190 fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
20191 history_event(
20192 "SignalApplied",
20193 json!({
20194 "sequence": sequence,
20195 "signal_name": MESSAGE_STREAM_SIGNAL,
20196 "value": fixture_envelope(json!([delivery])),
20197 }),
20198 )
20199 }
20200
20201 let client = Client::new("http://127.0.0.1:8080").expect("client");
20202 let mut worker = Worker::new(client, "rust-workers");
20203 worker.register_workflow(
20204 "rust.message-stream-partial-batches",
20205 |ctx, _input| async move {
20206 let stream = ctx.message_stream("orders")?;
20207 let first = stream.receive(10).await?;
20208 let second = stream.receive(10).await?;
20209 Ok(json!([
20210 first
20211 .into_iter()
20212 .map(|message| message.message_id)
20213 .collect::<Vec<_>>(),
20214 second
20215 .into_iter()
20216 .map(|message| message.message_id)
20217 .collect::<Vec<_>>(),
20218 ]))
20219 },
20220 );
20221
20222 let first = delivery("message-1", 1, "one");
20223 let second = delivery("message-2", 2, "two");
20224 let decision = worker
20225 .execute_workflow_task_decision(workflow_task(
20226 "rust.message-stream-partial-batches",
20227 vec![
20228 opened(1),
20229 received(first.clone()),
20230 applied(1, first),
20231 opened(2),
20232 received(second.clone()),
20233 applied(2, second),
20234 ],
20235 DEFAULT_CODEC,
20236 ))
20237 .expect("cold replay preserves both authored receive boundaries");
20238
20239 assert_eq!(decision.commands.len(), 1);
20240 assert_eq!(decision.commands[0]["type"], "complete_workflow");
20241 assert_eq!(
20242 decode_wire_value(&decision.commands[0]["result"], DEFAULT_CODEC)
20243 .expect("workflow result"),
20244 json!([["message-1"], ["message-2"]])
20245 );
20246 assert_eq!(
20247 decision.message_stream_cursors,
20248 vec![json!({"stream_name": "orders", "through_position": 2})]
20249 );
20250 assert!(decision.message_stream_waits.is_empty());
20251 }
20252
20253 #[test]
20254 fn empty_message_stream_opens_internal_signal_wait_and_reports_position() {
20255 let ctx = workflow_context(Vec::new());
20256 let stream = ctx.message_stream("orders").expect("message stream");
20257 let mut receive = Box::pin(stream.receive(10));
20258 let mut task_context = TaskContext::from_waker(noop_waker_ref());
20259
20260 assert!(matches!(
20261 receive.as_mut().poll(&mut task_context),
20262 Poll::Pending
20263 ));
20264 assert_eq!(
20265 ctx.take_commands().expect("message-stream wait command"),
20266 vec![json!({
20267 "type": "open_signal_wait",
20268 "signal_name": MESSAGE_STREAM_SIGNAL,
20269 })]
20270 );
20271 let (cursors, waits) = ctx.message_stream_metadata().expect("stream metadata");
20272 assert!(cursors.is_empty());
20273 assert_eq!(
20274 waits,
20275 vec![json!({"stream_name": "orders", "after_position": 0})]
20276 );
20277 }
20278
20279 #[test]
20280 fn continue_as_new_cursor_checkpoint_preserves_global_pending_position() {
20281 let ctx = workflow_context(vec![history_event(
20282 "SignalReceived",
20283 json!({
20284 "signal_name": MESSAGE_STREAM_SIGNAL,
20285 "arguments": fixture_envelope(json!([{
20286 "schema": MESSAGE_STREAM_CURSOR_SCHEMA,
20287 "stream_name": "orders",
20288 "through_position": 2,
20289 }])),
20290 "payload_codec": DEFAULT_CODEC,
20291 }),
20292 )]);
20293 let stream = ctx.message_stream("orders").expect("message stream");
20294 let mut receive = Box::pin(stream.receive(10));
20295 let mut task_context = TaskContext::from_waker(noop_waker_ref());
20296
20297 assert!(matches!(
20298 receive.as_mut().poll(&mut task_context),
20299 Poll::Pending
20300 ));
20301 let (cursors, waits) = ctx.message_stream_metadata().expect("stream metadata");
20302 assert_eq!(
20303 cursors,
20304 vec![json!({"stream_name": "orders", "through_position": 2})]
20305 );
20306 assert_eq!(
20307 waits,
20308 vec![json!({"stream_name": "orders", "after_position": 2})]
20309 );
20310 }
20311
20312 #[test]
20313 fn message_stream_delivery_preserves_typed_avro_arguments_across_replay() {
20314 let mut empty_map = BTreeMap::new();
20315 let mut nested = BTreeMap::new();
20316 nested.insert(
20317 "value".to_string(),
20318 AvroValue::Array(vec![AvroValue::Bytes(b"nested".to_vec())]),
20319 );
20320 let values = vec![
20321 AvroValue::Bytes(vec![0, 255]),
20322 AvroValue::Long(1),
20323 AvroValue::Double(1.0),
20324 AvroValue::Array(Vec::new()),
20325 AvroValue::Map(std::mem::take(&mut empty_map)),
20326 AvroValue::Map(nested),
20327 ];
20328 let payload = encode_avro_value(&AvroValue::Array(values.clone())).expect("payload");
20329 let transport = vec![json!({
20330 "schema": MESSAGE_STREAM_SCHEMA,
20331 "stream_name": "orders",
20332 "message_id": "message-1",
20333 "position": 1,
20334 "payload_envelope": payload,
20335 })];
20336
20337 for _ in 0..2 {
20338 let Some(MessageStreamDelivery::Message(message)) =
20339 decode_message_stream_delivery(transport.clone()).expect("delivery")
20340 else {
20341 panic!("message delivery expected");
20342 };
20343 assert_eq!(message.arguments, values);
20344 assert!(matches!(message.arguments[1], AvroValue::Long(1)));
20345 assert!(matches!(message.arguments[2], AvroValue::Double(1.0)));
20346 }
20347 }
20348
20349 #[test]
20350 fn cold_worker_replacement_consumes_message_stream_wait_arrivals_once_in_order() {
20351 fn delivery(message_id: &str, position: u64, value: &str) -> Value {
20352 let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
20353 value.to_string(),
20354 )]))
20355 .expect("message payload");
20356 json!({
20357 "schema": MESSAGE_STREAM_SCHEMA,
20358 "stream_name": "orders",
20359 "message_id": message_id,
20360 "position": position,
20361 "payload_envelope": payload,
20362 })
20363 }
20364
20365 fn opened(sequence: u64) -> HistoryEvent {
20366 history_event(
20367 "SignalWaitOpened",
20368 json!({
20369 "sequence": sequence,
20370 "signal_name": MESSAGE_STREAM_SIGNAL,
20371 }),
20372 )
20373 }
20374
20375 fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
20376 history_event(
20377 "SignalApplied",
20378 json!({
20379 "sequence": sequence,
20380 "signal_name": MESSAGE_STREAM_SIGNAL,
20381 "value": fixture_envelope(json!([delivery])),
20382 }),
20383 )
20384 }
20385
20386 fn worker() -> Worker {
20387 let client = Client::new("http://127.0.0.1:8080").expect("client");
20388 let mut worker = Worker::new(client, "rust-workers");
20389 worker.register_workflow("rust.message-stream", |ctx, _input| async move {
20390 let stream = ctx.message_stream("orders")?;
20391 let first = stream.receive_one().await?;
20392 let second = stream.receive_one().await?;
20393 Ok(json!([first.message_id, second.message_id]))
20394 });
20395 worker
20396 }
20397
20398 fn task_with_resume(history: Vec<HistoryEvent>, delivery: Value) -> WorkflowTask {
20399 let mut task = workflow_task("rust.message-stream", history, DEFAULT_CODEC);
20400 task.signal_name = Some(MESSAGE_STREAM_SIGNAL.to_string());
20401 task.signal_arguments = Some(fixture_envelope(json!([delivery])));
20402 task
20403 }
20404
20405 let waiting = worker()
20406 .execute_workflow_task_decision(workflow_task(
20407 "rust.message-stream",
20408 Vec::new(),
20409 DEFAULT_CODEC,
20410 ))
20411 .expect("first worker opens the stream wait");
20412 assert_eq!(
20413 waiting.commands,
20414 vec![json!({
20415 "type": "open_signal_wait",
20416 "signal_name": MESSAGE_STREAM_SIGNAL,
20417 })]
20418 );
20419 assert!(waiting.message_stream_cursors.is_empty());
20420 assert_eq!(
20421 waiting.message_stream_waits,
20422 vec![json!({"stream_name": "orders", "after_position": 0})]
20423 );
20424
20425 let first_delivery = delivery("message-1", 1, "one");
20426 let first_arrival = worker()
20427 .execute_workflow_task_decision(task_with_resume(
20428 vec![opened(1)],
20429 first_delivery.clone(),
20430 ))
20431 .expect("replacement worker consumes the first arrival");
20432 assert_eq!(
20433 first_arrival.commands,
20434 vec![json!({
20435 "type": "open_signal_wait",
20436 "signal_name": MESSAGE_STREAM_SIGNAL,
20437 })]
20438 );
20439 assert_eq!(
20440 first_arrival.message_stream_cursors,
20441 vec![json!({"stream_name": "orders", "through_position": 1})]
20442 );
20443 assert_eq!(
20444 first_arrival.message_stream_waits,
20445 vec![json!({"stream_name": "orders", "after_position": 1})]
20446 );
20447
20448 let second_delivery = delivery("message-2", 2, "two");
20449 let first_applied = applied(1, first_delivery);
20450 let completed = worker()
20451 .execute_workflow_task_decision(task_with_resume(
20452 vec![opened(1), first_applied.clone(), opened(2)],
20453 second_delivery.clone(),
20454 ))
20455 .expect("next replacement worker consumes the second arrival");
20456 assert_eq!(completed.commands.len(), 1);
20457 assert_eq!(completed.commands[0]["type"], "complete_workflow");
20458 assert_eq!(
20459 decode_wire_value(&completed.commands[0]["result"], DEFAULT_CODEC)
20460 .expect("workflow result"),
20461 json!(["message-1", "message-2"])
20462 );
20463 assert_eq!(
20464 completed.message_stream_cursors,
20465 vec![json!({"stream_name": "orders", "through_position": 2})]
20466 );
20467 assert!(completed.message_stream_waits.is_empty());
20468
20469 let replay_history = vec![
20470 opened(1),
20471 first_applied,
20472 opened(2),
20473 applied(2, second_delivery),
20474 ];
20475 for _cold_worker_or_restart in 0..2 {
20476 let replayed = worker()
20477 .execute_workflow_task_decision(workflow_task(
20478 "rust.message-stream",
20479 replay_history.clone(),
20480 DEFAULT_CODEC,
20481 ))
20482 .expect("cold worker replays each logical message exactly once");
20483 assert_eq!(replayed.commands.len(), 1);
20484 assert_eq!(
20485 decode_wire_value(&replayed.commands[0]["result"], DEFAULT_CODEC)
20486 .expect("replayed workflow result"),
20487 json!(["message-1", "message-2"])
20488 );
20489 assert_eq!(
20490 replayed.message_stream_cursors,
20491 vec![json!({"stream_name": "orders", "through_position": 2})]
20492 );
20493 assert!(replayed.message_stream_waits.is_empty());
20494 }
20495 }
20496
20497 #[test]
20498 fn message_stream_capability_and_completion_require_protocol_one_fifteen() {
20499 assert!(!worker_protocol_supports_message_streams("1.14"));
20500 assert!(worker_protocol_supports_message_streams("1.15"));
20501 assert!(worker_protocol_supports_message_streams("1.16"));
20502 assert!(worker_protocol_supports_message_streams(
20503 WORKER_PROTOCOL_VERSION
20504 ));
20505 assert_eq!(MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION, "1.15");
20506 }
20507
20508 #[test]
20509 fn condition_wait_history_cannot_be_consumed_as_a_typed_signal_wait() {
20510 let ctx = workflow_context(vec![
20511 history_event(
20512 "ConditionWaitOpened",
20513 json!({
20514 "sequence": 1,
20515 "condition_wait_id": "condition:1",
20516 "condition_wait_occurrence_id": "rust:condition-wait:0",
20517 "condition_key": "signal:finish",
20518 "condition_definition_fingerprint": "sha256:signal-finish-v1",
20519 }),
20520 ),
20521 history_event(
20522 "ConditionWaitSatisfied",
20523 json!({
20524 "sequence": 1,
20525 "condition_wait_id": "condition:1",
20526 "condition_wait_occurrence_id": "rust:condition-wait:0",
20527 "condition_key": "signal:finish",
20528 "condition_definition_fingerprint": "sha256:signal-finish-v1",
20529 }),
20530 ),
20531 history_event(
20532 "SignalReceived",
20533 json!({"signal_name": "finish", "arguments": []}),
20534 ),
20535 ]);
20536 let mut signal = Box::pin(ctx.wait_signal("finish"));
20537 let mut task_context = TaskContext::from_waker(noop_waker_ref());
20538
20539 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
20540 signal.as_mut().poll(&mut task_context)
20541 else {
20542 panic!("condition history must not resolve as a typed signal wait");
20543 };
20544 assert_eq!(failure.reason, "recorded_command_mismatch");
20545 assert_eq!(failure.expected.as_deref(), Some("condition wait"));
20546 }
20547
20548 #[test]
20549 fn replay_orders_signal_waits_and_timers_in_one_command_stream() {
20550 let signal_then_timer = vec![
20551 history_event(
20552 "SignalWaitOpened",
20553 json!({"sequence": 1, "signal_name": "go"}),
20554 ),
20555 history_event(
20556 "SignalApplied",
20557 json!({
20558 "sequence": 1,
20559 "signal_name": "go",
20560 "value": fixture_envelope(json!(["now"])),
20561 }),
20562 ),
20563 history_event(
20564 "TimerScheduled",
20565 json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
20566 ),
20567 history_event(
20568 "TimerFired",
20569 json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
20570 ),
20571 ];
20572
20573 let ctx = workflow_context(signal_then_timer.clone());
20574 let mut signal = Box::pin(ctx.wait_signal("go"));
20575 let mut task_context = TaskContext::from_waker(noop_waker_ref());
20576 assert!(matches!(
20577 signal.as_mut().poll(&mut task_context),
20578 Poll::Ready(Ok(arguments)) if arguments == vec![json!("now")]
20579 ));
20580 let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5)));
20581 assert!(matches!(
20582 timer.as_mut().poll(&mut task_context),
20583 Poll::Ready(Ok(()))
20584 ));
20585 ctx.ensure_history_consumed()
20586 .expect("signal and timer history consumed in order");
20587
20588 let reordered = workflow_context(signal_then_timer);
20589 let mut timer_first = Box::pin(reordered.sleep(Duration::from_secs(5)));
20590 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
20591 timer_first.as_mut().poll(&mut task_context)
20592 else {
20593 panic!("timer cannot consume signal-wait-first history");
20594 };
20595 assert_eq!(failure.reason, "recorded_command_mismatch");
20596 assert_eq!(failure.sequence, Some(1));
20597 assert_eq!(failure.expected.as_deref(), Some("signal wait"));
20598
20599 let timer_then_signal = vec![
20600 history_event(
20601 "TimerScheduled",
20602 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20603 ),
20604 history_event(
20605 "TimerFired",
20606 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20607 ),
20608 history_event(
20609 "SignalWaitOpened",
20610 json!({"sequence": 2, "signal_name": "go"}),
20611 ),
20612 history_event(
20613 "SignalApplied",
20614 json!({
20615 "sequence": 2,
20616 "signal_name": "go",
20617 "value": fixture_envelope(json!([])),
20618 }),
20619 ),
20620 ];
20621 let reordered = workflow_context(timer_then_signal);
20622 let mut signal_first = Box::pin(reordered.wait_signal("go"));
20623 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
20624 signal_first.as_mut().poll(&mut task_context)
20625 else {
20626 panic!("signal wait cannot consume timer-first history");
20627 };
20628 assert_eq!(failure.reason, "recorded_command_mismatch");
20629 assert_eq!(failure.sequence, Some(1));
20630 assert_eq!(failure.expected.as_deref(), Some("timer"));
20631 }
20632
20633 #[test]
20634 fn workflow_history_rejects_duplicate_or_colliding_command_sequences() {
20635 let duplicate_timer = WorkflowState::new(
20636 vec![
20637 history_event(
20638 "TimerScheduled",
20639 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20640 ),
20641 history_event(
20642 "TimerScheduled",
20643 json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
20644 ),
20645 ],
20646 "rust-workers".to_string(),
20647 DEFAULT_CODEC.to_string(),
20648 None,
20649 )
20650 .expect_err("one workflow sequence cannot schedule two timers");
20651 assert!(matches!(
20652 duplicate_timer,
20653 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20654 if reason == "timer_schedule_missing_or_duplicate"
20655 ));
20656
20657 let colliding_kinds = WorkflowState::new(
20658 vec![
20659 history_event(
20660 "TimerScheduled",
20661 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20662 ),
20663 history_event(
20664 "ActivityCompleted",
20665 json!({"sequence": 1, "activity_type": "same-sequence"}),
20666 ),
20667 ],
20668 "rust-workers".to_string(),
20669 DEFAULT_CODEC.to_string(),
20670 None,
20671 )
20672 .expect_err("one workflow sequence cannot identify two command kinds");
20673 assert!(matches!(
20674 colliding_kinds,
20675 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20676 if reason == "durable_command_sequence_collision"
20677 ));
20678
20679 let duplicate_signal_wait = WorkflowState::new(
20680 vec![
20681 history_event(
20682 "SignalWaitOpened",
20683 json!({"sequence": 1, "signal_name": "go"}),
20684 ),
20685 history_event(
20686 "SignalWaitOpened",
20687 json!({"sequence": 1, "signal_name": "go"}),
20688 ),
20689 ],
20690 "rust-workers".to_string(),
20691 DEFAULT_CODEC.to_string(),
20692 None,
20693 )
20694 .expect_err("one workflow sequence cannot open two signal waits");
20695 assert!(matches!(
20696 duplicate_signal_wait,
20697 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20698 if reason == "signal_wait_open_missing_or_duplicate"
20699 ));
20700 }
20701
20702 #[test]
20703 fn workflow_history_accepts_a_first_command_after_global_sequence_gaps() {
20704 let result = encode_value_envelope(&json!({"captured": true}), DEFAULT_CODEC)
20705 .expect("side-effect result");
20706 let ctx = workflow_context(vec![history_event(
20707 "SideEffectRecorded",
20708 json!({"sequence": 99, "result": result}),
20709 )]);
20710
20711 let replayed: Value = ctx
20712 .side_effect(|| panic!("recorded side effect must not run"))
20713 .expect("positive global workflow sequence is valid");
20714 assert_eq!(replayed, json!({"captured": true}));
20715 ctx.ensure_history_consumed().expect("history consumed");
20716 }
20717
20718 #[test]
20719 fn workflow_history_rejects_zero_and_descending_command_sequences() {
20720 let result =
20721 encode_value_envelope(&json!("captured"), DEFAULT_CODEC).expect("side-effect result");
20722 let zero = WorkflowState::new(
20723 vec![history_event(
20724 "SideEffectRecorded",
20725 json!({"sequence": 0, "result": result.clone()}),
20726 )],
20727 "rust-workers".to_string(),
20728 DEFAULT_CODEC.to_string(),
20729 None,
20730 )
20731 .expect_err("durable command sequences must be positive");
20732 assert!(matches!(
20733 zero,
20734 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20735 if reason == "durable_command_sequence_invalid"
20736 ));
20737
20738 let descending = WorkflowState::new(
20739 vec![
20740 history_event(
20741 "SideEffectRecorded",
20742 json!({"sequence": 3, "result": result}),
20743 ),
20744 history_event(
20745 "VersionMarkerRecorded",
20746 json!({
20747 "sequence": 2,
20748 "change_id": "descending-marker",
20749 "version": 1,
20750 "min_supported": 1,
20751 "max_supported": 1,
20752 }),
20753 ),
20754 ],
20755 "rust-workers".to_string(),
20756 DEFAULT_CODEC.to_string(),
20757 None,
20758 )
20759 .expect_err("new durable commands must remain strictly ordered");
20760 let Error::NonDeterministicReplay(failure) = descending else {
20761 panic!("expected typed replay failure");
20762 };
20763 assert_eq!(failure.reason, "durable_command_sequence_mismatch");
20764 assert_eq!(failure.sequence, Some(2));
20765 assert_eq!(
20766 failure.expected.as_deref(),
20767 Some("workflow sequence greater than 3")
20768 );
20769 assert_eq!(failure.actual.as_deref(), Some("2"));
20770 }
20771
20772 #[test]
20773 fn workflow_task_replay_completes_after_signals_create_sequence_gaps() {
20774 fn worker() -> Worker {
20775 let client = Client::new("http://127.0.0.1:8080").expect("client");
20776 let mut worker = Worker::new(client, "rust-workers");
20777 worker.register_workflow("rust.finish-after-gaps", |ctx, _input| async move {
20778 ctx.wait_signal("finish").await?;
20779 let marker: String =
20780 ctx.side_effect(|| panic!("recorded side effect must not run"))?;
20781 assert_eq!(marker, "after-finish");
20782 Ok(json!("finished"))
20783 });
20784 worker
20785 }
20786
20787 let marker = encode_value_envelope(&json!("after-finish"), DEFAULT_CODEC)
20788 .expect("side-effect result");
20789 let task = workflow_task(
20790 "rust.finish-after-gaps",
20791 vec![
20792 history_event(
20793 "SignalWaitOpened",
20794 json!({"sequence": 1, "signal_name": "finish"}),
20795 ),
20796 history_event(
20797 "SignalReceived",
20798 json!({
20799 "signal_id": "increment-3",
20800 "signal_name": "increment",
20801 "workflow_sequence": 2,
20802 "payload_codec": DEFAULT_CODEC,
20803 "arguments": fixture_envelope(json!([3])),
20804 }),
20805 ),
20806 history_event(
20807 "SignalReceived",
20808 json!({
20809 "signal_id": "increment-5",
20810 "signal_name": "increment",
20811 "workflow_sequence": 3,
20812 "payload_codec": DEFAULT_CODEC,
20813 "arguments": fixture_envelope(json!([5])),
20814 }),
20815 ),
20816 history_event(
20817 "SignalReceived",
20818 json!({
20819 "signal_id": "finish",
20820 "signal_name": "finish",
20821 "workflow_sequence": 4,
20822 "payload_codec": DEFAULT_CODEC,
20823 "arguments": fixture_envelope(json!([])),
20824 }),
20825 ),
20826 history_event(
20827 "SignalApplied",
20828 json!({
20829 "sequence": 1,
20830 "signal_id": "finish",
20831 "signal_name": "finish",
20832 "payload_codec": DEFAULT_CODEC,
20833 "value": fixture_envelope(json!([])),
20834 }),
20835 ),
20836 history_event(
20837 "SideEffectRecorded",
20838 json!({"sequence": 5, "result": marker}),
20839 ),
20840 ],
20841 DEFAULT_CODEC,
20842 );
20843
20844 for _original_or_cold_worker in 0..2 {
20845 let commands = worker()
20846 .execute_workflow_task(task.clone())
20847 .expect("signal gaps preserve deterministic replay");
20848 assert_eq!(commands.len(), 1, "replay emits only terminal completion");
20849 assert_eq!(commands[0]["type"], "complete_workflow");
20850 assert_eq!(
20851 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
20852 json!("finished")
20853 );
20854 }
20855 }
20856
20857 #[test]
20858 fn workflow_sleep_rejects_unrepresentable_rounded_duration() {
20859 let ctx = workflow_context(Vec::new());
20860 let mut sleep = Box::pin(ctx.start_timer(Duration::new(u64::MAX, 1)));
20861 let mut task_context = TaskContext::from_waker(noop_waker_ref());
20862 assert!(matches!(
20863 sleep.as_mut().poll(&mut task_context),
20864 Poll::Ready(Err(Error::TimerDurationOverflow))
20865 ));
20866 assert!(ctx.take_commands().expect("commands").is_empty());
20867 }
20868
20869 #[test]
20870 fn workflow_memo_update_emits_canonical_command_and_replays_once() {
20871 let entries = AvroValue::Map(BTreeMap::from([
20872 ("text".to_string(), AvroValue::String("same".to_string())),
20873 (
20874 "nested".to_string(),
20875 AvroValue::Map(BTreeMap::from([
20876 ("beta".to_string(), AvroValue::Long(2)),
20877 ("alpha".to_string(), AvroValue::Long(1)),
20878 ])),
20879 ),
20880 ("long".to_string(), AvroValue::Long(7)),
20881 ("double".to_string(), AvroValue::Double(7.0)),
20882 ("binary".to_string(), AvroValue::Bytes(b"same".to_vec())),
20883 ]));
20884 let ctx = workflow_context(Vec::new());
20885 ctx.upsert_memo(entries.clone()).expect("valid memo update");
20886 let commands = ctx.take_commands().expect("commands");
20887
20888 assert_eq!(commands.len(), 1);
20889 assert_eq!(commands[0]["type"], "upsert_memo");
20890 let server_entries = json!({
20891 "codec": "avro",
20892 "blob": "wwHioz3/VYAiNw4KDGJpbmFyeQgIc2FtZQxkb3VibGUGAAAAAAAAHEAIbG9uZwQODG5lc3RlZA4ECmFscGhhBAIIYmV0YQQEAAh0ZXh0CghzYW1lAA==",
20893 });
20894 assert_eq!(
20895 commands[0]["entries"]
20896 .as_object()
20897 .expect("entries envelope")
20898 .keys()
20899 .collect::<Vec<_>>(),
20900 vec!["blob", "codec"]
20901 );
20902 assert_eq!(commands[0]["entries"], server_entries);
20903 let wire_entries =
20904 decode_wire_avro_value(&commands[0]["entries"], DEFAULT_CODEC).expect("memo entries");
20905 assert_eq!(wire_entries, entries);
20906
20907 let history = vec![history_event(
20908 "MemoUpserted",
20909 json!({
20910 "sequence": 1,
20911 "entries": server_entries.clone(),
20912 "merged": server_entries,
20913 }),
20914 )];
20915 let replay = workflow_context(history.clone());
20916 replay
20917 .upsert_memo(entries.clone())
20918 .expect("matching replay identity");
20919 assert!(replay.take_commands().expect("replay commands").is_empty());
20920
20921 let changed_types = AvroValue::Map(BTreeMap::from([
20922 ("text".to_string(), AvroValue::Bytes(b"same".to_vec())),
20923 (
20924 "nested".to_string(),
20925 AvroValue::Map(BTreeMap::from([
20926 ("alpha".to_string(), AvroValue::Long(1)),
20927 ("beta".to_string(), AvroValue::Long(2)),
20928 ])),
20929 ),
20930 ("long".to_string(), AvroValue::Double(7.0)),
20931 ("double".to_string(), AvroValue::Long(7)),
20932 ("binary".to_string(), AvroValue::String("same".to_string())),
20933 ]));
20934 let error = workflow_context(history)
20935 .upsert_memo(changed_types)
20936 .expect_err("memo replay identity must preserve Avro value types");
20937 assert!(matches!(
20938 error,
20939 Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20940 ));
20941 }
20942
20943 #[test]
20944 fn workflow_memo_update_rejects_changed_replay_identity_and_invalid_keys() {
20945 let original = encode_value_envelope(&json!({"stage": "original"}), DEFAULT_CODEC)
20946 .expect("memo envelope");
20947 let replay = workflow_context(vec![history_event(
20948 "MemoUpserted",
20949 json!({
20950 "sequence": 1,
20951 "entries": original.clone(),
20952 "merged": original
20953 }),
20954 )]);
20955 let error = replay
20956 .upsert_memo(json!({"stage": "changed"}))
20957 .expect_err("changed memo update must fail replay");
20958 assert!(matches!(
20959 error,
20960 Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20961 ));
20962
20963 let invalid = workflow_context(Vec::new())
20964 .upsert_memo(
20965 json!({"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx": true}),
20966 )
20967 .expect_err("oversized key");
20968 assert!(matches!(invalid, Error::InvalidMemoUpdate(_)));
20969 }
20970
20971 #[test]
20972 fn workflow_memo_replay_distinguishes_signed_zero_identity() {
20973 let negative_zero = AvroValue::Map(BTreeMap::from([(
20974 "reading".to_string(),
20975 AvroValue::Double(-0.0),
20976 )]));
20977 let negative_zero_envelope =
20978 encode_typed_envelope(&negative_zero, DEFAULT_CODEC).expect("negative zero envelope");
20979 let history = vec![history_event(
20980 "MemoUpserted",
20981 json!({
20982 "sequence": 1,
20983 "entries": negative_zero_envelope.clone(),
20984 "merged": negative_zero_envelope,
20985 }),
20986 )];
20987
20988 workflow_context(history.clone())
20989 .upsert_memo(negative_zero)
20990 .expect("matching negative-zero history identity");
20991
20992 let error = workflow_context(history)
20993 .upsert_memo(AvroValue::Map(BTreeMap::from([(
20994 "reading".to_string(),
20995 AvroValue::Double(0.0),
20996 )])))
20997 .expect_err("positive zero must not consume negative-zero memo history");
20998 assert!(matches!(
20999 error,
21000 Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
21001 ));
21002 }
21003
21004 #[test]
21005 fn workflow_memo_capability_requires_flag_and_command_advertisement() {
21006 let supported = json!({
21007 "workflow_memo_updates": {"supported": true, "minimum_protocol_version": "1.14"},
21008 "supported_workflow_task_commands": ["complete_workflow", "upsert_memo"]
21009 });
21010 assert!(runtime_supports_workflow_memo_updates(Some(&supported)));
21011 assert!(!runtime_supports_workflow_memo_updates(Some(&json!({
21012 "workflow_memo_updates": {"supported": false},
21013 "supported_workflow_task_commands": ["upsert_memo"]
21014 }))));
21015 assert!(commands_use_workflow_memo_updates(&[json!({
21016 "type": "upsert_memo",
21017 "entries": {"stage": "processing"}
21018 })]));
21019 }
21020
21021 #[test]
21022 fn workflow_task_replay_completes_without_rescheduling_recorded_commands() {
21023 let client = Client::new("http://127.0.0.1:8080").expect("client");
21024 let mut worker = Worker::new(client, "rust-workers");
21025 worker.register_workflow("rust.timer", |ctx, _input| async move {
21026 ctx.sleep(Duration::from_secs(5)).await?;
21027 ctx.activity("after-timer", json!([])).await
21028 });
21029
21030 let task = |history_events| WorkflowTask {
21031 task_id: "wft-rust-timer-1".to_string(),
21032 workflow_command_id: None,
21033 workflow_id: Some("wf-rust-timer".to_string()),
21034 run_id: Some("run-rust-timer".to_string()),
21035 workflow_type: "rust.timer".to_string(),
21036 cancel_requested: false,
21037 payload_codec: DEFAULT_CODEC.to_string(),
21038 arguments: Some(
21039 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
21040 ),
21041 history_events,
21042 total_history_events: None,
21043 history_size_bytes: None,
21044 continue_as_new_recommended: None,
21045 history_budget_pressure: None,
21046 next_history_page_token: None,
21047 workflow_task_attempt: 1,
21048 workflow_signal_id: None,
21049 signal_name: None,
21050 signal_arguments: None,
21051 workflow_update_id: None,
21052 update_name: None,
21053 lease_owner: Some("rust-worker".to_string()),
21054 };
21055
21056 let initial = worker
21057 .execute_workflow_task(task(Vec::new()))
21058 .expect("initial timer task");
21059 assert_eq!(
21060 initial,
21061 vec![json!({"type": "start_timer", "delay_seconds": 5})]
21062 );
21063
21064 let activity_result =
21065 encode_value_envelope(&json!("done"), DEFAULT_CODEC).expect("activity result");
21066 let replayed = worker
21067 .execute_workflow_task(task(vec![
21068 history_event(
21069 "TimerScheduled",
21070 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
21071 ),
21072 history_event(
21073 "TimerFired",
21074 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
21075 ),
21076 history_event(
21077 "ActivityCompleted",
21078 json!({
21079 "sequence": 2,
21080 "activity_type": "after-timer",
21081 "payload_codec": DEFAULT_CODEC,
21082 "result": activity_result,
21083 }),
21084 ),
21085 ]))
21086 .expect("replayed workflow task");
21087 assert_eq!(replayed.len(), 1);
21088 assert_eq!(replayed[0]["type"], "complete_workflow");
21089 assert_eq!(
21090 decode_wire_value(&replayed[0]["result"], DEFAULT_CODEC).expect("result"),
21091 json!("done")
21092 );
21093 }
21094
21095 #[test]
21096 fn workflow_continue_as_new_emits_arguments_type_and_queue_once() {
21097 let client = Client::new("http://127.0.0.1:8080").expect("client");
21098 let mut worker = Worker::new(client, "rust-workers");
21099 worker.register_workflow("rust.continue", |ctx, _input| async move {
21100 ctx.continue_as_new_with_options(
21101 ContinueAsNewOptions::new()
21102 .workflow_type("rust.next")
21103 .task_queue("next-workers"),
21104 json!([2, {"cursor": "next"}]),
21105 )
21106 });
21107
21108 let commands = worker
21109 .execute_workflow_task(workflow_task("rust.continue", Vec::new(), DEFAULT_CODEC))
21110 .expect("continue-as-new command");
21111
21112 assert_eq!(commands.len(), 1);
21113 assert_eq!(commands[0]["type"], "continue_as_new");
21114 assert_eq!(commands[0]["workflow_type"], "rust.next");
21115 assert_eq!(commands[0]["queue"], "next-workers");
21116 assert_eq!(
21117 decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC)
21118 .expect("continue-as-new arguments"),
21119 json!([2, {"cursor": "next"}])
21120 );
21121 }
21122
21123 #[test]
21124 fn continue_as_new_preserves_typed_arguments() {
21125 let client = Client::new("http://127.0.0.1:8080").expect("client");
21126 let mut worker = Worker::new(client, "rust-workers");
21127 worker.register_workflow_avro_value("rust.typed-continue", |ctx, _input| async move {
21128 ctx.continue_as_new(AvroValue::Array(vec![typed_fidelity_probe()]))?;
21129 unreachable!("continue-as-new returns a control-flow error")
21130 });
21131
21132 let commands = worker
21133 .execute_workflow_task(workflow_task(
21134 "rust.typed-continue",
21135 Vec::new(),
21136 DEFAULT_CODEC,
21137 ))
21138 .expect("typed continue-as-new command");
21139
21140 assert_eq!(commands[0]["type"], "continue_as_new");
21141 assert_eq!(
21142 decode_wire_avro_value(&commands[0]["arguments"], DEFAULT_CODEC)
21143 .expect("typed continue arguments"),
21144 AvroValue::Array(vec![typed_fidelity_probe()])
21145 );
21146 }
21147
21148 #[test]
21149 fn recorded_continue_as_new_is_consumed_without_duplicate_successor_command() {
21150 let client = Client::new("http://127.0.0.1:8080").expect("client");
21151 let mut worker = Worker::new(client, "rust-workers");
21152 worker.register_workflow("rust.continue", |ctx, _input| async move {
21153 ctx.continue_as_new(json!([2]))
21154 });
21155 let task = workflow_task(
21156 "rust.continue",
21157 vec![history_event(
21158 "WorkflowContinuedAsNew",
21159 json!({"sequence": 1, "continued_to_run_id": "run-next"}),
21160 )],
21161 DEFAULT_CODEC,
21162 );
21163
21164 for _worker_restart_or_redelivery in 0..2 {
21165 let commands = worker
21166 .execute_workflow_task(task.clone())
21167 .expect("recorded transition replays");
21168 assert!(
21169 commands.is_empty(),
21170 "replay must not emit another successor"
21171 );
21172 }
21173 }
21174
21175 #[test]
21176 fn continue_as_new_rejects_invalid_overrides_before_emitting_a_command() {
21177 let ctx = workflow_context(Vec::new());
21178 let error = ctx
21179 .continue_as_new_with_options(ContinueAsNewOptions::new().task_queue(" "), json!([1]))
21180 .expect_err("blank queue must be rejected");
21181
21182 let Error::InvalidContinueAsNewOptions(error) = error else {
21183 panic!("expected typed continue-as-new validation error");
21184 };
21185 assert_eq!(error.field, "task_queue");
21186 assert!(ctx.take_commands().expect("commands").is_empty());
21187 }
21188
21189 #[test]
21190 fn workflow_context_exposes_server_history_budget() {
21191 let client = Client::new("http://127.0.0.1:8080").expect("client");
21192 let mut worker = Worker::new(client, "rust-workers");
21193 worker.register_workflow("rust.history-budget", |ctx, _input| async move {
21194 let budget = ctx.history_budget()?;
21195 Ok(json!({
21196 "events": budget.event_count,
21197 "bytes": budget.size_bytes,
21198 "recommended": budget.continue_as_new_recommended,
21199 "pressure": budget.pressure,
21200 }))
21201 });
21202 let task: WorkflowTask = serde_json::from_value(json!({
21203 "task_id": "task-history-budget",
21204 "workflow_type": "rust.history-budget",
21205 "payload_codec": DEFAULT_CODEC,
21206 "history_events": [],
21207 "total_history_events": 480,
21208 "history_size_bytes": 1_048_576,
21209 "continue_as_new_recommended": true,
21210 "history_budget_pressure": "continue_as_new_recommended",
21211 }))
21212 .expect("published workflow task");
21213
21214 let commands = worker
21215 .execute_workflow_task(task)
21216 .expect("history-budget workflow");
21217 let result = decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("result");
21218 assert_eq!(result["events"], 480);
21219 assert_eq!(result["bytes"], 1_048_576);
21220 assert_eq!(result["recommended"], true);
21221 assert_eq!(result["pressure"], "continue_as_new_recommended");
21222 }
21223
21224 #[test]
21225 fn uncaught_workflow_handler_error_emits_terminal_failure_command() {
21226 let client = Client::new("http://127.0.0.1:8080").expect("client");
21227 let mut worker = Worker::new(client, "rust-workers");
21228 worker.register_workflow("rust.failing", |_ctx, _input| async move {
21229 Err(Error::Codec("rust_conformance_failure".to_string()))
21230 });
21231 let task = WorkflowTask {
21232 task_id: "wft-rust-failing-1".to_string(),
21233 workflow_command_id: None,
21234 workflow_id: Some("wf-rust-failing".to_string()),
21235 run_id: Some("run-rust-failing".to_string()),
21236 workflow_type: "rust.failing".to_string(),
21237 cancel_requested: false,
21238 payload_codec: DEFAULT_CODEC.to_string(),
21239 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21240 history_events: Vec::new(),
21241 total_history_events: Some(0),
21242 history_size_bytes: None,
21243 continue_as_new_recommended: None,
21244 history_budget_pressure: None,
21245 next_history_page_token: None,
21246 workflow_task_attempt: 1,
21247 workflow_signal_id: None,
21248 signal_name: None,
21249 signal_arguments: None,
21250 workflow_update_id: None,
21251 update_name: None,
21252 lease_owner: Some("rust-worker".to_string()),
21253 };
21254
21255 let commands = worker
21256 .execute_workflow_task(task)
21257 .expect("handler failure becomes a workflow command");
21258
21259 assert_eq!(commands.len(), 1);
21260 assert_eq!(commands[0]["type"], "fail_workflow");
21261 assert_eq!(commands[0]["exception_type"], "RustWorkflowError");
21262 assert_eq!(commands[0]["exception_class"], "durable_workflow::Error");
21263 assert_eq!(commands[0]["non_retryable"], false);
21264 assert_eq!(
21265 commands[0]["message"],
21266 "codec error: rust_conformance_failure"
21267 );
21268 assert_eq!(
21269 commands[0]["exception"]["message"],
21270 "codec error: rust_conformance_failure"
21271 );
21272 }
21273
21274 #[test]
21275 fn ordinary_handler_error_preserves_commands_queued_in_the_same_decision() {
21276 let client = Client::new("http://127.0.0.1:8080").expect("client");
21277 let mut worker = Worker::new(client, "rust-workers");
21278 worker.register_workflow("rust.failing-after-side-effect", |ctx, _input| async move {
21279 let _: String = ctx.side_effect(|| "captured".to_string())?;
21280 Err(Error::WorkerLoop("application failure".to_string()))
21281 });
21282
21283 let commands = worker
21284 .execute_workflow_task(workflow_task(
21285 "rust.failing-after-side-effect",
21286 Vec::new(),
21287 DEFAULT_CODEC,
21288 ))
21289 .expect("ordinary failure remains a workflow decision");
21290
21291 assert_eq!(commands.len(), 2);
21292 assert_eq!(commands[0]["type"], "record_side_effect");
21293 assert_eq!(commands[1]["type"], "fail_workflow");
21294 }
21295
21296 #[test]
21297 fn uncaught_recorded_activity_failure_claims_only_its_persisted_boundary() {
21298 let client = Client::new("http://127.0.0.1:8080").expect("client");
21299 let mut worker = Worker::new(client, "rust-workers");
21300 worker.register_workflow("rust.redrive", |ctx, _input| async move {
21301 ctx.activity("greet", json!([])).await?;
21302 Ok(Value::Null)
21303 });
21304 worker.register_workflow("rust.redrive-translated", |ctx, _input| async move {
21305 match ctx.activity("greet", json!([])).await {
21306 Err(Error::ActivityFailed(_)) => Err(Error::WorkerLoop("translated".to_string())),
21307 result => result,
21308 }
21309 });
21310
21311 let failed = history_event(
21312 "ActivityFailed",
21313 json!({
21314 "sequence": 1,
21315 "activity_type": "greet",
21316 "activity_execution_id": "activity-1",
21317 "message": "failed"
21318 }),
21319 );
21320 let commands = worker
21321 .execute_workflow_task(workflow_task(
21322 "rust.redrive",
21323 vec![failed.clone()],
21324 DEFAULT_CODEC,
21325 ))
21326 .expect("recorded failure becomes a workflow command");
21327 assert_eq!(commands[0]["type"], "fail_workflow");
21328 assert_eq!(commands[0]["failed_step_sequence"], 1);
21329 assert_eq!(commands[0]["failed_activity_execution_id"], "activity-1");
21330
21331 let translated = worker
21332 .execute_workflow_task(workflow_task(
21333 "rust.redrive-translated",
21334 vec![failed],
21335 DEFAULT_CODEC,
21336 ))
21337 .expect("translated failure becomes a workflow command");
21338 assert_eq!(translated[0]["type"], "fail_workflow");
21339 assert!(translated[0].get("failed_step_sequence").is_none());
21340 assert!(translated[0].get("failed_activity_execution_id").is_none());
21341
21342 let without_identity = worker
21343 .execute_workflow_task(workflow_task(
21344 "rust.redrive",
21345 vec![history_event(
21346 "ActivityFailed",
21347 json!({"sequence": 1, "activity_type": "greet", "message": "failed"}),
21348 )],
21349 DEFAULT_CODEC,
21350 ))
21351 .expect("incomplete identity still fails the workflow");
21352 assert!(without_identity[0].get("failed_step_sequence").is_none());
21353
21354 let timed_out = worker
21355 .execute_workflow_task(workflow_task(
21356 "rust.redrive",
21357 vec![history_event(
21358 "ActivityTimedOut",
21359 json!({
21360 "sequence": 1,
21361 "activity_type": "greet",
21362 "activity_execution_id": "activity-timeout",
21363 }),
21364 )],
21365 DEFAULT_CODEC,
21366 ))
21367 .expect("timeout still fails the workflow");
21368 assert!(timed_out[0].get("failed_step_sequence").is_none());
21369 }
21370
21371 #[test]
21372 fn redriven_history_reuses_completed_prefix_and_reschedules_only_failed_step() {
21373 let client = Client::new("http://127.0.0.1:8080").expect("client");
21374 let mut worker = Worker::new(client, "rust-workers");
21375 worker.register_workflow("rust.redrive-successor", |ctx, _input| async move {
21376 let first = ctx.activity("first", json!([])).await?;
21377 let second = ctx.activity("second", json!([first.clone()])).await?;
21378 Ok(json!({"first": first, "second": second}))
21379 });
21380 let first = history_event(
21381 "ActivityCompleted",
21382 json!({
21383 "sequence": 1,
21384 "activity_type": "first",
21385 "result": encode_value_envelope(&json!("recorded"), DEFAULT_CODEC).expect("first result"),
21386 "payload_codec": DEFAULT_CODEC,
21387 "reused_from_run_id": "failed-run",
21388 "reused_activity_execution_id": "original-first",
21389 }),
21390 );
21391
21392 let retry = worker
21393 .execute_workflow_task(workflow_task(
21394 "rust.redrive-successor",
21395 vec![first.clone()],
21396 DEFAULT_CODEC,
21397 ))
21398 .expect("reused result replays before failed step");
21399 assert_eq!(retry.len(), 1);
21400 assert_eq!(retry[0]["type"], "schedule_activity");
21401 assert_eq!(retry[0]["activity_type"], "second");
21402 assert_eq!(
21403 decode_wire_value(&retry[0]["arguments"], DEFAULT_CODEC).expect("second arguments"),
21404 json!(["recorded"]),
21405 );
21406
21407 let second = history_event(
21408 "ActivityCompleted",
21409 json!({
21410 "sequence": 2,
21411 "activity_type": "second",
21412 "result": encode_value_envelope(&json!("retried"), DEFAULT_CODEC).expect("second result"),
21413 "payload_codec": DEFAULT_CODEC,
21414 }),
21415 );
21416 let completed = worker
21417 .execute_workflow_task(workflow_task(
21418 "rust.redrive-successor",
21419 vec![first, second],
21420 DEFAULT_CODEC,
21421 ))
21422 .expect("retried step completes the successor");
21423 assert_eq!(completed.len(), 1);
21424 assert_eq!(completed[0]["type"], "complete_workflow");
21425 assert_eq!(
21426 decode_wire_value(&completed[0]["result"], DEFAULT_CODEC).expect("workflow result"),
21427 json!({"first": "recorded", "second": "retried"}),
21428 );
21429 }
21430
21431 #[test]
21432 fn handler_error_cannot_hide_an_unconsumed_committed_side_effect() {
21433 let client = Client::new("http://127.0.0.1:8080").expect("client");
21434 let mut worker = Worker::new(client, "rust-workers");
21435 worker.register_workflow("rust.removed-side-effect", |_ctx, _input| async move {
21436 Err(Error::WorkerLoop("application failure".to_string()))
21437 });
21438 let result =
21439 encode_value_envelope(&json!("committed"), DEFAULT_CODEC).expect("side-effect result");
21440
21441 let error = worker
21442 .execute_workflow_task(workflow_task(
21443 "rust.removed-side-effect",
21444 vec![history_event(
21445 "SideEffectRecorded",
21446 json!({"sequence": 1, "result": result}),
21447 )],
21448 DEFAULT_CODEC,
21449 ))
21450 .expect_err("removed committed history must not become fail_workflow");
21451
21452 let Error::NonDeterministicReplay(failure) = error else {
21453 panic!("expected typed replay failure");
21454 };
21455 assert_eq!(failure.reason, "recorded_commands_unconsumed");
21456 assert_eq!(failure.sequence, Some(1));
21457 assert_eq!(failure.expected.as_deref(), Some("side effect"));
21458 }
21459
21460 #[test]
21461 fn replay_error_discards_side_effect_queued_before_incompatible_marker_check() {
21462 let client = Client::new("http://127.0.0.1:8080").expect("client");
21463 let mut worker = Worker::new(client, "rust-workers");
21464 worker.register_workflow(
21465 "rust.side-effect-before-marker-error",
21466 |ctx, _input| async move {
21467 assert_eq!(ctx.get_version("restart-safe", 1, 1)?, 1);
21468 let _: String = ctx.side_effect(|| "must-not-commit".to_string())?;
21469 ctx.get_version("restart-safe", 2, 2)?;
21470 Ok(Value::Null)
21471 },
21472 );
21473
21474 let error = worker
21475 .execute_workflow_task(workflow_task(
21476 "rust.side-effect-before-marker-error",
21477 vec![history_event(
21478 "VersionMarkerRecorded",
21479 json!({
21480 "sequence": 1,
21481 "change_id": "restart-safe",
21482 "version": 1,
21483 "min_supported": 1,
21484 "max_supported": 1,
21485 }),
21486 )],
21487 DEFAULT_CODEC,
21488 ))
21489 .expect_err("replay error must return no queued workflow commands");
21490
21491 let Error::NonDeterministicReplay(failure) = error else {
21492 panic!("expected typed replay failure");
21493 };
21494 assert_eq!(failure.reason, "version_marker_incompatible_range");
21495 assert_eq!(failure.sequence, Some(1));
21496 }
21497
21498 #[test]
21499 fn workflow_task_replay_keeps_recorded_unfired_timer_pending_without_rescheduling() {
21500 let client = Client::new("http://127.0.0.1:8080").expect("client");
21501 let mut worker = Worker::new(client, "rust-workers");
21502 worker.register_workflow("rust.timer.pending", |ctx, _input| async move {
21503 ctx.sleep(Duration::from_secs(5)).await?;
21504 Ok(json!({"status": "timer fired"}))
21505 });
21506
21507 let task = WorkflowTask {
21508 task_id: "wft-rust-timer-pending".to_string(),
21509 workflow_command_id: None,
21510 workflow_id: Some("wf-rust-timer".to_string()),
21511 run_id: Some("run-rust-timer".to_string()),
21512 workflow_type: "rust.timer.pending".to_string(),
21513 cancel_requested: false,
21514 payload_codec: DEFAULT_CODEC.to_string(),
21515 arguments: Some(
21516 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
21517 ),
21518 history_events: vec![history_event(
21519 "TimerScheduled",
21520 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
21521 )],
21522 total_history_events: Some(1),
21523 history_size_bytes: None,
21524 continue_as_new_recommended: None,
21525 history_budget_pressure: None,
21526 next_history_page_token: None,
21527 workflow_task_attempt: 1,
21528 workflow_signal_id: None,
21529 signal_name: None,
21530 signal_arguments: None,
21531 workflow_update_id: None,
21532 update_name: None,
21533 lease_owner: Some("rust-worker".to_string()),
21534 };
21535
21536 for _redelivery_or_restart in 0..2 {
21537 let commands = worker
21538 .execute_workflow_task(task.clone())
21539 .expect("recorded timer remains pending");
21540 assert!(
21541 commands.is_empty(),
21542 "recorded timer must not be rescheduled"
21543 );
21544 }
21545 }
21546
21547 #[test]
21548 fn workflow_task_rejects_recorded_command_removed_from_workflow_code() {
21549 let client = Client::new("http://127.0.0.1:8080").expect("client");
21550 let mut worker = Worker::new(client, "rust-workers");
21551 worker.register_workflow("rust.timer.removed", |_ctx, _input| async move {
21552 Ok(json!({"status": "completed"}))
21553 });
21554 let task = WorkflowTask {
21555 task_id: "wft-rust-timer-removed".to_string(),
21556 workflow_command_id: None,
21557 workflow_id: Some("wf-rust-timer".to_string()),
21558 run_id: Some("run-rust-timer".to_string()),
21559 workflow_type: "rust.timer.removed".to_string(),
21560 cancel_requested: false,
21561 payload_codec: DEFAULT_CODEC.to_string(),
21562 arguments: Some(
21563 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
21564 ),
21565 history_events: vec![
21566 history_event(
21567 "TimerScheduled",
21568 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
21569 ),
21570 history_event(
21571 "TimerFired",
21572 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
21573 ),
21574 ],
21575 total_history_events: Some(2),
21576 history_size_bytes: None,
21577 continue_as_new_recommended: None,
21578 history_budget_pressure: None,
21579 next_history_page_token: None,
21580 workflow_task_attempt: 1,
21581 workflow_signal_id: None,
21582 signal_name: None,
21583 signal_arguments: None,
21584 workflow_update_id: None,
21585 update_name: None,
21586 lease_owner: Some("rust-worker".to_string()),
21587 };
21588
21589 let Error::NonDeterministicReplay(failure) = worker
21590 .execute_workflow_task(task)
21591 .expect_err("removed timer must fail replay")
21592 else {
21593 panic!("expected typed replay failure");
21594 };
21595 assert_eq!(failure.reason, "recorded_commands_unconsumed");
21596 assert_eq!(failure.sequence, Some(1));
21597 }
21598
21599 #[test]
21600 fn workflow_context_emits_explicit_child_workflow_contract() {
21601 let ctx = WorkflowContext {
21602 state: Arc::new(Mutex::new(
21603 WorkflowState::new_with_identity(
21604 Vec::new(),
21605 Some("wf-parent".to_string()),
21606 Some("run-parent".to_string()),
21607 "parent-workers".to_string(),
21608 DEFAULT_CODEC.to_string(),
21609 None,
21610 )
21611 .expect("workflow state"),
21612 )),
21613 };
21614 let options = ChildWorkflowOptions::new("python-workers")
21615 .parent_close_policy(ParentClosePolicy::RequestCancel)
21616 .retry_policy(ChildWorkflowRetryPolicy {
21617 max_attempts: Some(3),
21618 backoff_seconds: vec![1, 5],
21619 non_retryable_error_types: vec!["ValidationError".to_string()],
21620 })
21621 .execution_timeout_seconds(600)
21622 .run_timeout_seconds(120);
21623 let mut call = Box::pin(ctx.start_child_workflow(
21624 "python.fulfil-order",
21625 options,
21626 json!([{"order_id": "order-42"}]),
21627 ));
21628 let mut task_context = TaskContext::from_waker(noop_waker_ref());
21629
21630 assert!(matches!(
21631 call.as_mut().poll(&mut task_context),
21632 Poll::Pending
21633 ));
21634 let commands = ctx.take_commands().expect("commands");
21635 assert_eq!(commands.len(), 1);
21636 let command = &commands[0];
21637 assert_eq!(command["type"], "start_child_workflow");
21638 assert_eq!(command["workflow_type"], "python.fulfil-order");
21639 assert_eq!(command["queue"], "python-workers");
21640 assert_eq!(command["parent_close_policy"], "request_cancel");
21641 assert_eq!(command["retry_policy"]["max_attempts"], 3);
21642 assert_eq!(command["execution_timeout_seconds"], 600);
21643 assert_eq!(command["run_timeout_seconds"], 120);
21644 assert_eq!(
21645 decode_wire_value(&command["arguments"], DEFAULT_CODEC).expect("child args"),
21646 json!([{"order_id": "order-42"}])
21647 );
21648 }
21649
21650 fn child_parent_worker() -> Worker {
21651 let client = Client::new("http://127.0.0.1:8080").expect("client");
21652 let mut worker = Worker::new(client, "rust-parent-workers");
21653 worker.register_workflow("rust.parent", |ctx, _input| async move {
21654 let child = ctx
21655 .start_child_workflow(
21656 "python.child",
21657 ChildWorkflowOptions::new("python-child-workers")
21658 .parent_close_policy(ParentClosePolicy::Terminate),
21659 json!([{"codec_probe": [1, true, "rust"]}]),
21660 )
21661 .await?;
21662 Ok(json!({
21663 "parent_workflow_id": child.parent.workflow_id,
21664 "parent_run_id": child.parent.run_id,
21665 "child_workflow_id": child.child.workflow_id,
21666 "child_run_id": child.child.run_id,
21667 "child_workflow_type": child.child_workflow_type,
21668 "result": child.result,
21669 }))
21670 });
21671 worker
21672 }
21673
21674 fn child_parent_task(event_type: &str, payload: Value) -> WorkflowTask {
21675 WorkflowTask {
21676 task_id: "wft-child-parent".to_string(),
21677 workflow_command_id: None,
21678 workflow_id: Some("wf-parent".to_string()),
21679 run_id: Some("run-parent".to_string()),
21680 workflow_type: "rust.parent".to_string(),
21681 cancel_requested: false,
21682 payload_codec: DEFAULT_CODEC.to_string(),
21683 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21684 history_events: vec![
21685 HistoryEvent {
21686 event_type: "ChildWorkflowScheduled".to_string(),
21687 payload: json!({
21688 "sequence": 1,
21689 "child_call_id": "call-child",
21690 "child_workflow_instance_id": "wf-child",
21691 "child_workflow_run_id": "run-child",
21692 "child_workflow_type": "python.child",
21693 }),
21694 raw: HashMap::new(),
21695 },
21696 HistoryEvent {
21697 event_type: event_type.to_string(),
21698 payload,
21699 raw: HashMap::new(),
21700 },
21701 ],
21702 total_history_events: Some(2),
21703 history_size_bytes: None,
21704 continue_as_new_recommended: None,
21705 history_budget_pressure: None,
21706 next_history_page_token: None,
21707 workflow_task_attempt: 1,
21708 workflow_signal_id: None,
21709 signal_name: None,
21710 signal_arguments: None,
21711 workflow_update_id: None,
21712 update_name: None,
21713 lease_owner: Some("rust-worker".to_string()),
21714 }
21715 }
21716
21717 #[test]
21718 fn committed_child_result_replays_without_starting_a_duplicate() {
21719 let worker = child_parent_worker();
21720 let task = child_parent_task(
21721 "ChildRunCompleted",
21722 json!({
21723 "sequence": 1,
21724 "child_call_id": "call-child",
21725 "child_workflow_instance_id": "wf-child",
21726 "child_workflow_run_id": "run-child",
21727 "child_workflow_type": "python.child",
21728 "payload_codec": DEFAULT_CODEC,
21729 "result": fixture_envelope(json!({"from":"python","ok":true})),
21730 }),
21731 );
21732
21733 for _restart in 0..2 {
21734 let commands = worker
21735 .execute_workflow_task(task.clone())
21736 .expect("replayed parent task");
21737 assert_eq!(commands.len(), 1);
21738 assert_eq!(commands[0]["type"], "complete_workflow");
21739 assert!(!commands
21740 .iter()
21741 .any(|command| command["type"] == "start_child_workflow"));
21742 let output =
21743 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
21744 assert_eq!(output["parent_workflow_id"], "wf-parent");
21745 assert_eq!(output["parent_run_id"], "run-parent");
21746 assert_eq!(output["child_workflow_id"], "wf-child");
21747 assert_eq!(output["child_run_id"], "run-child");
21748 assert_eq!(output["result"], json!({"from": "python", "ok": true}));
21749 }
21750 }
21751
21752 #[test]
21753 fn typed_child_arguments_and_results_survive_replay() {
21754 let client = Client::new("http://127.0.0.1:8080").expect("client");
21755 let mut worker = Worker::new(client, "rust-parent-workers");
21756 worker.register_workflow_avro_value("rust.typed-parent", |ctx, _input| async move {
21757 let child = ctx
21758 .start_child_workflow_avro_value(
21759 "python.typed-child",
21760 ChildWorkflowOptions::new("python-workers"),
21761 AvroValue::Array(vec![typed_fidelity_probe()]),
21762 )
21763 .await?;
21764 Ok(child.result)
21765 });
21766
21767 let initial = worker
21768 .execute_workflow_task(workflow_task(
21769 "rust.typed-parent",
21770 Vec::new(),
21771 DEFAULT_CODEC,
21772 ))
21773 .expect("typed child start");
21774 assert_eq!(initial[0]["type"], "start_child_workflow");
21775 assert_eq!(
21776 decode_wire_avro_value(&initial[0]["arguments"], DEFAULT_CODEC)
21777 .expect("typed child arguments"),
21778 AvroValue::Array(vec![typed_fidelity_probe()])
21779 );
21780
21781 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
21782 .expect("typed child result");
21783 let task = workflow_task(
21784 "rust.typed-parent",
21785 vec![
21786 history_event(
21787 "ChildWorkflowScheduled",
21788 json!({
21789 "sequence": 1,
21790 "child_call_id": "call-typed",
21791 "child_workflow_instance_id": "wf-child",
21792 "child_workflow_run_id": "run-child",
21793 "child_workflow_type": "python.typed-child",
21794 }),
21795 ),
21796 history_event(
21797 "ChildRunCompleted",
21798 json!({
21799 "sequence": 1,
21800 "child_call_id": "call-typed",
21801 "child_workflow_instance_id": "wf-child",
21802 "child_workflow_run_id": "run-child",
21803 "child_workflow_type": "python.typed-child",
21804 "payload_codec": DEFAULT_CODEC,
21805 "result": result,
21806 }),
21807 ),
21808 ],
21809 DEFAULT_CODEC,
21810 );
21811
21812 let commands = worker
21813 .execute_workflow_task(task)
21814 .expect("typed child replay");
21815 assert_eq!(commands[0]["type"], "complete_workflow");
21816 assert_eq!(
21817 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
21818 .expect("typed parent result"),
21819 typed_fidelity_probe()
21820 );
21821 }
21822
21823 #[test]
21824 fn pending_child_replays_after_restart_without_starting_a_duplicate() {
21825 let worker = child_parent_worker();
21826 let mut task = child_parent_task("unused", Value::Null);
21827 task.history_events.truncate(1);
21828 task.total_history_events = Some(1);
21829
21830 for _redelivery_or_restart in 0..2 {
21831 let commands = worker
21832 .execute_workflow_task(task.clone())
21833 .expect("recorded child remains pending");
21834 assert!(
21835 commands.is_empty(),
21836 "recorded pending child must not be started again"
21837 );
21838 }
21839 }
21840
21841 #[test]
21842 fn child_cancellation_becomes_stable_parent_failure_command() {
21843 let worker = child_parent_worker();
21844 let task = child_parent_task(
21845 "ChildRunCancelled",
21846 json!({
21847 "sequence": 1,
21848 "child_workflow_instance_id": "wf-child",
21849 "child_workflow_run_id": "run-child",
21850 "child_workflow_type": "python.child",
21851 "failure_id": "failure-child",
21852 "failure_category": "cancelled",
21853 "message": "cancelled by parent-close policy",
21854 }),
21855 );
21856
21857 let commands = worker
21858 .execute_workflow_task(task)
21859 .expect("parent settlement");
21860 assert_eq!(commands.len(), 1);
21861 assert_eq!(commands[0]["type"], "fail_workflow");
21862 assert_eq!(commands[0]["exception_type"], "ChildWorkflowCancelled");
21863 assert_eq!(
21864 commands[0]["exception"]["properties"]["reason"],
21865 "cancelled"
21866 );
21867 assert_eq!(
21868 commands[0]["exception"]["properties"]["child_workflow_run_id"],
21869 "run-child"
21870 );
21871 }
21872
21873 #[test]
21874 fn workflow_can_handle_typed_child_failure() {
21875 let client = Client::new("http://127.0.0.1:8080").expect("client");
21876 let mut worker = Worker::new(client, "rust-parent-workers");
21877 worker.register_workflow("rust.handled-parent", |ctx, _input| async move {
21878 match ctx
21879 .start_child_workflow(
21880 "python.child",
21881 ChildWorkflowOptions::new("python-child-workers"),
21882 json!([]),
21883 )
21884 .await
21885 {
21886 Err(Error::ChildWorkflowFailed(failure)) => Ok(json!({
21887 "reason": failure.reason,
21888 "failure_id": failure.failure_id,
21889 "exception_class": failure.exception_class,
21890 "child_run_id": failure.child_workflow_run_id,
21891 })),
21892 Err(error) => Err(error),
21893 Ok(_) => Err(Error::WorkerLoop(
21894 "child unexpectedly succeeded".to_string(),
21895 )),
21896 }
21897 });
21898 let mut task = child_parent_task(
21899 "ChildRunFailed",
21900 json!({
21901 "sequence": 1,
21902 "child_workflow_instance_id": "wf-child",
21903 "child_workflow_run_id": "run-child",
21904 "child_workflow_type": "python.child",
21905 "failure_id": "failure-child",
21906 "failure_category": "child_workflow",
21907 "message": "payment rejected",
21908 "exception": {
21909 "type": "PaymentRejected",
21910 "class": "payments.PaymentRejected",
21911 "message": "payment rejected"
21912 }
21913 }),
21914 );
21915 task.workflow_type = "rust.handled-parent".to_string();
21916
21917 let commands = worker.execute_workflow_task(task).expect("handled failure");
21918 assert_eq!(commands[0]["type"], "complete_workflow");
21919 let output =
21920 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
21921 assert_eq!(output["reason"], "child_workflow");
21922 assert_eq!(output["failure_id"], "failure-child");
21923 assert_eq!(output["exception_class"], "payments.PaymentRejected");
21924 assert_eq!(output["child_run_id"], "run-child");
21925 }
21926
21927 #[test]
21928 fn rust_hello_world_uses_signal_arguments_from_resume_payload() {
21929 let client = Client::new("http://127.0.0.1:8080").expect("client");
21930 let mut worker = Worker::new(client, "rust-workers");
21931
21932 worker.register_workflow("rust.hello_workflow", |ctx, _input| async move {
21933 let signal = ctx.wait_signal("start").await?;
21934 let name = signal
21935 .first()
21936 .and_then(|value| value.as_str())
21937 .unwrap_or("world");
21938 let greeting = ctx.activity("rust.hello_activity", json!([name])).await?;
21939 Ok(json!({
21940 "greeting": greeting,
21941 "language": "rust"
21942 }))
21943 });
21944
21945 let signal_arguments =
21946 encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC).expect("signal arguments");
21947 let task = WorkflowTask {
21948 task_id: "wft-rust-signal-1".to_string(),
21949 workflow_command_id: None,
21950 workflow_id: Some("wf-rust-hello".to_string()),
21951 run_id: Some("run-rust-hello".to_string()),
21952 workflow_type: "rust.hello_workflow".to_string(),
21953 cancel_requested: false,
21954 payload_codec: DEFAULT_CODEC.to_string(),
21955 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21956 history_events: vec![HistoryEvent {
21957 event_type: "SignalReceived".to_string(),
21958 payload: json!({
21959 "signal_id": "sig-rust-1",
21960 "signal_name": "start"
21961 }),
21962 raw: HashMap::new(),
21963 }],
21964 total_history_events: Some(1),
21965 history_size_bytes: None,
21966 continue_as_new_recommended: None,
21967 history_budget_pressure: None,
21968 next_history_page_token: None,
21969 workflow_task_attempt: 1,
21970 workflow_signal_id: Some("sig-rust-1".to_string()),
21971 signal_name: Some("start".to_string()),
21972 signal_arguments: Some(signal_arguments),
21973 workflow_update_id: None,
21974 update_name: None,
21975 lease_owner: Some("rust-worker".to_string()),
21976 };
21977
21978 let commands = worker.execute_workflow_task(task).expect("workflow task");
21979
21980 assert_eq!(commands.len(), 1);
21981 assert_eq!(commands[0]["type"], "schedule_activity");
21982 assert_eq!(commands[0]["activity_type"], "rust.hello_activity");
21983 assert_eq!(
21984 decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC).expect("activity args"),
21985 json!(["Rust"])
21986 );
21987 }
21988
21989 #[test]
21990 fn workflow_task_appends_paginated_history_events() {
21991 let mut task = WorkflowTask {
21992 task_id: "wft-rust-pages-1".to_string(),
21993 workflow_command_id: None,
21994 workflow_id: Some("wf-rust-pages".to_string()),
21995 run_id: Some("run-rust-pages".to_string()),
21996 workflow_type: "rust.hello_workflow".to_string(),
21997 cancel_requested: false,
21998 payload_codec: DEFAULT_CODEC.to_string(),
21999 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
22000 history_events: vec![HistoryEvent {
22001 event_type: "WorkflowStarted".to_string(),
22002 payload: json!({}),
22003 raw: HashMap::new(),
22004 }],
22005 total_history_events: Some(3),
22006 history_size_bytes: None,
22007 continue_as_new_recommended: None,
22008 history_budget_pressure: None,
22009 next_history_page_token: Some("MQ==".to_string()),
22010 workflow_task_attempt: 1,
22011 workflow_signal_id: None,
22012 signal_name: None,
22013 signal_arguments: None,
22014 workflow_update_id: None,
22015 update_name: None,
22016 lease_owner: Some("rust-worker".to_string()),
22017 };
22018
22019 task.append_history_page(WorkflowTaskHistoryPage {
22020 history_events: vec![
22021 HistoryEvent {
22022 event_type: "SignalReceived".to_string(),
22023 payload: json!({
22024 "signal_id": "sig-rust-1",
22025 "signal_name": "start",
22026 "arguments": encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC)
22027 .expect("signal arguments")
22028 }),
22029 raw: HashMap::new(),
22030 },
22031 HistoryEvent {
22032 event_type: "MarkerRecorded".to_string(),
22033 payload: json!({"sequence": 3}),
22034 raw: HashMap::new(),
22035 },
22036 ],
22037 total_history_events: Some(3),
22038 next_history_page_token: None,
22039 });
22040
22041 assert_eq!(task.history_events.len(), 3);
22042 assert_eq!(task.total_history_events, Some(3));
22043 assert_eq!(task.next_history_page_token, None);
22044
22045 let signal = task
22046 .history_events
22047 .iter()
22048 .find(|event| event.event_type == "SignalReceived")
22049 .expect("signal event");
22050 assert_eq!(
22051 decode_signal_event_arguments(signal, DEFAULT_CODEC).expect("signal arguments"),
22052 vec![AvroValue::String("Rust".to_string())]
22053 );
22054 }
22055
22056 #[tokio::test]
22057 async fn query_handler_reads_ordered_cross_codec_signals_without_commands() {
22058 let client = Client::new("http://127.0.0.1:8080").expect("client");
22059 let mut worker = Worker::new(client, "rust-workers");
22060 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
22061 worker.register_query("counter", "current", |ctx, _args| async move {
22062 let mut count = 0_i64;
22063 for signal in ctx.signal_events() {
22064 let value = signal
22065 .arguments
22066 .first()
22067 .and_then(Value::as_i64)
22068 .unwrap_or_default();
22069 match signal.name.as_str() {
22070 "increment" => count += value,
22071 "set" => count = value,
22072 _ => {}
22073 }
22074 }
22075 Ok(json!(count))
22076 });
22077
22078 let task = QueryTask {
22079 query_task_id: "query-rust-counter".to_string(),
22080 query_task_attempt: 1,
22081 lease_owner: Some("rust-worker".to_string()),
22082 workflow_id: Some("counter-1".to_string()),
22083 run_id: Some("run-counter-1".to_string()),
22084 workflow_type: "counter".to_string(),
22085 query_name: "current".to_string(),
22086 payload_codec: DEFAULT_CODEC.to_string(),
22087 workflow_arguments: Some(
22088 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
22089 ),
22090 query_arguments: Some(
22091 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("query arguments"),
22092 ),
22093 history_events: vec![
22094 HistoryEvent {
22095 event_type: "SignalReceived".to_string(),
22096 payload: json!({
22097 "signal_id": "php-signal-1",
22098 "signal_name": "increment",
22099 "workflow_sequence": 1,
22100 "payload_codec": DEFAULT_CODEC,
22101 "arguments": encode_value_envelope(&json!([3]), DEFAULT_CODEC).expect("php avro signal")
22102 }),
22103 raw: HashMap::new(),
22104 },
22105 HistoryEvent {
22106 event_type: "SignalReceived".to_string(),
22107 payload: json!({
22108 "signal_id": "python-signal-2",
22109 "signal_name": "increment",
22110 "workflow_sequence": 2,
22111 "payload_codec": DEFAULT_CODEC,
22112 "arguments": encode_value_envelope(&json!([5]), DEFAULT_CODEC).expect("python avro signal")
22113 }),
22114 raw: HashMap::new(),
22115 },
22116 HistoryEvent {
22117 event_type: "SignalReceived".to_string(),
22118 payload: json!({
22119 "signal_id": "rust-signal-3",
22120 "signal_name": "set",
22121 "workflow_sequence": 3,
22122 "payload_codec": DEFAULT_CODEC,
22123 "arguments": encode_value_envelope(&json!([0]), DEFAULT_CODEC).expect("rust avro signal")
22124 }),
22125 raw: HashMap::new(),
22126 },
22127 ],
22128 history_export: None,
22129 run_status: Some("completed".to_string()),
22130 };
22131
22132 let result = worker.execute_query_task(task).await.expect("query result");
22133 assert_eq!(result.into_json().expect("query projection"), json!(0));
22134 }
22135
22136 #[tokio::test]
22137 async fn replayed_queries_read_running_completed_and_cold_restarted_instance_state() {
22138 let worker = replay_counter_worker();
22139 let running_history = json!([
22140 {
22141 "type": "ActivityCompleted",
22142 "payload": {
22143 "sequence": 1,
22144 "activity_type": "load-counter",
22145 "payload_codec": DEFAULT_CODEC,
22146 "result": fixture_envelope(json!("loaded"))
22147 }
22148 },
22149 {
22150 "type": "SignalWaitOpened",
22151 "payload": {
22152 "sequence": 3,
22153 "signal_name": "increment"
22154 }
22155 },
22156 {
22157 "type": "SignalReceived",
22158 "payload": {
22159 "signal_id": "signal-3",
22160 "signal_name": "increment",
22161 "workflow_sequence": 2,
22162 "payload_codec": DEFAULT_CODEC,
22163 "arguments": fixture_envelope(json!([3]))
22164 }
22165 },
22166 {
22167 "type": "SignalApplied",
22168 "payload": {
22169 "sequence": 3,
22170 "signal_id": "signal-3",
22171 "signal_name": "increment",
22172 "payload_codec": DEFAULT_CODEC,
22173 "value": fixture_envelope(json!([3]))
22174 }
22175 }
22176 ]);
22177
22178 let running = worker
22179 .execute_query_task(replay_counter_query(
22180 "current",
22181 running_history.clone(),
22182 "running",
22183 ))
22184 .await
22185 .expect("running replay query");
22186 assert_eq!(
22187 running.clone().into_json().expect("query projection"),
22188 json!({"loaded": "loaded", "count": 3, "finished": false})
22189 );
22190
22191 let detached = worker
22192 .execute_query_task(replay_counter_query(
22193 "detached-mutation",
22194 running_history.clone(),
22195 "running",
22196 ))
22197 .await
22198 .expect("query mutates only its detached state clone");
22199 assert_eq!(detached.into_json().expect("query projection"), json!(999));
22200 let failed = worker
22201 .execute_query_task(replay_counter_query(
22202 "failed-mutation",
22203 running_history.clone(),
22204 "running",
22205 ))
22206 .await
22207 .expect_err("failed query");
22208 assert_eq!(failed.reason, "query_rejected");
22209 let unchanged = worker
22210 .execute_query_task(replay_counter_query("current", running_history, "running"))
22211 .await
22212 .expect("later query reconstructs unchanged state");
22213 assert_eq!(unchanged, running);
22214
22215 let restarted_worker = replay_counter_worker();
22216 let empty_arguments = fixture_envelope(json!([]));
22217 let loaded_result = fixture_envelope(json!("loaded"));
22218 let signal_three = fixture_blob(json!([3]));
22219 let signal_five = fixture_blob(json!([5]));
22220 let restarted_task: QueryTask = serde_json::from_value(json!({
22221 "query_task_id": "query-after-restart",
22222 "workflow_id": "counter-1",
22223 "run_id": "run-counter-1",
22224 "workflow_type": "replay-counter",
22225 "query_name": "current",
22226 "payload_codec": DEFAULT_CODEC,
22227 "workflow_arguments": empty_arguments.clone(),
22228 "query_arguments": empty_arguments,
22229 "history_events": [],
22230 "history_export": {
22231 "payloads": {"codec": DEFAULT_CODEC},
22232 "history_events": [
22233 {
22234 "type": "ActivityCompleted",
22235 "payload": {
22236 "sequence": 1,
22237 "activity_type": "load-counter",
22238 "payload_codec": DEFAULT_CODEC,
22239 "result": null
22240 }
22241 },
22242 {
22243 "type": "SignalWaitOpened",
22244 "payload": {
22245 "sequence": 3,
22246 "signal_name": "increment"
22247 }
22248 },
22249 {
22250 "type": "SignalReceived",
22251 "payload": {
22252 "signal_id": "signal-3",
22253 "signal_name": "increment",
22254 "workflow_sequence": 2
22255 }
22256 },
22257 {
22258 "type": "SignalApplied",
22259 "payload": {
22260 "sequence": 3,
22261 "signal_id": "signal-3",
22262 "signal_name": "increment"
22263 }
22264 },
22265 {
22266 "type": "SignalWaitOpened",
22267 "payload": {
22268 "sequence": 5,
22269 "signal_name": "increment"
22270 }
22271 },
22272 {
22273 "type": "SignalReceived",
22274 "payload": {
22275 "signal_id": "signal-5",
22276 "signal_name": "increment",
22277 "workflow_sequence": 4
22278 }
22279 },
22280 {
22281 "type": "SignalApplied",
22282 "payload": {
22283 "sequence": 5,
22284 "signal_id": "signal-5",
22285 "signal_name": "increment"
22286 }
22287 }
22288 ],
22289 "activities": [{
22290 "sequence": 1,
22291 "activity_type": "load-counter",
22292 "payload_codec": DEFAULT_CODEC,
22293 "result": loaded_result
22294 }],
22295 "signals": [
22296 {
22297 "id": "signal-3",
22298 "name": "increment",
22299 "workflow_sequence": 2,
22300 "payload_codec": DEFAULT_CODEC,
22301 "arguments": signal_three
22302 },
22303 {
22304 "id": "signal-5",
22305 "name": "increment",
22306 "workflow_sequence": 4,
22307 "payload_codec": DEFAULT_CODEC,
22308 "arguments": signal_five
22309 }
22310 ]
22311 },
22312 "run_status": "completed"
22313 }))
22314 .expect("cold replay query task");
22315 let completed = restarted_worker
22316 .execute_query_task(restarted_task)
22317 .await
22318 .expect("completed cold replay query");
22319 assert_eq!(
22320 completed.into_json().expect("query projection"),
22321 json!({"loaded": "loaded", "count": 8, "finished": true})
22322 );
22323 }
22324
22325 #[tokio::test]
22326 async fn replayed_query_replay_failures_are_machine_readable() {
22327 let worker = replay_counter_worker();
22328 let task = replay_counter_query(
22329 "current",
22330 json!([{
22331 "type": "ActivityCompleted",
22332 "payload": {
22333 "sequence": 1,
22334 "payload_codec": DEFAULT_CODEC,
22335 "result": {"codec": DEFAULT_CODEC, "blob": "{"}
22336 }
22337 }]),
22338 "running",
22339 );
22340 let failure = worker
22341 .execute_query_task(task)
22342 .await
22343 .expect_err("invalid replay history payload");
22344 assert_eq!(failure.reason, "query_payload_decode_failed");
22345 assert_eq!(failure.failure_type, "QueryPayloadDecodeFailed");
22346 assert!(failure.message.contains("invalid_payload_framing"));
22347 }
22348
22349 #[tokio::test]
22350 async fn query_task_restores_compact_history_from_export() {
22351 let client = Client::new("http://127.0.0.1:8080").expect("client");
22352 let mut worker = Worker::new(client, "rust-workers");
22353 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
22354 worker.register_query("counter", "current", |ctx, _args| async move {
22355 Ok(json!(ctx.signals("increment")[0][0]))
22356 });
22357 let empty_arguments = fixture_envelope(json!([]));
22358 let exported_signal = fixture_blob(json!([9]));
22359 let task: QueryTask = serde_json::from_value(json!({
22360 "query_task_id": "query-export",
22361 "workflow_type": "counter",
22362 "query_name": "current",
22363 "payload_codec": DEFAULT_CODEC,
22364 "workflow_arguments": empty_arguments.clone(),
22365 "query_arguments": empty_arguments,
22366 "history_events": [],
22367 "history_export": {
22368 "payloads": {"codec": DEFAULT_CODEC},
22369 "history_events": [{
22370 "type": "SignalReceived",
22371 "payload": {"signal_id": "signal-export", "signal_name": "increment"}
22372 }],
22373 "signals": [{
22374 "id": "signal-export",
22375 "name": "increment",
22376 "status": "applied",
22377 "workflow_sequence": 1,
22378 "payload_codec": DEFAULT_CODEC,
22379 "arguments": exported_signal
22380 }]
22381 }
22382 }))
22383 .expect("query task");
22384
22385 let result = worker.execute_query_task(task).await.expect("query result");
22386 assert_eq!(result.into_json().expect("query projection"), json!(9));
22387 }
22388
22389 #[tokio::test]
22390 async fn query_task_failures_have_stable_reasons() {
22391 let client = Client::new("http://127.0.0.1:8080").expect("client");
22392 let mut worker = Worker::new(client, "rust-workers");
22393 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
22394 worker.register_query(
22395 "counter",
22396 "current",
22397 |_ctx, _args| async move { Ok(json!(0)) },
22398 );
22399
22400 let base_task = QueryTask {
22401 query_task_id: "query-errors".to_string(),
22402 query_task_attempt: 1,
22403 lease_owner: None,
22404 workflow_id: Some("counter-errors".to_string()),
22405 run_id: Some("run-errors".to_string()),
22406 workflow_type: "counter".to_string(),
22407 query_name: "missing".to_string(),
22408 payload_codec: DEFAULT_CODEC.to_string(),
22409 workflow_arguments: Some(fixture_envelope(json!([]))),
22410 query_arguments: Some(fixture_envelope(json!([]))),
22411 history_events: Vec::new(),
22412 history_export: None,
22413 run_status: Some("running".to_string()),
22414 };
22415
22416 let unknown = worker
22417 .execute_query_task(base_task.clone())
22418 .await
22419 .expect_err("unknown query");
22420 assert_eq!(unknown.reason, "rejected_unknown_query");
22421
22422 let mut malformed = base_task;
22423 malformed.query_name = "current".to_string();
22424 malformed.query_arguments = Some(json!({"codec": DEFAULT_CODEC, "blob": "{"}));
22425 let malformed = worker
22426 .execute_query_task(malformed)
22427 .await
22428 .expect_err("malformed payload");
22429 assert_eq!(malformed.reason, "query_payload_decode_failed");
22430
22431 let client = Client::new("http://127.0.0.1:8080").expect("client");
22432 let mut unavailable_worker = Worker::new(client, "rust-workers");
22433 unavailable_worker
22434 .register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
22435 let empty_arguments = fixture_envelope(json!([]));
22436 let unavailable_task: QueryTask = serde_json::from_value(json!({
22437 "query_task_id": "query-unavailable",
22438 "workflow_type": "counter",
22439 "query_name": "current",
22440 "payload_codec": DEFAULT_CODEC,
22441 "workflow_arguments": empty_arguments.clone(),
22442 "query_arguments": empty_arguments
22443 }))
22444 .expect("query task");
22445 let unavailable = unavailable_worker
22446 .execute_query_task(unavailable_task)
22447 .await
22448 .expect_err("query handler unavailable");
22449 assert_eq!(unavailable.reason, "query_handler_unavailable");
22450 }
22451
22452 #[tokio::test]
22453 async fn client_query_decodes_result_and_typed_failure() {
22454 let server = MockWorkerServer::start();
22455 let client = Client::builder(server.base_url())
22456 .timeout(Duration::from_secs(2))
22457 .build()
22458 .expect("client");
22459
22460 let result = client
22461 .query_workflow("counter-1", "current", json!([]))
22462 .await
22463 .expect("query result");
22464 assert_eq!(result, json!({"count": 8}));
22465
22466 let error = client
22467 .query_workflow("counter-1", "missing", json!([]))
22468 .await
22469 .expect_err("unknown query");
22470 let Error::QueryFailed(failure) = error else {
22471 panic!("expected typed query failure");
22472 };
22473 assert_eq!(failure.status, 404);
22474 assert_eq!(failure.reason, "rejected_unknown_query");
22475 }
22476
22477 #[tokio::test]
22478 async fn public_client_surfaces_send_and_receive_lossless_avro_values() {
22479 let server = MockWorkerServer::start();
22480 let client = Client::builder(server.base_url())
22481 .timeout(Duration::from_secs(2))
22482 .build()
22483 .expect("client");
22484 let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
22485
22486 client
22487 .start_workflow(
22488 "typed.echo",
22489 "rust-workers",
22490 "typed-start",
22491 arguments.clone(),
22492 )
22493 .await
22494 .expect("typed workflow start");
22495 assert_eq!(
22496 decode_wire_avro_value(
22497 &server.request_body("/api/workflows")["input"],
22498 DEFAULT_CODEC,
22499 )
22500 .expect("typed start input"),
22501 arguments
22502 );
22503
22504 client
22505 .signal_workflow("typed-1", "changed", arguments.clone())
22506 .await
22507 .expect("typed signal");
22508 assert_eq!(
22509 decode_wire_avro_value(
22510 &server.request_body("/api/workflows/typed-1/signal/changed")["input"],
22511 DEFAULT_CODEC,
22512 )
22513 .expect("typed signal input"),
22514 arguments
22515 );
22516
22517 assert_eq!(
22518 client
22519 .query_workflow_avro_value("typed-1", "inspect", arguments.clone())
22520 .await
22521 .expect("typed query"),
22522 typed_fidelity_probe()
22523 );
22524 assert_eq!(
22525 decode_wire_avro_value(
22526 &server.request_body("/api/workflows/typed-1/query/inspect")["input"],
22527 DEFAULT_CODEC,
22528 )
22529 .expect("typed query input"),
22530 arguments
22531 );
22532
22533 assert_eq!(
22534 client
22535 .update_workflow_avro_value(
22536 "typed-1",
22537 "replace",
22538 arguments.clone(),
22539 Some("typed-request"),
22540 )
22541 .await
22542 .expect("typed update"),
22543 typed_fidelity_probe()
22544 );
22545 let update = server.request_body("/api/workflows/typed-1/update/replace");
22546 assert_eq!(update["request_id"], "typed-request");
22547 assert_eq!(
22548 decode_wire_avro_value(&update["input"], DEFAULT_CODEC).expect("typed update input"),
22549 arguments
22550 );
22551
22552 let handle = WorkflowHandle {
22553 client: client.clone(),
22554 workflow_id: "typed-1".to_string(),
22555 run_id: Some("run-typed-1".to_string()),
22556 workflow_type: "typed.echo".to_string(),
22557 };
22558 assert_eq!(
22559 handle
22560 .result_avro_value(WorkflowResultOptions::default())
22561 .await
22562 .expect("typed workflow result"),
22563 typed_fidelity_probe()
22564 );
22565
22566 client
22567 .complete_activity_task(
22568 "activity-typed",
22569 "attempt-typed",
22570 "rust-worker",
22571 typed_fidelity_probe(),
22572 DEFAULT_CODEC,
22573 )
22574 .await
22575 .expect("typed activity completion");
22576 assert_eq!(
22577 decode_wire_avro_value(
22578 &server.request_body("/api/worker/activity-tasks/activity-typed/complete")
22579 ["result"],
22580 DEFAULT_CODEC,
22581 )
22582 .expect("typed activity result"),
22583 typed_fidelity_probe()
22584 );
22585 client
22586 .fail_activity_task(
22587 "activity-typed",
22588 "attempt-typed",
22589 "rust-worker",
22590 "typed failure",
22591 true,
22592 )
22593 .await
22594 .expect("activity failure");
22595 }
22596
22597 #[tokio::test]
22598 async fn lifecycle_commands_support_instance_and_selected_run_targets() {
22599 let server = MockWorkerServer::start();
22600 let client = Client::builder(server.base_url())
22601 .timeout(Duration::from_secs(2))
22602 .build()
22603 .expect("client");
22604
22605 let options = WorkflowCommandOptions::new()
22606 .reason("cleanup requested")
22607 .request_id("cancel-17");
22608 let cancelled = client
22609 .cancel_workflow("wf-lifecycle", options)
22610 .await
22611 .expect("instance cancellation");
22612 assert_eq!(cancelled.command, WorkflowCommandKind::Cancel);
22613 assert_eq!(cancelled.run_id.as_deref(), Some("run-current"));
22614 assert_eq!(cancelled.outcome.as_deref(), Some("cancelled"));
22615 assert_eq!(
22616 server.request_body("/api/workflows/wf-lifecycle/cancel"),
22617 json!({"reason":"cleanup requested","request_id":"cancel-17"})
22618 );
22619
22620 let terminated = client
22621 .terminate_workflow(
22622 "wf-lifecycle",
22623 WorkflowCommandOptions::new().reason("forced stop"),
22624 )
22625 .await
22626 .expect("instance termination");
22627 assert_eq!(terminated.command, WorkflowCommandKind::Terminate);
22628 assert_eq!(terminated.outcome.as_deref(), Some("terminated"));
22629
22630 client
22631 .cancel_workflow_run(
22632 "wf-lifecycle",
22633 "run-current",
22634 WorkflowCommandOptions::default(),
22635 )
22636 .await
22637 .expect("selected run cancellation");
22638 client
22639 .terminate_workflow_run(
22640 "wf-lifecycle",
22641 "run-current",
22642 WorkflowCommandOptions::default(),
22643 )
22644 .await
22645 .expect("selected run termination");
22646
22647 for (command, error) in [
22648 (
22649 WorkflowCommandKind::Cancel,
22650 client
22651 .cancel_workflow_run(
22652 "wf-lifecycle",
22653 "run-stale",
22654 WorkflowCommandOptions::default(),
22655 )
22656 .await
22657 .expect_err("stale cancellation must be rejected"),
22658 ),
22659 (
22660 WorkflowCommandKind::Terminate,
22661 client
22662 .terminate_workflow_run(
22663 "wf-lifecycle",
22664 "run-stale",
22665 WorkflowCommandOptions::default(),
22666 )
22667 .await
22668 .expect_err("stale termination must be rejected"),
22669 ),
22670 ] {
22671 let Error::WorkflowCommandRejected(rejection) = error else {
22672 panic!("expected typed command rejection");
22673 };
22674 assert_eq!(rejection.command, command);
22675 assert_eq!(rejection.status, 409);
22676 assert_eq!(rejection.reason, "historical_run_command_rejected");
22677 assert_eq!(rejection.run_id.as_deref(), Some("run-stale"));
22678 assert_eq!(rejection.target_scope.as_deref(), Some("run"));
22679 }
22680 }
22681
22682 #[tokio::test]
22683 async fn redrive_targets_a_failed_run_and_returns_successor_identity() {
22684 let server = MockWorkerServer::start();
22685 let client = Client::builder(server.base_url())
22686 .timeout(Duration::from_secs(2))
22687 .build()
22688 .expect("client");
22689
22690 let result = client
22691 .redrive_workflow_run("wf-lifecycle", "run-failed", Some("retry-1"))
22692 .await
22693 .expect("redrive accepted");
22694 assert_eq!(result.workflow_id, "wf-lifecycle");
22695 assert_eq!(result.source_run_id, "run-failed");
22696 assert_eq!(result.run_id, "run-successor");
22697 assert_eq!(result.resume_step_sequence, 2);
22698 assert_eq!(
22699 server.request_body("/api/workflows/wf-lifecycle/runs/run-failed/redrive"),
22700 json!({"request_id":"retry-1"})
22701 );
22702
22703 let repeated = client
22704 .redrive_workflow_run("wf-lifecycle", "run-failed-existing", Some("retry-1"))
22705 .await
22706 .expect("idempotent redrive response accepted");
22707 assert_eq!(repeated.run_id, "run-successor");
22708
22709 let error = client
22710 .redrive_workflow_run("wf-lifecycle", "run-completed", None)
22711 .await
22712 .expect_err("completed run must be rejected");
22713 let Error::Http { status, body } = error else {
22714 panic!("expected HTTP redrive rejection");
22715 };
22716 assert_eq!(status.as_u16(), 409);
22717 assert!(body.contains("run_not_failed"));
22718 assert_eq!(
22719 server.request_body("/api/workflows/wf-lifecycle/runs/run-completed/redrive"),
22720 json!({})
22721 );
22722
22723 let handle = WorkflowHandle {
22724 client: client.clone(),
22725 workflow_id: "wf-lifecycle".to_string(),
22726 run_id: Some("run-failed".to_string()),
22727 workflow_type: "test".to_string(),
22728 };
22729 assert_eq!(
22730 handle.redrive(None).await.expect("handle redrive").run_id,
22731 "run-successor"
22732 );
22733 let missing_run = WorkflowHandle {
22734 run_id: None,
22735 ..handle
22736 };
22737 assert!(matches!(
22738 missing_run.redrive(None).await,
22739 Err(Error::Codec(_))
22740 ));
22741 }
22742
22743 #[tokio::test]
22744 async fn workflow_start_options_send_server_enforced_deadlines() {
22745 let server = MockWorkerServer::start();
22746 let client = Client::builder(server.base_url())
22747 .timeout(Duration::from_secs(2))
22748 .build()
22749 .expect("client");
22750
22751 let handle = client
22752 .start_workflow_with_options(
22753 "rust.timeout",
22754 "rust-timeouts",
22755 "wf-start-options",
22756 WorkflowStartOptions::new()
22757 .execution_timeout_seconds(30)
22758 .run_timeout_seconds(1),
22759 json!([]),
22760 )
22761 .await
22762 .expect("workflow start");
22763
22764 assert_eq!(handle.run_id.as_deref(), Some("run-start-options"));
22765 let body = server.request_body("/api/workflows");
22766 assert_eq!(body["execution_timeout_seconds"], 30);
22767 assert_eq!(body["run_timeout_seconds"], 1);
22768
22769 let invalid = client
22770 .start_workflow_with_options(
22771 "rust.timeout",
22772 "rust-timeouts",
22773 "wf-invalid-options",
22774 WorkflowStartOptions::new()
22775 .execution_timeout_seconds(1)
22776 .run_timeout_seconds(2),
22777 json!([]),
22778 )
22779 .await
22780 .expect_err("invalid deadline ordering");
22781 assert!(invalid
22782 .to_string()
22783 .contains("run_timeout_seconds cannot exceed execution_timeout_seconds"));
22784 }
22785
22786 #[tokio::test]
22787 async fn workflow_result_returns_each_typed_terminal_outcome() {
22788 let server = MockWorkerServer::start();
22789 let client = Client::builder(server.base_url())
22790 .timeout(Duration::from_secs(2))
22791 .build()
22792 .expect("client");
22793 let options = WorkflowResultOptions {
22794 poll_interval: Duration::ZERO,
22795 timeout: Duration::from_secs(1),
22796 };
22797
22798 let failed = WorkflowHandle {
22799 client: client.clone(),
22800 workflow_id: "wf-failed".to_string(),
22801 run_id: Some("run-failed".to_string()),
22802 workflow_type: "failure".to_string(),
22803 }
22804 .result(options)
22805 .await
22806 .expect_err("failed outcome");
22807 let Error::WorkflowFailed(failure) = failed else {
22808 panic!("expected WorkflowFailed");
22809 };
22810 assert_eq!(failure.workflow_id, "wf-failed");
22811 assert_eq!(failure.run_id.as_deref(), Some("run-failed"));
22812 assert_eq!(failure.failure_id.as_deref(), Some("failure-17"));
22813 assert_eq!(failure.failure_category.as_deref(), Some("application"));
22814 assert_eq!(failure.exception_type.as_deref(), Some("PaymentError"));
22815 assert_eq!(
22816 failure.exception_class.as_deref(),
22817 Some("billing::PaymentError")
22818 );
22819 assert_eq!(failure.non_retryable, Some(true));
22820
22821 for (workflow_id, expected_kind, expected_reason) in [
22822 (
22823 "wf-cancelled",
22824 WorkflowTerminalKind::Cancelled,
22825 "cleanup requested",
22826 ),
22827 (
22828 "wf-terminated",
22829 WorkflowTerminalKind::Terminated,
22830 "forced stop",
22831 ),
22832 (
22833 "wf-timed-out",
22834 WorkflowTerminalKind::TimedOut,
22835 "run_timeout",
22836 ),
22837 ] {
22838 let error = WorkflowHandle {
22839 client: client.clone(),
22840 workflow_id: workflow_id.to_string(),
22841 run_id: None,
22842 workflow_type: "terminal".to_string(),
22843 }
22844 .result(options)
22845 .await
22846 .expect_err("typed terminal outcome");
22847 let outcome = match error {
22848 Error::WorkflowCancelled(outcome) => outcome,
22849 Error::WorkflowTerminated(outcome) => outcome,
22850 Error::WorkflowTimedOut(outcome) => outcome,
22851 other => panic!("unexpected terminal error: {other}"),
22852 };
22853 assert_eq!(outcome.kind, expected_kind);
22854 assert_eq!(outcome.workflow_id, workflow_id);
22855 assert_eq!(outcome.reason, expected_reason);
22856 }
22857
22858 let wait_timeout = WorkflowHandle {
22859 client,
22860 workflow_id: "wf-waiting".to_string(),
22861 run_id: Some("run-waiting".to_string()),
22862 workflow_type: "waiting".to_string(),
22863 }
22864 .result(WorkflowResultOptions {
22865 poll_interval: Duration::ZERO,
22866 timeout: Duration::ZERO,
22867 })
22868 .await
22869 .expect_err("client wait timeout");
22870 let Error::WorkflowTimedOut(timeout) = wait_timeout else {
22871 panic!("expected typed client timeout");
22872 };
22873 assert_eq!(timeout.reason, "result_wait_timeout");
22874 assert_eq!(timeout.failure_category.as_deref(), Some("client_timeout"));
22875 assert_eq!(timeout.run_id.as_deref(), Some("run-waiting"));
22876 }
22877
22878 #[tokio::test]
22879 async fn workflow_result_follows_chain_and_selected_result_preserves_history() {
22880 let server = MockWorkerServer::start();
22881 let client = Client::builder(server.base_url())
22882 .timeout(Duration::from_secs(2))
22883 .build()
22884 .expect("client");
22885
22886 let handle = WorkflowHandle {
22887 client,
22888 workflow_id: "wf-selected".to_string(),
22889 run_id: Some("run-selected".to_string()),
22890 workflow_type: "selected".to_string(),
22891 };
22892 let options = WorkflowResultOptions {
22893 poll_interval: Duration::ZERO,
22894 timeout: Duration::from_secs(1),
22895 };
22896
22897 let current = handle
22898 .result(options)
22899 .await
22900 .expect("instance result follows the current run");
22901 assert_eq!(current, json!("current run output"));
22902
22903 let error = handle
22904 .result_selected_run(options)
22905 .await
22906 .expect_err("the selected run is cancelled even though the current run completed");
22907
22908 let Error::WorkflowCancelled(outcome) = error else {
22909 panic!("expected selected run cancellation");
22910 };
22911 assert_eq!(outcome.run_id.as_deref(), Some("run-selected"));
22912 assert_eq!(outcome.reason, "selected run cancelled");
22913 assert_eq!(
22914 server.request_count("/api/workflows/wf-selected/runs/run-selected"),
22915 1
22916 );
22917 assert_eq!(server.request_count("/api/workflows/wf-selected"), 1);
22918 }
22919
22920 #[tokio::test]
22921 async fn poll_responses_decode_http_conflict_drain_as_a_stable_stop() {
22922 let server = MockWorkerServer::draining_polls();
22923 let client = Client::builder(server.base_url())
22924 .timeout(Duration::from_secs(2))
22925 .build()
22926 .expect("client");
22927
22928 let workflow = client
22929 .poll_workflow_task_response("draining-worker", "rust-workers", Duration::ZERO)
22930 .await
22931 .expect("workflow drain response");
22932 let activity = client
22933 .poll_activity_task_response("draining-worker", "rust-workers", Duration::ZERO)
22934 .await
22935 .expect("activity drain response");
22936 let query = client
22937 .poll_query_task_response("draining-worker", "rust-workers", Duration::ZERO)
22938 .await
22939 .expect("query drain response");
22940
22941 for outcome in [workflow.outcome(), activity.outcome(), query.outcome()] {
22942 assert_eq!(
22943 outcome,
22944 WorkerPollOutcome::Stop {
22945 poll_status: Some("draining".to_string()),
22946 reason: Some("worker_draining".to_string()),
22947 }
22948 );
22949 }
22950
22951 assert!(client
22952 .poll_workflow_task("draining-worker", "rust-workers", Duration::ZERO)
22953 .await
22954 .expect("compatibility poll")
22955 .is_none());
22956 }
22957
22958 #[tokio::test]
22959 async fn managed_worker_honors_drain_stop_for_every_task_family() {
22960 let server = MockWorkerServer::draining_polls();
22961 let client = Client::builder(server.base_url())
22962 .timeout(Duration::from_secs(2))
22963 .build()
22964 .expect("client");
22965
22966 let mut workflow_worker = Worker::new(client.clone(), "rust-workers")
22967 .worker_id("draining-workflow-worker")
22968 .poll_timeout(Duration::ZERO);
22969 workflow_worker.register_workflow("counter", |_ctx, _args| async { Ok(Value::Null) });
22970 workflow_worker
22971 .run()
22972 .await
22973 .expect("workflow drain is a clean stop");
22974
22975 let mut activity_worker = Worker::new(client.clone(), "rust-workers")
22976 .worker_id("draining-activity-worker")
22977 .poll_timeout(Duration::ZERO);
22978 activity_worker.register_activity("write", |_ctx, _args| async { Ok(Value::Null) });
22979 activity_worker
22980 .run()
22981 .await
22982 .expect("activity drain is a clean stop");
22983
22984 let mut query_worker = Worker::new(client, "rust-workers")
22985 .worker_id("draining-query-worker")
22986 .poll_timeout(Duration::ZERO);
22987 query_worker.register_query("counter", "current", |_ctx, _args| async {
22988 Ok(Value::Null)
22989 });
22990 query_worker
22991 .run()
22992 .await
22993 .expect("query drain is a clean stop");
22994 }
22995
22996 #[tokio::test]
22997 async fn activity_cancellation_and_late_completion_remain_machine_readable() {
22998 let server = MockWorkerServer::start();
22999 let client = Client::builder(server.base_url())
23000 .timeout(Duration::from_secs(2))
23001 .build()
23002 .expect("client");
23003
23004 let heartbeat = client
23005 .heartbeat_activity_task(
23006 "activity-cancel",
23007 "attempt-cancel",
23008 "rust-worker",
23009 typed_fidelity_probe(),
23010 )
23011 .await
23012 .expect("cancellation heartbeat");
23013 assert!(heartbeat.cancel_requested);
23014 assert!(heartbeat.should_stop());
23015 assert_eq!(heartbeat.reason.as_deref(), Some("run_cancelled"));
23016 assert_eq!(heartbeat.run_closed_reason.as_deref(), Some("cancelled"));
23017 let heartbeat_body =
23018 server.request_body("/api/worker/activity-tasks/activity-cancel/heartbeat");
23019 assert_eq!(heartbeat_body["details"]["codec"], DEFAULT_CODEC);
23020 assert_eq!(
23021 decode_wire_avro_value(&heartbeat_body["details"], DEFAULT_CODEC)
23022 .expect("typed heartbeat details"),
23023 typed_fidelity_probe()
23024 );
23025
23026 let error = client
23027 .complete_activity_task(
23028 "activity-cancel",
23029 "attempt-cancel",
23030 "rust-worker",
23031 json!({"late":true}),
23032 DEFAULT_CODEC,
23033 )
23034 .await
23035 .expect_err("late completion must be refused");
23036 assert!(activity_task_rejection_is_final(&error));
23037 let Error::ActivityTaskRejected(rejection) = error else {
23038 panic!("expected typed activity rejection");
23039 };
23040 assert_eq!(rejection.status, 409);
23041 assert_eq!(rejection.reason, "run_cancelled");
23042 assert!(rejection.cancel_requested);
23043 assert_eq!(rejection.can_continue, Some(false));
23044 }
23045
23046 #[tokio::test]
23047 async fn managed_worker_survives_late_completion_and_restart_during_cancellation() {
23048 let server = MockWorkerServer::cancelled_activity();
23049 let client = Client::builder(server.base_url())
23050 .timeout(Duration::from_secs(2))
23051 .build()
23052 .expect("client");
23053 let cancellation_observed = Arc::new(AtomicBool::new(false));
23054 let observed = Arc::clone(&cancellation_observed);
23055 let mut worker = Worker::new(client.clone(), "rust-workers")
23056 .worker_id("rust-cancel-worker")
23057 .poll_timeout(Duration::from_millis(10));
23058 worker.register_activity("cancel-aware", move |ctx, _args| {
23059 let observed = Arc::clone(&observed);
23060 async move {
23061 let heartbeat = ctx.heartbeat(json!({"stage":"running"})).await?;
23062 observed.store(heartbeat.should_stop(), Ordering::SeqCst);
23063 Ok(json!({"late":"completion"}))
23064 }
23065 });
23066
23067 assert_eq!(
23068 worker.run_once().await.expect("cancelled attempt handled"),
23069 1
23070 );
23071 assert!(cancellation_observed.load(Ordering::SeqCst));
23072 assert_eq!(
23073 server.request_count("/api/worker/activity-tasks/activity-cancel/complete"),
23074 1
23075 );
23076
23077 let mut restarted = Worker::new(client, "rust-workers")
23078 .worker_id("rust-cancel-worker-restarted")
23079 .poll_timeout(Duration::from_millis(10));
23080 restarted.register_activity("cancel-aware", |_ctx, _args| async move { Ok(Value::Null) });
23081 assert_eq!(
23082 restarted
23083 .run_once()
23084 .await
23085 .expect("replacement worker continues polling"),
23086 0
23087 );
23088 }
23089
23090 #[tokio::test]
23091 async fn managed_worker_absorbs_selected_run_terminal_timeout_completion_race() {
23092 let response = r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"outcome":"completed","recorded":false,"run_id":"run-selected-timeout","run_status":"failed","created_task_ids":[],"reason":"run_timed_out"}"#;
23093 let server = MockWorkerServer::workflow_completion("409 Conflict", response);
23094 let client = Client::builder(server.base_url())
23095 .timeout(Duration::from_secs(2))
23096 .build()
23097 .expect("client");
23098
23099 let direct_error = client
23100 .complete_workflow_task(
23101 "workflow-timeout-task",
23102 "timeout-worker",
23103 3,
23104 vec![json!({
23105 "type": "complete_workflow",
23106 "result": fixture_envelope(Value::Null)
23107 })],
23108 )
23109 .await
23110 .expect_err("the low-level client preserves the completion rejection");
23111 let Error::Http { status, body } = direct_error else {
23112 panic!("expected the original HTTP completion rejection");
23113 };
23114 assert_eq!(status, reqwest::StatusCode::CONFLICT);
23115 assert_eq!(
23116 serde_json::from_str::<Value>(&body).expect("response body")["reason"],
23117 "run_timed_out"
23118 );
23119
23120 let mut worker = Worker::new(client, "rust-workers")
23121 .worker_id("timeout-worker")
23122 .poll_timeout(Duration::from_millis(10));
23123 worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
23124 Ok(json!({"late": "result"}))
23125 });
23126
23127 assert_eq!(
23128 worker
23129 .run_once()
23130 .await
23131 .expect("authoritative selected-run timeout settles the tick"),
23132 1
23133 );
23134 assert_eq!(
23135 server.request_count("/api/worker/workflow-tasks/workflow-timeout-task/complete"),
23136 2,
23137 "both the direct client proof and managed worker must see the rejection"
23138 );
23139 }
23140
23141 #[tokio::test]
23142 async fn managed_worker_does_not_swallow_nearby_completion_errors() {
23143 for (name, status, response) in [
23144 ("bare conflict", "409 Conflict", r#"{"message":"conflict"}"#),
23145 (
23146 "command was recorded",
23147 "409 Conflict",
23148 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":true,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
23149 ),
23150 (
23151 "lease conflict",
23152 "409 Conflict",
23153 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"lease_expired"}"#,
23154 ),
23155 (
23156 "nonterminal run",
23157 "409 Conflict",
23158 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"waiting","reason":"run_timed_out"}"#,
23159 ),
23160 (
23161 "different selected run",
23162 "409 Conflict",
23163 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-reused-workflow-current","run_status":"failed","reason":"run_timed_out"}"#,
23164 ),
23165 (
23166 "different task attempt",
23167 "409 Conflict",
23168 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":4,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
23169 ),
23170 (
23171 "authentication failure",
23172 "401 Unauthorized",
23173 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
23174 ),
23175 (
23176 "authorization failure",
23177 "403 Forbidden",
23178 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
23179 ),
23180 (
23181 "protocol failure",
23182 "400 Bad Request",
23183 r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.2","requested_version":"1.3"}"#,
23184 ),
23185 (
23186 "malformed command",
23187 "422 Unprocessable Entity",
23188 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
23189 ),
23190 (
23191 "transient server failure",
23192 "503 Service Unavailable",
23193 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
23194 ),
23195 ] {
23196 let server = MockWorkerServer::workflow_completion(status, response);
23197 let client = Client::builder(server.base_url())
23198 .timeout(Duration::from_secs(2))
23199 .build()
23200 .expect("client");
23201 let mut worker = Worker::new(client, "rust-workers")
23202 .worker_id("timeout-worker")
23203 .poll_timeout(Duration::from_millis(10));
23204 worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
23205 Ok(json!({"late": "result"}))
23206 });
23207
23208 let error = worker
23209 .run_once()
23210 .await
23211 .expect_err(&format!("{name} must remain an error"));
23212 assert!(
23213 matches!(error, Error::Http { .. } | Error::Protocol(_)),
23214 "{name} returned an unexpected error variant: {error}"
23215 );
23216 }
23217 }
23218
23219 #[tokio::test]
23220 async fn worker_deregistration_uses_worker_plane_method_path_headers_and_result() {
23221 let server = MockWorkerServer::start();
23222 let client = Client::builder(server.base_url())
23223 .worker_token(Some("worker-secret".to_string()))
23224 .namespace("orders")
23225 .timeout(Duration::from_secs(2))
23226 .build()
23227 .expect("client");
23228 let path = "/api/worker/registrations/worker%2F%CE%B1%20space";
23229
23230 let result = client
23231 .deregister_worker_registration("worker/α space")
23232 .await
23233 .expect("deregister worker registration");
23234
23235 assert_eq!(server.method_for(path).as_deref(), Some("DELETE"));
23236 assert_eq!(
23237 server.worker_protocol_for(path).as_deref(),
23238 Some(WORKER_PROTOCOL_VERSION)
23239 );
23240 assert_eq!(server.control_protocol_for(path), None);
23241 assert_eq!(server.namespace_for(path).as_deref(), Some("orders"));
23242 assert_eq!(
23243 server.authorization_for(path).as_deref(),
23244 Some("Bearer worker-secret")
23245 );
23246 assert_eq!(
23247 result,
23248 WorkerDeregistrationEnvelope {
23249 worker_id: "deregistered-worker".to_string(),
23250 outcome: "deregistered".to_string(),
23251 recovered_workflow_task_count: 2,
23252 }
23253 );
23254 }
23255
23256 #[tokio::test]
23257 async fn low_level_registration_rejects_update_validators_before_transport() {
23258 let server = MockWorkerServer::start();
23259 let client = Client::builder(server.base_url())
23260 .timeout(Duration::from_secs(2))
23261 .build()
23262 .expect("client");
23263
23264 for update_validators in [json!(["approve"]), json!("approve")] {
23265 let error = client
23266 .register_worker_with_command_contracts(
23267 "validator-claiming-worker",
23268 "rust-workers",
23269 vec!["orders".to_string()],
23270 vec![],
23271 1,
23272 1,
23273 vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
23274 json!({
23275 "orders": {
23276 "queries": ["current"],
23277 "updates": ["approve"],
23278 "update_validators": update_validators,
23279 },
23280 }),
23281 )
23282 .await
23283 .expect_err("unsupported validator claims must fail before registration");
23284
23285 let Error::UnsupportedUpdateValidators { workflow_type } = error else {
23286 panic!("expected typed unsupported-validator failure");
23287 };
23288 assert_eq!(workflow_type, "orders");
23289 }
23290 assert_eq!(server.request_count("/api/worker/register"), 0);
23291 }
23292
23293 #[tokio::test]
23294 async fn low_level_registration_preserves_query_and_update_contracts() {
23295 let server = MockWorkerServer::start();
23296 let client = Client::builder(server.base_url())
23297 .timeout(Duration::from_secs(2))
23298 .build()
23299 .expect("client");
23300 let contracts = json!({
23301 "orders": {
23302 "queries": ["current"],
23303 "updates": ["approve"],
23304 "update_validators": [],
23305 },
23306 "payments": {
23307 "queries": ["status"],
23308 "updates": ["capture"],
23309 },
23310 });
23311
23312 client
23313 .register_worker_with_command_contracts(
23314 "command-worker",
23315 "rust-workers",
23316 vec!["orders".to_string(), "payments".to_string()],
23317 vec![],
23318 1,
23319 1,
23320 vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
23321 contracts.clone(),
23322 )
23323 .await
23324 .expect("query and update contracts must remain supported");
23325
23326 assert_eq!(
23327 server.request_body("/api/worker/register")["workflow_command_contracts"],
23328 contracts
23329 );
23330 assert!(server
23331 .request_body("/api/worker/register")
23332 .get("workflow_definition_fingerprints")
23333 .is_none());
23334 }
23335
23336 #[tokio::test]
23337 async fn rust_worker_advertises_source_backed_definition_and_clears_it_on_reregistration() {
23338 let server = MockWorkerServer::start();
23339 let client = Client::builder(server.base_url())
23340 .timeout(Duration::from_secs(2))
23341 .build()
23342 .expect("client");
23343 let mut worker = Worker::new(client, "rust-workers");
23344 worker.register_workflow("orders", |_ctx, _input| async { Ok(Value::Null) });
23345
23346 assert!(worker
23347 .set_workflow_definition_sources("missing", &[include_str!("lib.rs")])
23348 .is_err());
23349 assert!(worker
23350 .set_workflow_definition_sources("orders", &[])
23351 .is_err());
23352 worker
23353 .set_workflow_definition_sources("orders", &[include_str!("lib.rs")])
23354 .expect("embed workflow source");
23355 worker.register().await.expect("register worker");
23356 let first = server.request_bodies("/api/worker/register")[0]
23357 ["workflow_definition_fingerprints"]["orders"]
23358 .as_str()
23359 .expect("source-backed fingerprint")
23360 .to_string();
23361 assert!(first.starts_with("sha256:"));
23362
23363 worker
23364 .set_workflow_definition_sources("orders", &[include_str!("lib.rs"), "changed helper"])
23365 .expect("change embedded helper source");
23366 worker.register().await.expect("register changed source");
23367 let changed = server.request_bodies("/api/worker/register")[1]
23368 ["workflow_definition_fingerprints"]["orders"]
23369 .as_str()
23370 .expect("changed fingerprint")
23371 .to_string();
23372 assert_ne!(first, changed);
23373
23374 worker.register_workflow("orders", |_ctx, _input| async { Ok(Value::Null) });
23375 worker
23376 .register()
23377 .await
23378 .expect("register replacement handler");
23379 assert_eq!(
23380 server.request_bodies("/api/worker/register")[2]["workflow_definition_fingerprints"],
23381 json!({})
23382 );
23383 }
23384
23385 #[tokio::test]
23386 async fn role_scoped_tokens_are_never_used_for_the_opposite_plane() {
23387 let server = MockWorkerServer::start();
23388 let control_only = Client::builder(server.base_url())
23389 .control_token(Some("control-secret".to_string()))
23390 .build()
23391 .expect("control client");
23392
23393 let error = control_only
23394 .register_worker("worker", "queue", vec![], vec![], 1, 1)
23395 .await
23396 .expect_err("control token must not authorize a worker request");
23397 assert!(matches!(
23398 error,
23399 Error::MissingRoleCredentials { role: "worker", .. }
23400 ));
23401 assert_eq!(server.request_count("/api/worker/register"), 0);
23402
23403 let worker_only = Client::builder(server.base_url())
23404 .worker_token(Some("worker-secret".to_string()))
23405 .build()
23406 .expect("worker client");
23407 let error = worker_only
23408 .health()
23409 .await
23410 .expect_err("worker token must not authorize a control request");
23411 assert!(matches!(
23412 error,
23413 Error::MissingRoleCredentials {
23414 role: "control",
23415 ..
23416 }
23417 ));
23418 assert_eq!(server.request_count("/api/health"), 0);
23419 }
23420
23421 #[tokio::test]
23422 async fn shared_token_supports_worker_and_control_planes() {
23423 let server = MockWorkerServer::start();
23424 let client = Client::builder(server.base_url())
23425 .token(Some("shared-secret".to_string()))
23426 .build()
23427 .expect("client");
23428
23429 client.health().await.expect("control request");
23430 client
23431 .register_worker("worker", "queue", vec![], vec![], 1, 1)
23432 .await
23433 .expect("worker request");
23434
23435 assert_eq!(
23436 server.authorization_for("/api/health").as_deref(),
23437 Some("Bearer shared-secret")
23438 );
23439 assert_eq!(
23440 server.control_protocol_for("/api/health").as_deref(),
23441 Some(CONTROL_PLANE_VERSION)
23442 );
23443 assert_eq!(
23444 server.authorization_for("/api/worker/register").as_deref(),
23445 Some("Bearer shared-secret")
23446 );
23447 assert_eq!(
23448 server
23449 .worker_protocol_for("/api/worker/register")
23450 .as_deref(),
23451 Some(WORKER_PROTOCOL_VERSION)
23452 );
23453 }
23454
23455 #[tokio::test]
23456 async fn baseline_worker_endpoints_send_the_baseline_protocol() {
23457 let server = MockWorkerServer::start();
23458 let client = Client::builder(server.base_url())
23459 .timeout(Duration::from_secs(2))
23460 .build()
23461 .expect("client");
23462
23463 client
23464 .register_worker("capture-worker", "capture", vec![], vec![], 1, 1)
23465 .await
23466 .expect("register");
23467 client
23468 .heartbeat_worker("capture-worker", 1, 1)
23469 .await
23470 .expect("heartbeat");
23471 client
23472 .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
23473 .await
23474 .expect("workflow poll");
23475 client
23476 .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
23477 .await
23478 .expect("activity poll");
23479
23480 for path in [
23481 "/api/worker/register",
23482 "/api/worker/heartbeat",
23483 "/api/worker/workflow-tasks/poll",
23484 "/api/worker/activity-tasks/poll",
23485 ] {
23486 assert_eq!(
23487 server.worker_protocol_for(path).as_deref(),
23488 Some(WORKER_PROTOCOL_VERSION),
23489 "unexpected protocol for {path}"
23490 );
23491 }
23492
23493 assert_eq!(
23494 server.request_body("/api/worker/workflow-tasks/poll")["timeout_seconds"],
23495 1
23496 );
23497 assert_eq!(
23498 server.request_body("/api/worker/activity-tasks/poll")["timeout_seconds"],
23499 1
23500 );
23501 assert!(
23502 server.request_body("/api/worker/workflow-tasks/poll")["poll_request_id"]
23503 .as_str()
23504 .is_some_and(|id| id.starts_with("rust-workflow-poll-"))
23505 );
23506 assert!(
23507 server.request_body("/api/worker/activity-tasks/poll")["poll_request_id"]
23508 .as_str()
23509 .is_some_and(|id| id.starts_with("rust-activity-poll-"))
23510 );
23511 }
23512
23513 #[tokio::test]
23514 async fn query_task_endpoints_send_the_query_feature_protocol() {
23515 let server = MockWorkerServer::start();
23516 let client = Client::builder(server.base_url())
23517 .timeout(Duration::from_secs(2))
23518 .build()
23519 .expect("client");
23520
23521 client
23522 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
23523 .await
23524 .expect("query poll");
23525 client
23526 .complete_query_task(
23527 "query-capture",
23528 "capture-worker",
23529 1,
23530 json!(8),
23531 DEFAULT_CODEC,
23532 )
23533 .await
23534 .expect("query complete");
23535 client
23536 .fail_query_task(
23537 "query-capture",
23538 "capture-worker",
23539 1,
23540 "failed",
23541 "query_rejected",
23542 "QueryFailed",
23543 )
23544 .await
23545 .expect("query fail");
23546
23547 for path in [
23548 "/api/worker/query-tasks/poll",
23549 "/api/worker/query-tasks/query-capture/complete",
23550 "/api/worker/query-tasks/query-capture/fail",
23551 ] {
23552 assert_eq!(
23553 server.worker_protocol_for(path).as_deref(),
23554 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
23555 "unexpected protocol for {path}"
23556 );
23557 }
23558
23559 assert_eq!(
23560 server.request_body("/api/worker/query-tasks/poll")["timeout_seconds"],
23561 1
23562 );
23563 assert!(
23564 server.request_body("/api/worker/query-tasks/poll")["poll_request_id"]
23565 .as_str()
23566 .is_some_and(|id| id.starts_with("rust-query-poll-"))
23567 );
23568 }
23569
23570 #[tokio::test]
23571 async fn disconnected_client_polls_retry_once_with_the_same_request_id() {
23572 let server = MockWorkerServer::transient_worker_failures();
23573 let client = Client::builder(server.base_url())
23574 .timeout(Duration::from_secs(2))
23575 .build()
23576 .expect("client");
23577
23578 client
23579 .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
23580 .await
23581 .expect("workflow poll retry");
23582 client
23583 .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
23584 .await
23585 .expect("activity poll retry");
23586 client
23587 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
23588 .await
23589 .expect("query poll retry");
23590
23591 for path in [
23592 "/api/worker/workflow-tasks/poll",
23593 "/api/worker/activity-tasks/poll",
23594 "/api/worker/query-tasks/poll",
23595 ] {
23596 let bodies = server.request_bodies(path);
23597 assert_eq!(bodies.len(), 2, "{path} must be retried once");
23598 assert_eq!(
23599 bodies[0]["poll_request_id"], bodies[1]["poll_request_id"],
23600 "{path} must preserve the request binding across retry"
23601 );
23602 }
23603 }
23604
23605 #[tokio::test]
23606 async fn worker_poll_retries_preserve_request_id_across_consecutive_failures() {
23607 let server = MockWorkerServer::consecutive_poll_failures(2);
23608 let client = Client::builder(server.base_url())
23609 .timeout(Duration::from_secs(2))
23610 .build()
23611 .expect("client");
23612 let mut worker = Worker::new(client, "capture")
23613 .worker_id("capture-worker")
23614 .poll_timeout(Duration::from_millis(10))
23615 .retry_policy(WorkerRetryPolicy {
23616 max_retries: 2,
23617 initial_backoff: Duration::from_millis(1),
23618 max_backoff: Duration::from_millis(1),
23619 });
23620 worker.register_workflow(
23621 "capture.workflow",
23622 |_ctx, _input| async move { Ok(Value::Null) },
23623 );
23624 worker.register_activity(
23625 "capture.activity",
23626 |_ctx, _input| async move { Ok(Value::Null) },
23627 );
23628 worker.register_query("capture.workflow", "current", |_ctx, _args| async move {
23629 Ok(Value::Null)
23630 });
23631
23632 assert_eq!(worker.run_once().await.expect("poll retries"), 0);
23633
23634 for path in [
23635 "/api/worker/workflow-tasks/poll",
23636 "/api/worker/activity-tasks/poll",
23637 "/api/worker/query-tasks/poll",
23638 ] {
23639 let bodies = server.request_bodies(path);
23640 assert_eq!(bodies.len(), 3, "{path} must use exactly two retries");
23641 assert!(
23642 bodies
23643 .iter()
23644 .all(|body| body["poll_request_id"] == bodies[0]["poll_request_id"]),
23645 "{path} must preserve one request binding across every retry"
23646 );
23647 }
23648 }
23649
23650 fn storage_refusal(poll_id: Option<&str>, unavailable: bool, mid_poll: bool) -> Value {
23651 let reason = if unavailable {
23652 "storage_admission_unavailable"
23653 } else {
23654 "storage_pressure"
23655 };
23656 let mut body = json!({
23657 "reason": reason,
23658 "storage_state": if unavailable { "fenced" } else { "draining" },
23659 "retryable": true,
23660 "retry_after_seconds": 1,
23661 });
23662 if !mid_poll {
23663 body["request_admitted"] = json!(false);
23664 }
23665 if let Some(id) = poll_id {
23666 body["task"] = Value::Null;
23667 body["poll_status"] = json!(reason);
23668 body["poll_request_id"] = json!(id);
23669 body["retry_same_poll_request_id"] = json!(true);
23670 body["claim_admitted"] = json!(false);
23671 }
23672 body
23673 }
23674
23675 fn backend_refusal(path: &str, request: &str) -> Option<Value> {
23676 let operation = match path {
23677 "/api/worker/register" => "register_worker",
23678 "/api/worker/heartbeat" => "heartbeat_worker",
23679 "/api/worker/workflow-tasks/poll" => "poll_workflow_task",
23680 "/api/worker/activity-tasks/poll" => "poll_activity_task",
23681 "/api/worker/query-tasks/poll" => "poll_query_task",
23682 "/api/worker/update-validation-tasks/poll" => "poll_update_validation_task",
23683 _ => return None,
23684 };
23685 let request: Value = serde_json::from_str(request).ok()?;
23686 let mut response = json!({
23687 "reason": "backend_unavailable",
23688 "operation": operation,
23689 "outcome": "unknown",
23690 "retryable": true,
23691 "retry_after_seconds": 1,
23692 "worker_id": request["worker_id"],
23693 "task_queue": request.get("task_queue"),
23694 });
23695 if path.ends_with("/poll") {
23696 response["task"] = Value::Null;
23697 response["poll_status"] = json!("backend_unavailable");
23698 response["poll_request_id"] = request["poll_request_id"].clone();
23699 response["retry_same_poll_request_id"] = json!(true);
23700 }
23701 Some(response)
23702 }
23703
23704 fn backend_retry_override(
23705 path: &str,
23706 request: &str,
23707 number: usize,
23708 ) -> Option<(&'static str, String)> {
23709 if number > 8 {
23710 return None;
23711 }
23712 Some((
23713 "503 Service Unavailable",
23714 backend_refusal(path, request)?.to_string(),
23715 ))
23716 }
23717
23718 fn backend_unavailable_override(
23719 path: &str,
23720 request: &str,
23721 _number: usize,
23722 ) -> Option<(&'static str, String)> {
23723 Some((
23724 "503 Service Unavailable",
23725 backend_refusal(path, request)?.to_string(),
23726 ))
23727 }
23728
23729 fn storage_worker(server: &MockWorkerServer) -> Worker {
23730 Worker::new(Client::new(server.base_url()).expect("client"), "storage")
23731 .worker_id("storage-worker")
23732 .retry_policy(WorkerRetryPolicy {
23733 max_retries: 1,
23734 initial_backoff: Duration::from_millis(1),
23735 max_backoff: Duration::from_millis(1),
23736 })
23737 }
23738
23739 fn assert_identical_requests(server: &MockWorkerServer, path: &str, count: usize) {
23740 let requests = server.requests.lock().expect("requests");
23741 let bodies: Vec<_> = requests
23742 .iter()
23743 .filter(|request| request.path == path)
23744 .map(|request| &request.body)
23745 .collect();
23746 assert_eq!(bodies.len(), count, "{path}");
23747 assert!(bodies.iter().all(|body| body == &bodies[0]), "{path}");
23748 }
23749
23750 #[test]
23751 fn backend_recovery_requires_the_explicit_worker_contract() {
23752 for (path, operation) in [
23753 ("/worker/register", "register_worker"),
23754 ("/worker/heartbeat", "heartbeat_worker"),
23755 ("/worker/workflow-tasks/poll", "poll_workflow_task"),
23756 ("/worker/activity-tasks/poll", "poll_activity_task"),
23757 ("/worker/query-tasks/poll", "poll_query_task"),
23758 (
23759 "/worker/update-validation-tasks/poll",
23760 "poll_update_validation_task",
23761 ),
23762 ] {
23763 let poll_id = path.ends_with("/poll").then_some("same-poll");
23764 let request = json!({"worker_id":"same-worker","task_queue":"same-queue","poll_request_id":poll_id});
23765 let body = backend_refusal(&format!("/api{path}"), &request.to_string())
23766 .expect("worker operation");
23767 assert_eq!(body["operation"], operation);
23768 let error = Error::Http {
23769 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23770 body: body.to_string(),
23771 };
23772 assert_eq!(
23773 worker_backend_unavailable_retry_after(&error, path, poll_id),
23774 Some(Duration::from_secs(1))
23775 );
23776 assert!(!worker_operation_is_retryable(&error));
23777 for (field, value) in [
23778 ("operation", json!("wrong_operation")),
23779 ("outcome", json!("completed")),
23780 ("retryable", json!(false)),
23781 ("retry_after_seconds", json!(0)),
23782 ("worker_id", json!("")),
23783 ] {
23784 let mut invalid = body.clone();
23785 invalid[field] = value;
23786 let error = Error::Http {
23787 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23788 body: invalid.to_string(),
23789 };
23790 assert!(
23791 worker_backend_unavailable_retry_after(&error, path, poll_id).is_none(),
23792 "{path}: {field}"
23793 );
23794 assert!(!worker_operation_is_retryable(&error));
23795 }
23796 if let Some(poll_id) = poll_id {
23797 for (field, value) in [
23798 ("poll_request_id", json!("wrong-poll")),
23799 ("poll_status", json!("empty")),
23800 ("task", json!({"task_id":"claimed"})),
23801 ("retry_same_poll_request_id", json!(false)),
23802 ] {
23803 let mut invalid = body.clone();
23804 invalid[field] = value;
23805 let error = Error::Http {
23806 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23807 body: invalid.to_string(),
23808 };
23809 assert!(
23810 worker_backend_unavailable_retry_after(&error, path, Some(poll_id))
23811 .is_none(),
23812 "{path}: {field}"
23813 );
23814 }
23815 }
23816 let error = Error::Http {
23817 status: reqwest::StatusCode::UNAUTHORIZED,
23818 body: body.to_string(),
23819 };
23820 assert!(worker_backend_unavailable_retry_after(&error, path, poll_id).is_none());
23821 assert!(!worker_operation_is_retryable(&error));
23822 }
23823 }
23824
23825 #[tokio::test]
23826 async fn backend_recovery_outlives_generic_retry_budget_and_preserves_polls() {
23827 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23828 request_override: Some(backend_retry_override),
23829 ..MockWorkerBehavior::default()
23830 });
23831 let mut worker =
23832 storage_worker(&server).with_storage_admission(Arc::new(AtomicBool::new(false)));
23833 worker.register_workflow("backend.workflow", |_, _| async { Ok(Value::Null) });
23834 worker.register_activity("backend.activity", |_, _| async { Ok(Value::Null) });
23835 worker.register_query("backend.workflow", "state", |_, _| async {
23836 Ok(Value::Null)
23837 });
23838 worker.register().await.expect("registration recovery");
23839 worker
23840 .client
23841 .heartbeat_worker("storage-worker", 1, 1)
23842 .await
23843 .expect("heartbeat recovery");
23844 assert_eq!(worker.run_once().await.expect("poll recovery"), 0);
23845 for path in [
23846 "/api/worker/register",
23847 "/api/worker/heartbeat",
23848 "/api/worker/workflow-tasks/poll",
23849 "/api/worker/activity-tasks/poll",
23850 "/api/worker/query-tasks/poll",
23851 ] {
23852 assert_identical_requests(&server, path, 9);
23853 }
23854 }
23855
23856 #[tokio::test]
23857 async fn backend_recovery_wait_stops_with_worker() {
23858 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23859 request_override: Some(backend_unavailable_override),
23860 ..MockWorkerBehavior::default()
23861 });
23862 let mut worker = storage_worker(&server).retry_policy(WorkerRetryPolicy {
23863 max_retries: 1,
23864 initial_backoff: Duration::from_secs(2),
23865 max_backoff: Duration::from_secs(2),
23866 });
23867 worker.register_workflow("backend.workflow", |_, _| async { Ok(Value::Null) });
23868 let started = tokio::time::Instant::now();
23869 let _ = tokio::time::timeout(
23870 Duration::from_secs(1),
23871 worker.run_until(tokio::time::sleep(Duration::from_millis(100))),
23872 )
23873 .await
23874 .expect("shutdown interrupts backend wait");
23875 assert!(started.elapsed() < Duration::from_secs(1));
23876 assert!(server.request_count("/api/worker/register") >= 1);
23877 }
23878
23879 #[test]
23880 fn storage_admission_requires_an_explicit_identity_preserving_contract() {
23881 for unavailable in [false, true] {
23882 for mid_poll in [false, true] {
23883 let body = storage_refusal(Some("same-poll"), unavailable, mid_poll);
23884 let error = Error::Http {
23885 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23886 body: body.to_string(),
23887 };
23888 assert_eq!(
23889 worker_storage_admission_retry_after(&error, Some("same-poll")),
23890 Some(Duration::from_secs(1))
23891 );
23892 assert!(
23893 !worker_operation_is_retryable(&error),
23894 "storage is not a bounded generic retry"
23895 );
23896 for (field, value) in [
23897 ("poll_request_id", json!("wrong-poll")),
23898 ("task", json!({"task_id":"claimed"})),
23899 ("retryable", json!(false)),
23900 ("retry_after_seconds", json!(0)),
23901 ("retry_after_seconds", json!(true)),
23902 ("retry_after_seconds", json!(1.0)),
23903 ("storage_state", json!("normal")),
23904 ("poll_status", json!("empty")),
23905 ("claim_admitted", json!(true)),
23906 ("retry_same_poll_request_id", json!(false)),
23907 ("request_admitted", json!(true)),
23908 ] {
23909 let mut invalid = body.clone();
23910 invalid[field] = value;
23911 let error = Error::Http {
23912 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23913 body: invalid.to_string(),
23914 };
23915 assert!(
23916 worker_storage_admission_retry_after(&error, Some("same-poll")).is_none(),
23917 "{field}"
23918 );
23919 }
23920 }
23921 }
23922 let body = storage_refusal(None, false, false);
23923 let error = Error::Http {
23924 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23925 body: body.to_string(),
23926 };
23927 assert!(worker_storage_admission_retry_after(&error, None).is_some());
23928 assert!(worker_storage_admission_retry_after(&error, Some("")).is_none());
23929 let error = Error::Http {
23930 status: reqwest::StatusCode::FORBIDDEN,
23931 body: body.to_string(),
23932 };
23933 assert!(worker_storage_admission_retry_after(&error, None).is_none());
23934 let body = storage_refusal(None, false, true);
23935 let error = Error::Http {
23936 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23937 body: body.to_string(),
23938 };
23939 assert!(worker_storage_admission_retry_after(&error, None).is_none());
23940 }
23941
23942 #[tokio::test]
23943 async fn storage_poll_recovery_preserves_ambiguous_claim_identity() {
23944 for unavailable in [false, true] {
23945 for mid_poll in [false, true] {
23946 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23947 poll_failures_per_path: 1,
23948 storage_refusals: 7,
23949 storage_path: Some("/poll"),
23950 storage_unavailable: unavailable,
23951 storage_mid_poll: mid_poll,
23952 ..MockWorkerBehavior::default()
23953 });
23954 let mut worker = storage_worker(&server);
23955 worker.register_query("unused", "state", |_, _| async { Ok(Value::Null) });
23956 assert_eq!(worker.run_once().await.expect("storage recovery"), 0);
23957 for path in [
23958 "/api/worker/workflow-tasks/poll",
23959 "/api/worker/activity-tasks/poll",
23960 "/api/worker/query-tasks/poll",
23961 ] {
23962 assert_identical_requests(&server, path, 9);
23963 }
23964 }
23965 }
23966 }
23967
23968 #[tokio::test]
23969 async fn storage_refused_mutations_do_not_reserialize_or_change_client_scope() {
23970 struct CountedBody(Arc<AtomicUsize>);
23971 impl Serialize for CountedBody {
23972 fn serialize<S: Serializer>(
23973 &self,
23974 serializer: S,
23975 ) -> std::result::Result<S::Ok, S::Error> {
23976 let count = self.0.fetch_add(1, Ordering::SeqCst);
23977 json!({"serialization":count,"lease_owner":"worker","attempt":7})
23978 .serialize(serializer)
23979 }
23980 }
23981 for path in [
23982 "/api/worker/register",
23983 "/api/worker/heartbeat",
23984 "/api/worker/workflow-tasks/storage-task/complete",
23985 "/api/worker/workflow-tasks/storage-task/fail",
23986 "/api/worker/activity-tasks/storage-task/complete",
23987 "/api/worker/activity-tasks/storage-task/fail",
23988 "/api/worker/activity-tasks/storage-task/heartbeat",
23989 "/api/worker/query-tasks/storage-task/complete",
23990 "/api/worker/query-tasks/storage-task/fail",
23991 ] {
23992 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23993 storage_refusals: 7,
23994 storage_path: Some(path),
23995 ..MockWorkerBehavior::default()
23996 });
23997 let worker =
23998 storage_worker(&server).with_storage_admission(Arc::new(AtomicBool::new(false)));
23999 let calls = Arc::new(AtomicUsize::new(0));
24000 let _: Value = worker
24001 .client
24002 .request_json(
24003 reqwest::Method::POST,
24004 &path[4..],
24005 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
24006 Some(&CountedBody(Arc::clone(&calls))),
24007 )
24008 .await
24009 .expect("prepared request recovery");
24010 assert_eq!(calls.load(Ordering::SeqCst), 1);
24011 assert_identical_requests(&server, path, 8);
24012 }
24013 for worker_scope in [false, true] {
24014 let path = if worker_scope {
24015 "/api/health"
24016 } else {
24017 "/api/worker/register"
24018 };
24019 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
24020 storage_refusals: usize::MAX,
24021 storage_path: Some(path),
24022 ..MockWorkerBehavior::default()
24023 });
24024 let worker = storage_worker(&server);
24025 let client = worker.client.clone();
24026 let worker = worker.with_storage_admission(Arc::new(AtomicBool::new(false)));
24027 let error = if worker_scope {
24028 worker
24029 .client
24030 .health()
24031 .await
24032 .expect_err("control plane is not retried")
24033 } else {
24034 client
24035 .request_json::<Value, Value>(
24036 reqwest::Method::POST,
24037 "/worker/register",
24038 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
24039 Some(&json!({})),
24040 )
24041 .await
24042 .expect_err("direct client is not retried")
24043 };
24044 assert!(worker_storage_admission_body(&error).is_some());
24045 assert_eq!(server.request_count(path), 1);
24046 }
24047 }
24048
24049 #[tokio::test]
24050 async fn storage_activity_outcome_is_retained_without_reexecuting_handler() {
24051 for fail in [false, true] {
24052 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
24053 storage_activity: true,
24054 storage_refusals: 7,
24055 storage_path: Some("/storage-activity/"),
24056 ..MockWorkerBehavior::default()
24057 });
24058 let mut worker = storage_worker(&server);
24059 let calls = Arc::new(AtomicUsize::new(0));
24060 let observed = Arc::clone(&calls);
24061 worker.register_activity("storage.activity", move |ctx, _| {
24062 let calls = Arc::clone(&observed);
24063 async move {
24064 calls.fetch_add(1, Ordering::SeqCst);
24065 ctx.heartbeat(json!({"step":2})).await?;
24066 if fail {
24067 Err(Error::WorkerLoop("intentional handler failure".to_string()))
24068 } else {
24069 Ok(json!({"receipt":true}))
24070 }
24071 }
24072 });
24073 assert_eq!(worker.run_once().await.expect("activity settled"), 1);
24074 assert_eq!(calls.load(Ordering::SeqCst), 1);
24075 assert_identical_requests(
24076 &server,
24077 "/api/worker/activity-tasks/storage-activity/heartbeat",
24078 8,
24079 );
24080 let suffix = if fail { "fail" } else { "complete" };
24081 assert_identical_requests(
24082 &server,
24083 &format!("/api/worker/activity-tasks/storage-activity/{suffix}"),
24084 8,
24085 );
24086 let other = if fail { "complete" } else { "fail" };
24087 assert_eq!(
24088 server.request_count(&format!(
24089 "/api/worker/activity-tasks/storage-activity/{other}"
24090 )),
24091 0
24092 );
24093 }
24094 }
24095
24096 #[tokio::test]
24097 async fn storage_waits_are_interruptible_without_false_activity_failure() {
24098 for path in [
24099 "/api/worker/register",
24100 "/api/worker/heartbeat",
24101 "/api/worker/activity-tasks/poll",
24102 "/api/worker/activity-tasks/storage-activity/heartbeat",
24103 "/api/worker/activity-tasks/storage-activity/complete",
24104 ] {
24105 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
24106 storage_activity: true,
24107 storage_refusals: usize::MAX,
24108 storage_path: Some(path),
24109 ..MockWorkerBehavior::default()
24110 });
24111 let mut worker = storage_worker(&server).retry_policy(WorkerRetryPolicy::default());
24112 worker.register_activity("storage.activity", |ctx, _| async move {
24113 ctx.heartbeat(json!({"step":2})).await?;
24114 Ok(json!({"receipt":true}))
24115 });
24116 let shutdown = async {
24117 while server.request_count(path) == 0 {
24118 tokio::time::sleep(Duration::from_millis(1)).await;
24119 }
24120 };
24121 let result = tokio::time::timeout(Duration::from_secs(2), worker.run_until(shutdown))
24122 .await
24123 .expect("shutdown interrupts admission");
24124 assert!(
24125 result.is_err(),
24126 "a refused operation must not appear acknowledged: {path}, {result:?}"
24127 );
24128 assert_eq!(server.request_count(path), 1);
24129 assert_eq!(
24130 server.request_count("/api/worker/activity-tasks/storage-activity/fail"),
24131 0
24132 );
24133 assert_eq!(
24134 server.request_count("/api/worker/registrations/mock-worker"),
24135 usize::from(!path.ends_with("/register"))
24136 );
24137 }
24138 }
24139
24140 #[tokio::test]
24141 async fn storage_query_outcome_is_retained_without_reexecuting_handler() {
24142 for fail in [false, true] {
24143 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
24144 storage_query: true,
24145 storage_refusals: 7,
24146 storage_path: Some("/storage-query/"),
24147 ..MockWorkerBehavior::default()
24148 });
24149 let mut worker = storage_worker(&server);
24150 worker.register_workflow("storage.workflow", |_, _| async { Ok(Value::Null) });
24151 let calls = Arc::new(AtomicUsize::new(0));
24152 let observed = Arc::clone(&calls);
24153 worker.register_query("storage.workflow", "state", move |_, _| {
24154 let calls = Arc::clone(&observed);
24155 async move {
24156 calls.fetch_add(1, Ordering::SeqCst);
24157 if fail {
24158 Err(Error::WorkerLoop("intentional query failure".to_string()))
24159 } else {
24160 Ok(json!({"state":"waiting"}))
24161 }
24162 }
24163 });
24164 assert_eq!(worker.run_once().await.expect("query settled"), 1);
24165 assert_eq!(calls.load(Ordering::SeqCst), 1);
24166 let suffix = if fail { "fail" } else { "complete" };
24167 assert_identical_requests(
24168 &server,
24169 &format!("/api/worker/query-tasks/storage-query/{suffix}"),
24170 8,
24171 );
24172 let other = if fail { "complete" } else { "fail" };
24173 assert_eq!(
24174 server.request_count(&format!("/api/worker/query-tasks/storage-query/{other}")),
24175 0
24176 );
24177 }
24178 }
24179
24180 #[tokio::test]
24181 async fn storage_recovery_does_not_override_auth_lease_or_invalid_contract() {
24182 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
24183 storage_refusals: 7,
24184 storage_path: Some("/poll"),
24185 unauthorized_polls: true,
24186 ..MockWorkerBehavior::default()
24187 });
24188 let error = storage_worker(&server)
24189 .run_once()
24190 .await
24191 .expect_err("auth remains terminal");
24192 assert!(matches!(
24193 error,
24194 Error::Http {
24195 status: reqwest::StatusCode::UNAUTHORIZED,
24196 ..
24197 }
24198 ));
24199 assert_identical_requests(&server, "/api/worker/workflow-tasks/poll", 8);
24200
24201 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
24202 storage_refusals: 7,
24203 storage_path: Some("/activity-cancel/complete"),
24204 ..MockWorkerBehavior::default()
24205 });
24206 let worker =
24207 storage_worker(&server).with_storage_admission(Arc::new(AtomicBool::new(false)));
24208 let error = worker
24209 .client
24210 .complete_activity_task(
24211 "activity-cancel",
24212 "attempt-cancel",
24213 "worker",
24214 json!({}),
24215 DEFAULT_CODEC,
24216 )
24217 .await
24218 .expect_err("cancellation remains terminal");
24219 assert!(activity_task_rejection_is_final(&error));
24220 assert_identical_requests(
24221 &server,
24222 "/api/worker/activity-tasks/activity-cancel/complete",
24223 8,
24224 );
24225
24226 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
24227 storage_refusals: usize::MAX,
24228 storage_path: Some("/poll"),
24229 storage_wrong_poll_id: true,
24230 ..MockWorkerBehavior::default()
24231 });
24232 assert!(storage_worker(&server).run_once().await.is_err());
24233 assert_eq!(server.request_count("/api/worker/workflow-tasks/poll"), 1);
24234 }
24235
24236 #[tokio::test]
24237 async fn storage_pollers_stop_when_the_run_future_is_aborted() {
24238 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
24239 storage_refusals: usize::MAX,
24240 storage_path: Some("/poll"),
24241 ..MockWorkerBehavior::default()
24242 });
24243 let mut worker = storage_worker(&server).retry_policy(WorkerRetryPolicy::default());
24244 worker.register_activity("unused", |_, _| async { Ok(Value::Null) });
24245 let run = tokio::spawn(async move { worker.run().await });
24246 tokio::time::timeout(Duration::from_secs(2), async {
24247 while server.request_count("/api/worker/activity-tasks/poll") == 0 {
24248 tokio::time::sleep(Duration::from_millis(1)).await;
24249 }
24250 })
24251 .await
24252 .expect("poll started");
24253 run.abort();
24254 assert!(run.await.expect_err("cancelled run").is_cancelled());
24255 tokio::time::sleep(Duration::from_millis(250)).await;
24256 assert_eq!(server.request_count("/api/worker/activity-tasks/poll"), 1);
24257 }
24258
24259 #[tokio::test]
24260 async fn query_protocol_rejection_from_older_server_is_typed() {
24261 let server = MockWorkerServer::reject_query_protocol();
24262 let client = Client::builder(server.base_url())
24263 .timeout(Duration::from_secs(2))
24264 .build()
24265 .expect("client");
24266
24267 let error = client
24268 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
24269 .await
24270 .expect_err("server below query protocol floor must reject");
24271 let Error::Protocol(failure) = error else {
24272 panic!("expected typed protocol failure");
24273 };
24274
24275 assert_eq!(failure.status, 400);
24276 assert_eq!(failure.reason, "unsupported_protocol_version");
24277 assert_eq!(failure.supported_version.as_deref(), Some("1.7"));
24278 assert_eq!(
24279 failure.requested_version.as_deref(),
24280 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
24281 );
24282 assert_eq!(
24283 server
24284 .worker_protocol_for("/api/worker/query-tasks/poll")
24285 .as_deref(),
24286 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
24287 );
24288 }
24289
24290 #[tokio::test]
24291 async fn run_once_without_query_handlers_keeps_pre_query_server_compatibility() {
24292 let server = MockWorkerServer::reject_query_protocol();
24293 let client = Client::builder(server.base_url())
24294 .timeout(Duration::from_secs(2))
24295 .build()
24296 .expect("client");
24297 let mut worker = Worker::new(client, "rust-workers")
24298 .worker_id("baseline-worker")
24299 .poll_timeout(Duration::from_millis(10));
24300
24301 worker.register_workflow("baseline.workflow", |_ctx, _input| async move {
24302 Ok(Value::Null)
24303 });
24304
24305 assert_eq!(worker.run_once().await.expect("baseline run once"), 0);
24306 assert_eq!(
24307 server
24308 .worker_protocol_for("/api/worker/workflow-tasks/poll")
24309 .as_deref(),
24310 Some(WORKER_PROTOCOL_VERSION)
24311 );
24312 assert_eq!(
24313 server.worker_protocol_for("/api/worker/query-tasks/poll"),
24314 None,
24315 "a worker without query handlers must not use the query-task endpoint"
24316 );
24317 }
24318
24319 #[tokio::test]
24320 async fn completion_time_query_rejection_is_typed_without_stopping_worker() {
24321 let server = MockWorkerServer::reject_query_completion();
24322 let client = Client::builder(server.base_url())
24323 .timeout(Duration::from_secs(2))
24324 .build()
24325 .expect("client");
24326
24327 let error = client
24328 .complete_query_task("query-late", "late-worker", 1, json!(8), DEFAULT_CODEC)
24329 .await
24330 .expect_err("expired completion must be rejected");
24331 let Error::QueryFailed(failure) = error else {
24332 panic!("expected typed query failure");
24333 };
24334 assert_eq!(failure.status, 409);
24335 assert_eq!(failure.reason, "query_task_timed_out");
24336
24337 let mut worker = Worker::new(client, "rust-workers")
24338 .worker_id("late-worker")
24339 .poll_timeout(Duration::from_millis(10));
24340 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
24341 worker.register_query(
24342 "counter",
24343 "current",
24344 |_ctx, _args| async move { Ok(json!(8)) },
24345 );
24346
24347 assert_eq!(worker.run_once().await.expect("late task is handled"), 1);
24348 assert_eq!(
24349 worker
24350 .run_once()
24351 .await
24352 .expect("worker continues after late completion"),
24353 0
24354 );
24355 assert_eq!(
24356 server.request_count("/api/worker/query-tasks/query-late/complete"),
24357 2
24358 );
24359 assert_eq!(
24360 server.request_count("/api/worker/query-tasks/query-late/fail"),
24361 0,
24362 "a server completion rejection must not be reported as an encoding failure"
24363 );
24364 }
24365
24366 #[tokio::test]
24367 async fn normal_shutdown_joins_pollers_and_deregisters_once() {
24368 let server = MockWorkerServer::start();
24369 let client = Client::builder(server.base_url())
24370 .timeout(Duration::from_secs(2))
24371 .build()
24372 .expect("client");
24373 let mut worker = Worker::new(client, "rust-workers")
24374 .worker_id("joined-worker")
24375 .poll_timeout(Duration::from_millis(10));
24376 worker.register_workflow(
24377 "joined.workflow",
24378 |_ctx, _input| async move { Ok(Value::Null) },
24379 );
24380 worker.register_activity(
24381 "joined.activity",
24382 |_ctx, _input| async move { Ok(Value::Null) },
24383 );
24384 worker.register_query("joined.workflow", "state", |_ctx, _input| async move {
24385 Ok(Value::Null)
24386 });
24387
24388 worker
24389 .run_until(tokio::time::sleep(Duration::from_millis(20)))
24390 .await
24391 .expect("normal shutdown");
24392
24393 let deregistration_path = "/api/worker/registrations/mock-worker";
24394 assert_eq!(server.request_count(deregistration_path), 1);
24395 for poll_path in [
24396 "/api/worker/workflow-tasks/poll",
24397 "/api/worker/activity-tasks/poll",
24398 "/api/worker/query-tasks/poll",
24399 ] {
24400 assert!(server.request_count(poll_path) > 0, "missing {poll_path}");
24401 }
24402 assert_eq!(
24403 server.captured_paths().last().map(String::as_str),
24404 Some(deregistration_path),
24405 "deregistration must start only after every poller has joined"
24406 );
24407 }
24408
24409 #[tokio::test]
24410 async fn registration_failure_does_not_deregister() {
24411 let server = MockWorkerServer::rejected_registration();
24412 let client = Client::builder(server.base_url())
24413 .timeout(Duration::from_secs(2))
24414 .build()
24415 .expect("client");
24416 let worker = Worker::new(client, "rust-workers").worker_id("never-registered");
24417
24418 let error = worker
24419 .run_until(async {})
24420 .await
24421 .expect_err("registration must fail");
24422 assert!(matches!(
24423 error,
24424 Error::Http {
24425 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
24426 ..
24427 }
24428 ));
24429 assert!(server
24430 .captured_paths()
24431 .iter()
24432 .all(|path| !path.starts_with("/api/worker/registrations/")));
24433 }
24434
24435 #[tokio::test]
24436 async fn protocol_116_server_rejects_occurrence_identity_worker_registration() {
24437 let server = MockWorkerServer::rejected_registration_protocol();
24438 let client = Client::builder(server.base_url())
24439 .timeout(Duration::from_secs(2))
24440 .build()
24441 .expect("client");
24442 let worker = Worker::new(client, "rust-workers").worker_id("protocol-117-worker");
24443
24444 let error = worker
24445 .run_until(async {})
24446 .await
24447 .expect_err("a protocol 1.16 server must reject this worker");
24448 let Error::Protocol(failure) = error else {
24449 panic!("expected typed protocol rejection");
24450 };
24451 assert_eq!(failure.reason, "unsupported_protocol_version");
24452 assert_eq!(failure.supported_version.as_deref(), Some("1.16"));
24453 assert_eq!(failure.requested_version.as_deref(), Some("1.17"));
24454 assert_eq!(
24455 server
24456 .worker_protocol_for("/api/worker/register")
24457 .as_deref(),
24458 Some(WORKER_PROTOCOL_VERSION)
24459 );
24460 }
24461
24462 #[tokio::test]
24463 async fn declined_registration_does_not_deregister() {
24464 let server = MockWorkerServer::declined_registration();
24465 let client = Client::builder(server.base_url())
24466 .timeout(Duration::from_secs(2))
24467 .build()
24468 .expect("client");
24469 let worker = Worker::new(client, "rust-workers").worker_id("declined-worker");
24470
24471 let error = worker
24472 .run_until(async {})
24473 .await
24474 .expect_err("declined registration must fail");
24475 assert!(matches!(error, Error::WorkerLoop(_)));
24476 assert!(error.to_string().contains("was not accepted"));
24477 assert!(server
24478 .captured_paths()
24479 .iter()
24480 .all(|path| !path.starts_with("/api/worker/registrations/")));
24481 }
24482
24483 #[tokio::test]
24484 async fn deregistration_http_failure_is_returned_after_normal_shutdown() {
24485 let server = MockWorkerServer::rejected_deregistration();
24486 let client = Client::builder(server.base_url())
24487 .timeout(Duration::from_secs(2))
24488 .build()
24489 .expect("client");
24490 let worker = Worker::new(client, "rust-workers").worker_id("forbidden-cleanup");
24491
24492 let error = worker
24493 .run_until(async {})
24494 .await
24495 .expect_err("deregistration must fail");
24496 assert!(matches!(
24497 error,
24498 Error::Http {
24499 status: reqwest::StatusCode::FORBIDDEN,
24500 ..
24501 }
24502 ));
24503 assert_eq!(
24504 server.request_count("/api/worker/registrations/mock-worker"),
24505 1
24506 );
24507 }
24508
24509 #[tokio::test]
24510 async fn deregistration_protocol_failure_is_returned_after_normal_shutdown() {
24511 let server = MockWorkerServer::rejected_deregistration_protocol();
24512 let client = Client::builder(server.base_url())
24513 .timeout(Duration::from_secs(2))
24514 .build()
24515 .expect("client");
24516 let worker = Worker::new(client, "rust-workers").worker_id("protocol-cleanup");
24517
24518 let error = worker
24519 .run_until(async {})
24520 .await
24521 .expect_err("protocol rejection must fail shutdown");
24522 let Error::Protocol(failure) = error else {
24523 panic!("expected typed protocol failure");
24524 };
24525 assert_eq!(failure.reason, "unsupported_protocol_version");
24526 assert_eq!(
24527 failure.requested_version.as_deref(),
24528 Some(WORKER_PROTOCOL_VERSION)
24529 );
24530 assert_eq!(
24531 server.request_count("/api/worker/registrations/mock-worker"),
24532 1
24533 );
24534 }
24535
24536 #[tokio::test]
24537 async fn primary_poller_error_retains_deregistration_failure_context() {
24538 let server = MockWorkerServer::unauthorized_polls_and_rejected_deregistration();
24539 let client = Client::builder(server.base_url())
24540 .timeout(Duration::from_secs(2))
24541 .build()
24542 .expect("client");
24543 let mut worker = Worker::new(client, "rust-workers")
24544 .worker_id("combined-failure")
24545 .poll_timeout(Duration::from_millis(10));
24546 worker.register_workflow("combined.workflow", |_ctx, _input| async move {
24547 Ok(Value::Null)
24548 });
24549
24550 let error = worker
24551 .run()
24552 .await
24553 .expect_err("worker and cleanup must fail");
24554 let summary = error.to_string();
24555 assert!(summary.contains("authentication_failed"));
24556 assert!(summary.contains("worker cannot deregister"));
24557 let Error::WorkerShutdown {
24558 primary,
24559 deregistration,
24560 } = error
24561 else {
24562 panic!("expected combined worker shutdown error");
24563 };
24564 assert!(matches!(
24565 *primary,
24566 Error::Http {
24567 status: reqwest::StatusCode::UNAUTHORIZED,
24568 ..
24569 }
24570 ));
24571 assert!(matches!(
24572 *deregistration,
24573 Error::Http {
24574 status: reqwest::StatusCode::FORBIDDEN,
24575 ..
24576 }
24577 ));
24578 assert_eq!(
24579 server.request_count("/api/worker/registrations/mock-worker"),
24580 1
24581 );
24582 }
24583
24584 #[tokio::test]
24585 async fn activity_only_worker_can_shutdown_without_workflow_poller() {
24586 let server = MockWorkerServer::start();
24587 let client = Client::builder(server.base_url())
24588 .timeout(Duration::from_secs(2))
24589 .build()
24590 .expect("client");
24591 let mut worker = Worker::new(client, "rust-workers")
24592 .worker_id("activity-only-worker")
24593 .poll_timeout(Duration::from_millis(10));
24594
24595 worker.register_activity(
24596 "activity.only",
24597 |_ctx, _args| async move { Ok(Value::Null) },
24598 );
24599
24600 worker.run_until(async {}).await.expect("run worker");
24601 }
24602
24603 #[tokio::test]
24604 async fn workflow_only_worker_can_shutdown_without_activity_poller() {
24605 let server = MockWorkerServer::start();
24606 let client = Client::builder(server.base_url())
24607 .timeout(Duration::from_secs(2))
24608 .build()
24609 .expect("client");
24610 let mut worker = Worker::new(client, "rust-workers")
24611 .worker_id("workflow-only-worker")
24612 .poll_timeout(Duration::from_millis(10));
24613
24614 worker.register_workflow(
24615 "workflow.only",
24616 |_ctx, _input| async move { Ok(Value::Null) },
24617 );
24618
24619 worker.run_until(async {}).await.expect("run worker");
24620 }
24621
24622 #[tokio::test]
24623 async fn worker_heartbeat_observer_receives_server_acknowledgements() {
24624 let server = MockWorkerServer::start();
24625 let client = Client::builder(server.base_url())
24626 .timeout(Duration::from_secs(2))
24627 .build()
24628 .expect("client");
24629 let observations = Arc::new(Mutex::new(Vec::new()));
24630 let observed = Arc::clone(&observations);
24631 let mut worker = Worker::new(client, "rust-workers")
24632 .worker_id("observed-heartbeat-worker")
24633 .poll_timeout(Duration::from_millis(10))
24634 .on_worker_heartbeat(move |observation| {
24635 observed
24636 .lock()
24637 .expect("heartbeat observations")
24638 .push(observation.clone());
24639 });
24640
24641 worker.register_workflow("workflow.observed", |_ctx, _input| async move {
24642 Ok(Value::Null)
24643 });
24644 let acknowledged = Arc::clone(&observations);
24645 worker
24646 .run_until(async move {
24647 tokio::time::timeout(Duration::from_secs(2), async move {
24648 loop {
24649 if !acknowledged
24650 .lock()
24651 .expect("heartbeat observations")
24652 .is_empty()
24653 {
24654 break;
24655 }
24656 tokio::time::sleep(Duration::from_millis(1)).await;
24657 }
24658 })
24659 .await
24660 .expect("heartbeat acknowledgement within timeout");
24661 })
24662 .await
24663 .expect("run worker");
24664
24665 let observations = observations.lock().expect("heartbeat observations");
24666 let first = observations.first().expect("heartbeat acknowledgement");
24667 assert_eq!(first.worker_id, "observed-heartbeat-worker");
24668 assert_eq!(first.task_queue, "rust-workers");
24669 assert!(first.acknowledged_at_unix_millis > 0);
24670 assert_eq!(first.acknowledgement, json!({}));
24671 }
24672
24673 #[tokio::test]
24674 async fn delayed_worker_heartbeat_keeps_cadence_and_pollers_live() {
24675 let server = MockWorkerServer::delayed_heartbeat_worker();
24676 let client = Client::builder(server.base_url())
24677 .timeout(Duration::from_secs(3))
24678 .build()
24679 .expect("client");
24680 let observations = Arc::new(Mutex::new(Vec::new()));
24681 let observed = Arc::clone(&observations);
24682 let mut worker = Worker::new(client, "rust-snapshot-workers")
24683 .worker_id("rust-snapshot-worker")
24684 .poll_timeout(Duration::from_millis(10))
24685 .on_worker_heartbeat(move |observation| {
24686 observed
24687 .lock()
24688 .expect("heartbeat observations")
24689 .push(observation.clone());
24690 });
24691
24692 worker.register_workflow("snapshot", |ctx, _input| async move {
24693 ctx.wait_signal("finish").await?;
24694 Ok(json!({"status": "finished"}))
24695 });
24696 worker.register_query("snapshot", "current", |ctx, _args| async move {
24697 Ok(json!(ctx
24698 .signals("increment")
24699 .iter()
24700 .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
24701 .sum::<i64>()))
24702 });
24703 worker.register_activity("cancel-aware", |_ctx, _args| async move {
24704 Ok(json!({"late": "completion"}))
24705 });
24706
24707 worker
24708 .run_until(tokio::time::sleep(Duration::from_millis(3_800)))
24709 .await
24710 .expect("delayed heartbeat must allow a clean worker shutdown");
24711
24712 let observations = observations.lock().expect("heartbeat observations");
24713 assert!(
24714 observations.len() >= 3,
24715 "the immediate heartbeat, delayed acknowledgement, and next cadence heartbeat must complete"
24716 );
24717 assert!(
24718 observations.windows(2).all(|pair| {
24719 pair[1].acknowledged_at_unix_millis
24720 .saturating_sub(pair[0].acknowledged_at_unix_millis)
24721 >= 850
24722 }),
24723 "successful acknowledgements must not catch up faster than the advertised one-second cadence: {observations:?}"
24724 );
24725 drop(observations);
24726
24727 let heartbeat_times = server.request_times("/api/worker/heartbeat");
24728 let delayed_request_at = *heartbeat_times
24729 .get(1)
24730 .expect("intentionally delayed heartbeat request");
24731 let delay_window_start = delayed_request_at + Duration::from_millis(100);
24732 let delay_window_end = delayed_request_at + Duration::from_millis(1_400);
24733 for path in [
24734 "/api/worker/workflow-tasks/poll",
24735 "/api/worker/activity-tasks/poll",
24736 "/api/worker/query-tasks/poll",
24737 ] {
24738 assert!(
24739 server
24740 .request_times(path)
24741 .iter()
24742 .any(|received_at| *received_at >= delay_window_start
24743 && *received_at <= delay_window_end),
24744 "{path} must keep polling while a heartbeat acknowledgement is delayed"
24745 );
24746 }
24747 assert!(
24748 server.request_count("/api/worker/workflow-tasks/snapshot-wait-3/fail") >= 1,
24749 "workflow work must be settled"
24750 );
24751 assert!(
24752 server.request_count("/api/worker/activity-tasks/activity-cancel/complete") >= 1,
24753 "activity work must be settled"
24754 );
24755 assert!(
24756 server.request_count("/api/worker/query-tasks/snapshot-current/complete") >= 1,
24757 "query work must be settled"
24758 );
24759 }
24760
24761 #[tokio::test]
24762 async fn retried_worker_heartbeat_restarts_the_advertised_cadence() {
24763 let server = MockWorkerServer::heartbeat_retry_worker();
24764 let client = Client::builder(server.base_url())
24765 .timeout(Duration::from_secs(2))
24766 .build()
24767 .expect("client");
24768 let observations = Arc::new(Mutex::new(Vec::new()));
24769 let observed = Arc::clone(&observations);
24770 let worker = Worker::new(client, "rust-workers")
24771 .worker_id("heartbeat-retry-worker")
24772 .retry_policy(WorkerRetryPolicy {
24773 max_retries: 1,
24774 initial_backoff: Duration::from_millis(300),
24775 max_backoff: Duration::from_millis(300),
24776 })
24777 .on_worker_heartbeat(move |observation| {
24778 observed
24779 .lock()
24780 .expect("heartbeat observations")
24781 .push(observation.clone());
24782 });
24783
24784 worker
24785 .run_until(tokio::time::sleep(Duration::from_millis(2_700)))
24786 .await
24787 .expect("retryable heartbeat failure must remain bounded and recover");
24788
24789 let observations = observations.lock().expect("heartbeat observations");
24790 assert!(observations.len() >= 3, "heartbeat retry must recover");
24791 assert!(
24792 observations.windows(2).all(|pair| {
24793 pair[1]
24794 .acknowledged_at_unix_millis
24795 .saturating_sub(pair[0].acknowledged_at_unix_millis)
24796 >= 850
24797 }),
24798 "a successful retry must start a fresh advertised cadence: {observations:?}"
24799 );
24800 assert_eq!(
24801 server.request_count("/api/worker/heartbeat"),
24802 observations.len() + 1,
24803 "one retryable failure must add exactly one bounded request"
24804 );
24805 }
24806
24807 #[tokio::test]
24808 async fn query_enabled_worker_ignores_unmatched_signals_then_completes_once() {
24809 let server = MockWorkerServer::waiting_query_worker();
24810 let client = Client::builder(server.base_url())
24811 .timeout(Duration::from_secs(2))
24812 .build()
24813 .expect("client");
24814 let observations = Arc::new(Mutex::new(Vec::new()));
24815 let observed = Arc::clone(&observations);
24816 let mut worker = Worker::new(client, "rust-snapshot-workers")
24817 .worker_id("rust-snapshot-worker")
24818 .poll_timeout(Duration::from_millis(10))
24819 .on_worker_heartbeat(move |observation| {
24820 observed
24821 .lock()
24822 .expect("heartbeat observations")
24823 .push(observation.clone());
24824 });
24825
24826 worker.register_workflow("snapshot", |ctx, _input| async move {
24827 ctx.wait_signal("finish").await?;
24828 Ok(json!({"status": "finished"}))
24829 });
24830 worker.register_query("snapshot", "current", |ctx, _args| async move {
24831 let current = ctx
24832 .signals("increment")
24833 .iter()
24834 .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
24835 .sum::<i64>();
24836 Ok(json!(current))
24837 });
24838 worker.register_update("snapshot", "replace", |_ctx, args| async move { Ok(args) });
24839
24840 worker
24841 .run_until(tokio::time::sleep(Duration::from_millis(3_200)))
24842 .await
24843 .expect("pending workflow and query poller must remain live until shutdown");
24844
24845 assert!(
24846 observations.lock().expect("heartbeat observations").len() >= 4,
24847 "the immediate heartbeat and at least three advertised one-second intervals must be acknowledged"
24848 );
24849 assert!(
24850 server.request_count("/api/worker/workflow-tasks/poll") >= 3,
24851 "workflow polling must continue after empty replay acknowledgements"
24852 );
24853 assert!(
24854 server.request_count("/api/worker/query-tasks/poll") >= 2,
24855 "query polling must continue after serving the current query"
24856 );
24857 assert_eq!(
24858 server.request_body("/api/worker/register")["capabilities"],
24859 json!([
24860 CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY,
24861 DURABLE_SELECTION_CAPABILITY,
24862 MEMO_UPSERTS_CAPABILITY,
24863 TYPED_SEARCH_ATTRIBUTES_CAPABILITY,
24864 QUERY_TASKS_CAPABILITY,
24865 WORKFLOW_UPDATES_CAPABILITY,
24866 MESSAGE_STREAMS_CAPABILITY
24867 ])
24868 );
24869 assert_eq!(
24870 server.request_body("/api/worker/register")["workflow_command_contracts"]["snapshot"],
24871 json!({
24872 "queries": ["current"],
24873 "query_contracts": [],
24874 "signals": [],
24875 "signal_contracts": [],
24876 "updates": ["replace"],
24877 "update_contracts": [],
24878 "update_validators": [],
24879 })
24880 );
24881
24882 let opened = server.request_body("/api/worker/workflow-tasks/snapshot-open/complete");
24883 assert_eq!(
24884 opened["commands"],
24885 json!([{
24886 "type": "open_signal_wait",
24887 "signal_name": "finish",
24888 }])
24889 );
24890
24891 for task_id in ["snapshot-wait-3", "snapshot-wait-5"] {
24892 let fail_path = format!("/api/worker/workflow-tasks/{task_id}/fail");
24893 let completion_path = format!("/api/worker/workflow-tasks/{task_id}/complete");
24894 let failure = server.request_body(&fail_path);
24895 assert_eq!(
24896 failure["failure"]["type"],
24897 WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE
24898 );
24899 assert_eq!(server.request_count(&completion_path), 0);
24900 }
24901
24902 let query_completion =
24903 server.request_body("/api/worker/query-tasks/snapshot-current/complete");
24904 assert_eq!(query_completion["result"], json!(8));
24905
24906 let terminal_path = "/api/worker/workflow-tasks/snapshot-finish/complete";
24907 assert_eq!(
24908 server.request_count(terminal_path),
24909 1,
24910 "the matching signal must settle the workflow exactly once"
24911 );
24912 let terminal = server.request_body(terminal_path);
24913 assert_eq!(terminal["commands"].as_array().map(Vec::len), Some(1));
24914 assert_eq!(terminal["commands"][0]["type"], "complete_workflow");
24915 assert_eq!(
24916 decode_wire_value(&terminal["commands"][0]["result"], DEFAULT_CODEC)
24917 .expect("terminal workflow result"),
24918 json!({"status": "finished"})
24919 );
24920 }
24921
24922 #[tokio::test]
24923 async fn worker_retries_poll_and_heartbeat_transport_failures_independently() {
24924 let server = MockWorkerServer::transient_worker_failures();
24925 let client = Client::builder(server.base_url())
24926 .timeout(Duration::from_secs(2))
24927 .build()
24928 .expect("client");
24929 let mut worker = Worker::new(client, "rust-workers")
24930 .worker_id("retry-worker")
24931 .poll_timeout(Duration::from_millis(10))
24932 .retry_policy(WorkerRetryPolicy {
24933 max_retries: 2,
24934 initial_backoff: Duration::from_millis(1),
24935 max_backoff: Duration::from_millis(1),
24936 });
24937 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
24938 worker.register_activity(
24939 "counter.activity",
24940 |_ctx, _input| async move { Ok(Value::Null) },
24941 );
24942 worker.register_query(
24943 "counter",
24944 "current",
24945 |_ctx, _args| async move { Ok(json!(8)) },
24946 );
24947
24948 worker
24949 .run_until(tokio::time::sleep(Duration::from_millis(75)))
24950 .await
24951 .expect("transient failures must not stop the worker");
24952
24953 for path in [
24954 "/api/worker/heartbeat",
24955 "/api/worker/workflow-tasks/poll",
24956 "/api/worker/activity-tasks/poll",
24957 "/api/worker/query-tasks/poll",
24958 ] {
24959 assert!(
24960 server.request_count(path) >= 2,
24961 "{path} must continue after its transient failure"
24962 );
24963 }
24964 }
24965
24966 #[tokio::test]
24967 async fn worker_continues_after_long_poll_capacity_backpressure() {
24968 let server = MockWorkerServer::capacity_limited_activity_poll();
24969 let client = Client::builder(server.base_url())
24970 .timeout(Duration::from_secs(2))
24971 .build()
24972 .expect("client");
24973 let mut worker = Worker::new(client, "rust-workers")
24974 .worker_id("capacity-worker")
24975 .poll_timeout(Duration::from_millis(10))
24976 .retry_policy(WorkerRetryPolicy {
24977 max_retries: 0,
24978 initial_backoff: Duration::from_millis(1),
24979 max_backoff: Duration::from_millis(1),
24980 });
24981 worker.register_activity("capacity.activity", |_ctx, _input| async move {
24982 Ok(json!({"handled": true}))
24983 });
24984
24985 worker
24986 .run_until(tokio::time::sleep(Duration::from_millis(50)))
24987 .await
24988 .expect("capacity backpressure must not stop the worker");
24989
24990 assert!(
24991 server.request_count("/api/worker/activity-tasks/poll") >= 2,
24992 "the activity poller must continue after capacity backpressure"
24993 );
24994 assert_eq!(
24995 server.request_count("/api/worker/activity-tasks/capacity-activity/complete"),
24996 1,
24997 "the worker must complete work returned after capacity recovers"
24998 );
24999 }
25000
25001 #[test]
25002 fn worker_poll_capacity_backpressure_requires_the_typed_retryable_contract() {
25003 let capacity = Error::Http {
25004 status: reqwest::StatusCode::TOO_MANY_REQUESTS,
25005 body: r#"{"poll_status":"long_poll_capacity_exhausted","retryable":true,"retry_after_seconds":3}"#.to_string(),
25006 };
25007 assert_eq!(
25008 worker_poll_capacity_retry_after(&capacity),
25009 Some(Duration::from_secs(3))
25010 );
25011
25012 let rejected_capacity = Error::Http {
25013 status: reqwest::StatusCode::TOO_MANY_REQUESTS,
25014 body: r#"{"reason":"long_poll_capacity_exhausted","retryable":false,"retry_after_seconds":3}"#.to_string(),
25015 };
25016 assert_eq!(worker_poll_capacity_retry_after(&rejected_capacity), None);
25017 assert!(!worker_operation_is_retryable(&rejected_capacity));
25018
25019 let ordinary_rate_limit = Error::Http {
25020 status: reqwest::StatusCode::TOO_MANY_REQUESTS,
25021 body: r#"{"reason":"rate_limited","retryable":true,"retry_after_seconds":3}"#
25022 .to_string(),
25023 };
25024 assert_eq!(worker_poll_capacity_retry_after(&ordinary_rate_limit), None);
25025 assert!(worker_operation_is_retryable(&ordinary_rate_limit));
25026 }
25027
25028 #[tokio::test]
25029 async fn worker_bounds_transport_retries() {
25030 let server = MockWorkerServer::unavailable_polls();
25031 let client = Client::builder(server.base_url())
25032 .timeout(Duration::from_secs(2))
25033 .build()
25034 .expect("client");
25035 let mut worker = Worker::new(client, "rust-workers")
25036 .worker_id("bounded-retry-worker")
25037 .poll_timeout(Duration::from_millis(10))
25038 .retry_policy(WorkerRetryPolicy {
25039 max_retries: 2,
25040 initial_backoff: Duration::from_millis(1),
25041 max_backoff: Duration::from_millis(1),
25042 });
25043 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
25044
25045 let error = worker.run().await.expect_err("retry bound must terminate");
25046 assert!(matches!(error, Error::Transport(_)));
25047 assert_eq!(
25048 server.request_count("/api/worker/workflow-tasks/poll"),
25049 3,
25050 "one initial request plus exactly two retries"
25051 );
25052 }
25053
25054 #[tokio::test]
25055 async fn worker_retry_policy_can_disable_poll_retries() {
25056 let server = MockWorkerServer::unavailable_polls();
25057 let client = Client::builder(server.base_url())
25058 .timeout(Duration::from_secs(2))
25059 .build()
25060 .expect("client");
25061 let mut worker = Worker::new(client, "rust-workers")
25062 .worker_id("no-retry-worker")
25063 .poll_timeout(Duration::from_millis(10))
25064 .retry_policy(WorkerRetryPolicy {
25065 max_retries: 0,
25066 initial_backoff: Duration::from_millis(1),
25067 max_backoff: Duration::from_millis(1),
25068 });
25069 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
25070
25071 let error = worker
25072 .run_once()
25073 .await
25074 .expect_err("disabled retries must return the first transport failure");
25075 assert!(matches!(error, Error::Transport(_)));
25076 assert_eq!(
25077 server.request_count("/api/worker/workflow-tasks/poll"),
25078 1,
25079 "max_retries=0 must send only the initial request"
25080 );
25081 }
25082
25083 #[tokio::test]
25084 async fn worker_does_not_retry_authentication_failures() {
25085 let server = MockWorkerServer::unauthorized_polls();
25086 let client = Client::builder(server.base_url())
25087 .timeout(Duration::from_secs(2))
25088 .build()
25089 .expect("client");
25090 let mut worker = Worker::new(client, "rust-workers")
25091 .worker_id("unauthorized-worker")
25092 .poll_timeout(Duration::from_millis(10));
25093 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
25094
25095 let error = worker
25096 .run()
25097 .await
25098 .expect_err("authentication must terminate");
25099 let Error::Http { status, body } = error else {
25100 panic!("expected stable HTTP authentication error");
25101 };
25102 assert_eq!(status, reqwest::StatusCode::UNAUTHORIZED);
25103 assert!(body.contains("authentication_failed"));
25104 assert_eq!(
25105 server.request_count("/api/worker/workflow-tasks/poll"),
25106 1,
25107 "authentication failures must not be retried"
25108 );
25109 }
25110
25111 #[derive(Clone, Debug)]
25112 struct CapturedRequest {
25113 headers: String,
25114 method: String,
25115 path: String,
25116 authorization: Option<String>,
25117 namespace: Option<String>,
25118 worker_protocol: Option<String>,
25119 control_protocol: Option<String>,
25120 body: String,
25121 received_at: Instant,
25122 }
25123
25124 struct MockWorkerServer {
25125 addr: SocketAddr,
25126 stop: Arc<AtomicBool>,
25127 requests: Arc<Mutex<Vec<CapturedRequest>>>,
25128 thread: Option<thread::JoinHandle<()>>,
25129 }
25130
25131 type RequestOverride = fn(&str, &str, usize) -> Option<(&'static str, String)>;
25132
25133 #[derive(Clone, Copy, Default)]
25134 struct MockWorkerBehavior {
25135 response_override: Option<fn(&str) -> Option<(&'static str, String)>>,
25136 request_override: Option<RequestOverride>,
25137 storage_refusals: usize,
25138 storage_path: Option<&'static str>,
25139 storage_unavailable: bool,
25140 storage_mid_poll: bool,
25141 storage_activity: bool,
25142 storage_query: bool,
25143 storage_wrong_poll_id: bool,
25144 reject_query_protocol: bool,
25145 reject_query_completion: bool,
25146 waiting_query_worker: bool,
25147 decline_registration: bool,
25148 complete_named_signal: bool,
25149 poll_failures_per_path: usize,
25150 long_poll_capacity_responses_per_path: usize,
25151 heartbeat_failures: usize,
25152 heartbeat_failure_request: Option<usize>,
25153 delayed_heartbeat_request: Option<usize>,
25154 heartbeat_response_delay: Duration,
25155 concurrent_requests: bool,
25156 unauthorized_polls: bool,
25157 reject_registration: bool,
25158 reject_registration_protocol: bool,
25159 reject_deregistration: bool,
25160 reject_deregistration_protocol: bool,
25161 cancelled_activity: bool,
25162 draining_polls: bool,
25163 invalid_task_payload_codec: Option<InvalidTaskPayloadCodec>,
25164 workflow_completion_status: Option<&'static str>,
25165 workflow_completion_body: Option<&'static str>,
25166 }
25167
25168 impl MockWorkerServer {
25169 fn start() -> Self {
25170 Self::start_with_behavior(MockWorkerBehavior::default())
25171 }
25172
25173 fn reject_query_protocol() -> Self {
25174 Self::start_with_behavior(MockWorkerBehavior {
25175 reject_query_protocol: true,
25176 ..MockWorkerBehavior::default()
25177 })
25178 }
25179
25180 fn reject_query_completion() -> Self {
25181 Self::start_with_behavior(MockWorkerBehavior {
25182 reject_query_completion: true,
25183 ..MockWorkerBehavior::default()
25184 })
25185 }
25186
25187 fn waiting_query_worker() -> Self {
25188 Self::start_with_behavior(MockWorkerBehavior {
25189 waiting_query_worker: true,
25190 complete_named_signal: true,
25191 ..MockWorkerBehavior::default()
25192 })
25193 }
25194
25195 fn transient_worker_failures() -> Self {
25196 Self::start_with_behavior(MockWorkerBehavior {
25197 poll_failures_per_path: 1,
25198 heartbeat_failures: 1,
25199 ..MockWorkerBehavior::default()
25200 })
25201 }
25202
25203 fn consecutive_poll_failures(count: usize) -> Self {
25204 Self::start_with_behavior(MockWorkerBehavior {
25205 poll_failures_per_path: count,
25206 ..MockWorkerBehavior::default()
25207 })
25208 }
25209
25210 fn capacity_limited_activity_poll() -> Self {
25211 Self::start_with_behavior(MockWorkerBehavior {
25212 long_poll_capacity_responses_per_path: 1,
25213 ..MockWorkerBehavior::default()
25214 })
25215 }
25216
25217 fn delayed_heartbeat_worker() -> Self {
25218 Self::start_with_behavior(MockWorkerBehavior {
25219 waiting_query_worker: true,
25220 delayed_heartbeat_request: Some(2),
25221 heartbeat_response_delay: Duration::from_millis(1_500),
25222 concurrent_requests: true,
25223 cancelled_activity: true,
25224 ..MockWorkerBehavior::default()
25225 })
25226 }
25227
25228 fn heartbeat_retry_worker() -> Self {
25229 Self::start_with_behavior(MockWorkerBehavior {
25230 waiting_query_worker: true,
25231 heartbeat_failure_request: Some(2),
25232 concurrent_requests: true,
25233 ..MockWorkerBehavior::default()
25234 })
25235 }
25236
25237 fn unavailable_polls() -> Self {
25238 Self::start_with_behavior(MockWorkerBehavior {
25239 poll_failures_per_path: usize::MAX,
25240 ..MockWorkerBehavior::default()
25241 })
25242 }
25243
25244 fn unauthorized_polls() -> Self {
25245 Self::start_with_behavior(MockWorkerBehavior {
25246 unauthorized_polls: true,
25247 ..MockWorkerBehavior::default()
25248 })
25249 }
25250
25251 fn rejected_registration() -> Self {
25252 Self::start_with_behavior(MockWorkerBehavior {
25253 reject_registration: true,
25254 ..MockWorkerBehavior::default()
25255 })
25256 }
25257
25258 fn rejected_registration_protocol() -> Self {
25259 Self::start_with_behavior(MockWorkerBehavior {
25260 reject_registration_protocol: true,
25261 ..MockWorkerBehavior::default()
25262 })
25263 }
25264
25265 fn declined_registration() -> Self {
25266 Self::start_with_behavior(MockWorkerBehavior {
25267 decline_registration: true,
25268 ..MockWorkerBehavior::default()
25269 })
25270 }
25271
25272 fn rejected_deregistration() -> Self {
25273 Self::start_with_behavior(MockWorkerBehavior {
25274 reject_deregistration: true,
25275 ..MockWorkerBehavior::default()
25276 })
25277 }
25278
25279 fn rejected_deregistration_protocol() -> Self {
25280 Self::start_with_behavior(MockWorkerBehavior {
25281 reject_deregistration_protocol: true,
25282 ..MockWorkerBehavior::default()
25283 })
25284 }
25285
25286 fn unauthorized_polls_and_rejected_deregistration() -> Self {
25287 Self::start_with_behavior(MockWorkerBehavior {
25288 unauthorized_polls: true,
25289 reject_deregistration: true,
25290 ..MockWorkerBehavior::default()
25291 })
25292 }
25293
25294 fn cancelled_activity() -> Self {
25295 Self::start_with_behavior(MockWorkerBehavior {
25296 cancelled_activity: true,
25297 ..MockWorkerBehavior::default()
25298 })
25299 }
25300
25301 fn draining_polls() -> Self {
25302 Self::start_with_behavior(MockWorkerBehavior {
25303 draining_polls: true,
25304 ..MockWorkerBehavior::default()
25305 })
25306 }
25307
25308 fn invalid_task_payload_codec(codec: InvalidTaskPayloadCodec) -> Self {
25309 Self::start_with_behavior(MockWorkerBehavior {
25310 invalid_task_payload_codec: Some(codec),
25311 ..MockWorkerBehavior::default()
25312 })
25313 }
25314
25315 fn workflow_completion(status: &'static str, body: &'static str) -> Self {
25316 Self::start_with_behavior(MockWorkerBehavior {
25317 workflow_completion_status: Some(status),
25318 workflow_completion_body: Some(body),
25319 ..MockWorkerBehavior::default()
25320 })
25321 }
25322
25323 fn start_with_behavior(behavior: MockWorkerBehavior) -> Self {
25324 let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock server");
25325 listener
25326 .set_nonblocking(true)
25327 .expect("configure mock listener");
25328 let addr = listener.local_addr().expect("mock server address");
25329 let stop = Arc::new(AtomicBool::new(false));
25330 let server_stop = Arc::clone(&stop);
25331 let requests = Arc::new(Mutex::new(Vec::new()));
25332 let server_requests = Arc::clone(&requests);
25333 let thread = thread::spawn(move || {
25334 let mut request_threads = Vec::new();
25335 while !server_stop.load(Ordering::SeqCst) {
25336 match listener.accept() {
25337 Ok((mut stream, _)) => {
25338 if behavior.concurrent_requests {
25339 let requests = Arc::clone(&server_requests);
25340 request_threads.push(thread::spawn(move || {
25341 handle_mock_worker_request(&mut stream, &requests, behavior)
25342 }));
25343 } else {
25344 handle_mock_worker_request(&mut stream, &server_requests, behavior);
25345 }
25346 }
25347 Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
25348 let mut index = 0;
25349 while index < request_threads.len() {
25350 if request_threads[index].is_finished() {
25351 request_threads
25352 .swap_remove(index)
25353 .join()
25354 .expect("join mock request");
25355 } else {
25356 index += 1;
25357 }
25358 }
25359 thread::sleep(Duration::from_millis(5));
25360 }
25361 Err(_) => break,
25362 }
25363 }
25364 for request_thread in request_threads {
25365 request_thread.join().expect("join mock request");
25366 }
25367 });
25368
25369 Self {
25370 addr,
25371 stop,
25372 requests,
25373 thread: Some(thread),
25374 }
25375 }
25376
25377 fn base_url(&self) -> String {
25378 format!("http://{}", self.addr)
25379 }
25380
25381 fn worker_protocol_for(&self, path: &str) -> Option<String> {
25382 self.requests
25383 .lock()
25384 .expect("captured requests")
25385 .iter()
25386 .find(|request| request.path == path)
25387 .and_then(|request| request.worker_protocol.clone())
25388 }
25389
25390 fn control_protocol_for(&self, path: &str) -> Option<String> {
25391 self.requests
25392 .lock()
25393 .expect("captured requests")
25394 .iter()
25395 .find(|request| request.path == path)
25396 .and_then(|request| request.control_protocol.clone())
25397 }
25398
25399 fn method_for(&self, path: &str) -> Option<String> {
25400 self.requests
25401 .lock()
25402 .expect("captured requests")
25403 .iter()
25404 .find(|request| request.path == path)
25405 .map(|request| request.method.clone())
25406 }
25407
25408 fn authorization_for(&self, path: &str) -> Option<String> {
25409 self.requests
25410 .lock()
25411 .expect("captured requests")
25412 .iter()
25413 .find(|request| request.path == path)
25414 .and_then(|request| request.authorization.clone())
25415 }
25416
25417 fn namespace_for(&self, path: &str) -> Option<String> {
25418 self.requests
25419 .lock()
25420 .expect("captured requests")
25421 .iter()
25422 .find(|request| request.path == path)
25423 .and_then(|request| request.namespace.clone())
25424 }
25425
25426 fn request_count(&self, path: &str) -> usize {
25427 self.requests
25428 .lock()
25429 .expect("captured requests")
25430 .iter()
25431 .filter(|request| request.path == path)
25432 .count()
25433 }
25434
25435 fn captured_paths(&self) -> Vec<String> {
25436 self.requests
25437 .lock()
25438 .expect("captured requests")
25439 .iter()
25440 .map(|request| request.path.clone())
25441 .collect()
25442 }
25443
25444 fn request_times(&self, path: &str) -> Vec<Instant> {
25445 self.requests
25446 .lock()
25447 .expect("captured requests")
25448 .iter()
25449 .filter(|request| request.path == path)
25450 .map(|request| request.received_at)
25451 .collect()
25452 }
25453
25454 fn request_body(&self, path: &str) -> Value {
25455 let requests = self.requests.lock().expect("captured requests");
25456 let body = &requests
25457 .iter()
25458 .find(|request| request.path == path)
25459 .unwrap_or_else(|| panic!("missing request for {path}"))
25460 .body;
25461 serde_json::from_str(body).unwrap_or_else(|error| {
25462 panic!("invalid JSON request body for {path}: {error}: {body:?}")
25463 })
25464 }
25465
25466 fn request_bodies(&self, path: &str) -> Vec<Value> {
25467 self.requests
25468 .lock()
25469 .expect("captured requests")
25470 .iter()
25471 .filter(|request| request.path == path)
25472 .map(|request| {
25473 serde_json::from_str(&request.body).unwrap_or_else(|error| {
25474 panic!(
25475 "invalid JSON request body for {path}: {error}: {:?}",
25476 request.body
25477 )
25478 })
25479 })
25480 .collect()
25481 }
25482 }
25483
25484 impl Drop for MockWorkerServer {
25485 fn drop(&mut self) {
25486 self.stop.store(true, Ordering::SeqCst);
25487 let _ = TcpStream::connect(self.addr);
25488
25489 if let Some(thread) = self.thread.take() {
25490 thread.join().expect("join mock server");
25491 }
25492 }
25493 }
25494
25495 fn handle_mock_worker_request(
25496 stream: &mut TcpStream,
25497 requests: &Arc<Mutex<Vec<CapturedRequest>>>,
25498 behavior: MockWorkerBehavior,
25499 ) {
25500 let _ = stream.set_read_timeout(Some(Duration::from_millis(200)));
25501 let mut buffer = [0_u8; 8192];
25502 let mut request = Vec::new();
25503
25504 loop {
25505 match stream.read(&mut buffer) {
25506 Ok(0) => break,
25507 Ok(read) => {
25508 request.extend_from_slice(&buffer[..read]);
25509 if mock_request_is_complete(&request) {
25510 break;
25511 }
25512 }
25513 Err(error)
25514 if matches!(
25515 error.kind(),
25516 std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
25517 ) =>
25518 {
25519 break;
25520 }
25521 Err(_) => return,
25522 }
25523 }
25524
25525 let request = String::from_utf8_lossy(&request);
25526 let body = request
25527 .split_once("\r\n\r\n")
25528 .map(|(_, body)| body)
25529 .unwrap_or_default();
25530 let path = request
25531 .lines()
25532 .next()
25533 .and_then(|line| line.split_whitespace().nth(1))
25534 .unwrap_or_default();
25535 let method = request
25536 .lines()
25537 .next()
25538 .and_then(|line| line.split_whitespace().next())
25539 .unwrap_or_default();
25540 let authorization = request.lines().find_map(|line| {
25541 let (name, value) = line.split_once(':')?;
25542 name.eq_ignore_ascii_case("Authorization")
25543 .then(|| value.trim().to_string())
25544 });
25545 let namespace = request.lines().find_map(|line| {
25546 let (name, value) = line.split_once(':')?;
25547 name.eq_ignore_ascii_case("X-Namespace")
25548 .then(|| value.trim().to_string())
25549 });
25550 let worker_protocol = request.lines().find_map(|line| {
25551 let (name, value) = line.split_once(':')?;
25552 name.eq_ignore_ascii_case("X-Durable-Workflow-Protocol-Version")
25553 .then(|| value.trim().to_string())
25554 });
25555 let control_protocol = request.lines().find_map(|line| {
25556 let (name, value) = line.split_once(':')?;
25557 name.eq_ignore_ascii_case("X-Durable-Workflow-Control-Plane-Version")
25558 .then(|| value.trim().to_string())
25559 });
25560 let request_number = {
25561 let mut requests = requests.lock().expect("captured requests");
25562 requests.push(CapturedRequest {
25563 headers: request
25564 .split_once("\r\n\r\n")
25565 .map_or("", |(headers, _)| headers)
25566 .to_owned(),
25567 method: method.to_string(),
25568 path: path.to_string(),
25569 authorization,
25570 namespace,
25571 worker_protocol: worker_protocol.clone(),
25572 control_protocol,
25573 body: body.to_string(),
25574 received_at: Instant::now(),
25575 });
25576 requests
25577 .iter()
25578 .filter(|request| request.path == path)
25579 .count()
25580 };
25581
25582 if let Some(response) = behavior
25583 .request_override
25584 .and_then(|handler| handler(path, body, request_number))
25585 {
25586 write_mock_response(stream, response.0, &response.1);
25587 return;
25588 }
25589 if let Some(response) = behavior.response_override.and_then(|handler| handler(path)) {
25590 write_mock_response(stream, response.0, &response.1);
25591 return;
25592 }
25593 if path.ends_with("/poll") && request_number <= behavior.poll_failures_per_path {
25594 return;
25595 }
25596 let pressure_path = behavior
25597 .storage_path
25598 .is_some_and(|part| path.contains(part));
25599 let prior_failures = if path.ends_with("/poll") {
25600 behavior.poll_failures_per_path
25601 } else {
25602 0
25603 };
25604 if pressure_path
25605 && request_number.saturating_sub(prior_failures) <= behavior.storage_refusals
25606 {
25607 let request_body: Value = serde_json::from_str(body).unwrap_or(Value::Null);
25608 let poll_id = path
25609 .ends_with("/poll")
25610 .then(|| request_body["poll_request_id"].as_str().unwrap_or(""));
25611 let mut refusal = storage_refusal(
25612 poll_id,
25613 behavior.storage_unavailable,
25614 behavior.storage_mid_poll,
25615 );
25616 if behavior.storage_wrong_poll_id {
25617 refusal["poll_request_id"] = json!("wrong-poll");
25618 }
25619 write_mock_response(stream, "503 Service Unavailable", &refusal.to_string());
25620 return;
25621 }
25622 if path.contains("/storage-task/")
25623 || path.contains("/storage-activity/")
25624 || path.contains("/storage-query/")
25625 {
25626 write_mock_response(stream, "200 OK", "{}");
25627 return;
25628 }
25629 if behavior.storage_query && path == "/api/worker/query-tasks/poll" && request_number == 1 {
25630 write_mock_response(stream, "200 OK", &json!({"task":{
25631 "query_task_id":"storage-query", "query_task_attempt":7, "workflow_type":"storage.workflow",
25632 "query_name":"state", "workflow_id":"workflow", "run_id":"run", "payload_codec":"avro",
25633 "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC).unwrap(),
25634 "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC).unwrap(),
25635 "history_events":[], "run_status":"waiting", "lease_owner":"storage-worker"
25636 }}).to_string());
25637 return;
25638 }
25639 if behavior.storage_activity
25640 && path == "/api/worker/activity-tasks/poll"
25641 && request_number == 1
25642 {
25643 write_mock_response(stream, "200 OK", &json!({"task":{
25644 "task_id":"storage-activity", "activity_attempt_id":"storage-attempt", "activity_type":"storage.activity",
25645 "payload_codec":"avro", "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC).unwrap(),
25646 "attempt_number":7, "lease_owner":"storage-worker"
25647 }}).to_string());
25648 return;
25649 }
25650
25651 if path == "/api/worker/register" {
25652 if behavior.reject_registration_protocol {
25653 write_mock_response(
25654 stream,
25655 "400 Bad Request",
25656 r#"{"reason":"unsupported_protocol_version","message":"condition-wait occurrence identity requires worker protocol 1.17","supported_version":"1.16","requested_version":"1.17"}"#,
25657 );
25658 return;
25659 }
25660 if behavior.reject_registration {
25661 write_mock_response(
25662 stream,
25663 "503 Service Unavailable",
25664 r#"{"reason":"registration_unavailable","message":"registration failed"}"#,
25665 );
25666 return;
25667 }
25668 }
25669
25670 if path.starts_with("/api/worker/registrations/") {
25671 if behavior.reject_deregistration_protocol {
25672 write_mock_response(
25673 stream,
25674 "400 Bad Request",
25675 r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.17","requested_version":"1.19"}"#,
25676 );
25677 } else if behavior.reject_deregistration {
25678 write_mock_response(
25679 stream,
25680 "403 Forbidden",
25681 r#"{"reason":"authorization_failed","message":"worker cannot deregister"}"#,
25682 );
25683 } else {
25684 write_mock_response(
25685 stream,
25686 "200 OK",
25687 r#"{"worker_id":"deregistered-worker","outcome":"deregistered","recovered_workflow_task_count":2}"#,
25688 );
25689 }
25690 return;
25691 }
25692
25693 let is_poll = matches!(
25694 path,
25695 "/api/worker/workflow-tasks/poll"
25696 | "/api/worker/activity-tasks/poll"
25697 | "/api/worker/query-tasks/poll"
25698 );
25699 if is_poll && request_number <= behavior.long_poll_capacity_responses_per_path {
25700 write_mock_response(
25701 stream,
25702 "429 Too Many Requests",
25703 r#"{"task":null,"poll_status":"long_poll_capacity_exhausted","reason":"long_poll_capacity_exhausted","retryable":true,"retry_after_seconds":1}"#,
25704 );
25705 return;
25706 }
25707 if is_poll && request_number <= behavior.poll_failures_per_path {
25708 return;
25709 }
25710 if path == "/api/worker/heartbeat" && request_number <= behavior.heartbeat_failures {
25711 return;
25712 }
25713 if path == "/api/worker/heartbeat"
25714 && behavior.heartbeat_failure_request == Some(request_number)
25715 {
25716 return;
25717 }
25718 if path == "/api/worker/heartbeat"
25719 && behavior.delayed_heartbeat_request == Some(request_number)
25720 {
25721 thread::sleep(behavior.heartbeat_response_delay);
25722 }
25723 if behavior.unauthorized_polls && is_poll {
25724 write_mock_response(
25725 stream,
25726 "401 Unauthorized",
25727 r#"{"reason":"authentication_failed","message":"invalid worker token"}"#,
25728 );
25729 return;
25730 }
25731 if behavior.draining_polls && is_poll {
25732 write_mock_response(
25733 stream,
25734 "409 Conflict",
25735 r#"{"task":null,"poll_status":"draining","reason":"worker_draining","worker_status":"draining","drain_intent":"draining"}"#,
25736 );
25737 return;
25738 }
25739
25740 if let Some(codec_case) = behavior.invalid_task_payload_codec {
25741 if is_poll && request_number == 1 {
25742 let mut task = match path {
25743 "/api/worker/workflow-tasks/poll" => json!({
25744 "task_id": "codec-workflow",
25745 "workflow_type": "codec.workflow",
25746 "payload_codec": DEFAULT_CODEC,
25747 "workflow_task_attempt": 1,
25748 "lease_owner": "codec-worker"
25749 }),
25750 "/api/worker/activity-tasks/poll" => json!({
25751 "task_id": "codec-activity",
25752 "activity_attempt_id": "codec-activity-attempt",
25753 "activity_type": "codec.activity",
25754 "payload_codec": DEFAULT_CODEC,
25755 "attempt_number": 1,
25756 "lease_owner": "codec-worker"
25757 }),
25758 "/api/worker/query-tasks/poll" => json!({
25759 "query_task_id": "codec-query",
25760 "query_task_attempt": 1,
25761 "workflow_type": "codec.workflow",
25762 "query_name": "known",
25763 "payload_codec": DEFAULT_CODEC,
25764 "lease_owner": "codec-worker"
25765 }),
25766 _ => unreachable!("is_poll limits task codec probe paths"),
25767 };
25768 codec_case.apply(&mut task);
25769 write_mock_response(stream, "200 OK", &json!({"task": task}).to_string());
25770 return;
25771 }
25772
25773 if matches!(
25774 path,
25775 "/api/worker/workflow-tasks/codec-workflow/fail"
25776 | "/api/worker/activity-tasks/codec-activity/fail"
25777 | "/api/worker/query-tasks/codec-query/fail"
25778 ) {
25779 write_mock_response(stream, "200 OK", r#"{"outcome":"failed"}"#);
25780 return;
25781 }
25782 }
25783
25784 if behavior.reject_query_protocol && path.starts_with("/api/worker/query-tasks/") {
25785 let requested_version = worker_protocol.as_deref().unwrap_or("missing");
25786 let body = format!(
25787 r#"{{"reason":"unsupported_protocol_version","message":"Query tasks require worker protocol 1.8 or newer.","supported_version":"1.7","requested_version":"{requested_version}"}}"#
25788 );
25789 write_mock_response(stream, "400 Bad Request", &body);
25790 return;
25791 }
25792
25793 if behavior.reject_query_completion && path == "/api/worker/query-tasks/query-late/complete"
25794 {
25795 write_mock_response(
25796 stream,
25797 "409 Conflict",
25798 r#"{"reason":"query_task_timed_out","message":"query task timed out before completion"}"#,
25799 );
25800 return;
25801 }
25802
25803 if behavior.workflow_completion_status.is_some()
25804 && path == "/api/worker/workflow-tasks/poll"
25805 && request_number == 1
25806 {
25807 write_mock_response(
25808 stream,
25809 "200 OK",
25810 r#"{"task":{"task_id":"workflow-timeout-task","workflow_id":"reused-workflow-id","run_id":"run-selected-timeout","workflow_type":"timeout.workflow","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"workflow_task_attempt":3,"lease_owner":"timeout-worker"}}"#,
25811 );
25812 return;
25813 }
25814
25815 if path == "/api/worker/workflow-tasks/workflow-timeout-task/complete" {
25816 if let (Some(status), Some(body)) = (
25817 behavior.workflow_completion_status,
25818 behavior.workflow_completion_body,
25819 ) {
25820 write_mock_response(stream, status, body);
25821 return;
25822 }
25823 }
25824
25825 if behavior.waiting_query_worker {
25826 if behavior.complete_named_signal
25827 && path == "/api/worker/workflow-tasks/poll"
25828 && request_number == 1
25829 {
25830 let body = json!({
25831 "task": {
25832 "task_id": "snapshot-open",
25833 "workflow_id": "snapshot-1",
25834 "run_id": "snapshot-run-1",
25835 "workflow_type": "snapshot",
25836 "payload_codec": DEFAULT_CODEC,
25837 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25838 .expect("Avro workflow arguments"),
25839 "history_events": [],
25840 "workflow_task_attempt": 1,
25841 "lease_owner": "rust-snapshot-worker"
25842 }
25843 })
25844 .to_string();
25845 write_mock_response(stream, "200 OK", &body);
25846 return;
25847 }
25848
25849 let signal_request = request_number - usize::from(behavior.complete_named_signal);
25850 let signal_request_limit = 2 + usize::from(behavior.complete_named_signal);
25851 if path == "/api/worker/workflow-tasks/poll"
25852 && signal_request >= 1
25853 && signal_request <= signal_request_limit
25854 {
25855 let finish = behavior.complete_named_signal && signal_request == 3;
25856 let amounts = if signal_request == 1 {
25857 vec![3]
25858 } else {
25859 vec![3, 5]
25860 };
25861 let task_id = if signal_request == 1 {
25862 "snapshot-wait-3"
25863 } else if finish {
25864 "snapshot-finish"
25865 } else {
25866 "snapshot-wait-5"
25867 };
25868 let mut history_events = std::iter::once(json!({
25869 "event_type": "SignalWaitOpened",
25870 "payload": {"sequence": 1, "signal_name": "finish"}
25871 }))
25872 .chain(amounts.iter().enumerate().map(|(index, amount)| {
25873 json!({
25874 "event_type": "SignalReceived",
25875 "payload": {
25876 "signal_id": format!("increment-{amount}"),
25877 "signal_name": "increment",
25878 "workflow_sequence": index + 2,
25879 "payload_codec": DEFAULT_CODEC,
25880 "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
25881 .expect("Avro signal envelope")
25882 }
25883 })
25884 }))
25885 .collect::<Vec<_>>();
25886 let (resume_id, resume_name, resume_arguments) = if finish {
25887 history_events.push(json!({
25888 "event_type": "SignalReceived",
25889 "payload": {
25890 "signal_id": "finish",
25891 "signal_name": "finish",
25892 "workflow_sequence": 4,
25893 "payload_codec": DEFAULT_CODEC,
25894 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25895 .expect("Avro finish signal envelope")
25896 }
25897 }));
25898 (
25899 "finish".to_string(),
25900 "finish".to_string(),
25901 encode_value_envelope(&json!([]), DEFAULT_CODEC)
25902 .expect("Avro finish resume signal"),
25903 )
25904 } else {
25905 let amount = amounts.last().expect("amount");
25906 (
25907 format!("increment-{amount}"),
25908 "increment".to_string(),
25909 encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
25910 .expect("Avro increment resume signal"),
25911 )
25912 };
25913 let body = json!({
25914 "task": {
25915 "task_id": task_id,
25916 "workflow_id": "snapshot-1",
25917 "run_id": "snapshot-run-1",
25918 "workflow_type": "snapshot",
25919 "payload_codec": DEFAULT_CODEC,
25920 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25921 .expect("Avro workflow arguments"),
25922 "history_events": history_events,
25923 "workflow_task_attempt": 1,
25924 "workflow_signal_id": resume_id,
25925 "signal_name": resume_name,
25926 "signal_arguments": resume_arguments,
25927 "lease_owner": "rust-snapshot-worker"
25928 }
25929 })
25930 .to_string();
25931 write_mock_response(stream, "200 OK", &body);
25932 return;
25933 }
25934
25935 if path == "/api/worker/query-tasks/poll" && request_number == 1 {
25936 let history_events = [3, 5]
25937 .into_iter()
25938 .enumerate()
25939 .map(|(index, amount)| {
25940 json!({
25941 "event_type": "SignalReceived",
25942 "payload": {
25943 "signal_id": format!("increment-{amount}"),
25944 "signal_name": "increment",
25945 "workflow_sequence": index + 2,
25946 "payload_codec": DEFAULT_CODEC,
25947 "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
25948 .expect("Avro query signal envelope")
25949 }
25950 })
25951 })
25952 .collect::<Vec<_>>();
25953 let body = json!({
25954 "task": {
25955 "query_task_id": "snapshot-current",
25956 "query_task_attempt": 1,
25957 "lease_owner": "rust-snapshot-worker",
25958 "workflow_id": "snapshot-1",
25959 "run_id": "snapshot-run-1",
25960 "workflow_type": "snapshot",
25961 "query_name": "current",
25962 "payload_codec": DEFAULT_CODEC,
25963 "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25964 .expect("Avro workflow arguments"),
25965 "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25966 .expect("Avro query arguments"),
25967 "history_events": history_events,
25968 "run_status": "waiting"
25969 }
25970 })
25971 .to_string();
25972 write_mock_response(stream, "200 OK", &body);
25973 return;
25974 }
25975
25976 if path == "/api/worker/workflow-tasks/snapshot-wait-3/fail"
25977 || path == "/api/worker/workflow-tasks/snapshot-wait-5/fail"
25978 {
25979 write_mock_response(
25980 stream,
25981 "200 OK",
25982 r#"{"outcome":"waiting_for_history","recorded":true}"#,
25983 );
25984 return;
25985 }
25986
25987 if path == "/api/worker/workflow-tasks/snapshot-open/complete" {
25988 write_mock_response(stream, "200 OK", r#"{"outcome":"waiting","recorded":true}"#);
25989 return;
25990 }
25991
25992 if path == "/api/worker/workflow-tasks/snapshot-finish/complete" {
25993 write_mock_response(
25994 stream,
25995 "200 OK",
25996 r#"{"outcome":"completed","run_status":"completed","recorded":true}"#,
25997 );
25998 return;
25999 }
26000
26001 if path == "/api/worker/query-tasks/snapshot-current/complete" {
26002 write_mock_response(stream, "200 OK", r#"{"outcome":"completed"}"#);
26003 return;
26004 }
26005 }
26006
26007 if matches!(
26008 path,
26009 "/api/workflows/typed-1/query/inspect" | "/api/workflows/typed-1/update/replace"
26010 ) {
26011 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
26012 .expect("typed mock result");
26013 let body = json!({
26014 "result": typed_fidelity_probe().into_json().expect("result projection"),
26015 "result_envelope": result,
26016 })
26017 .to_string();
26018 write_mock_response(stream, "200 OK", &body);
26019 return;
26020 }
26021
26022 if path == "/api/workflows/typed-1" {
26023 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
26024 .expect("typed mock result");
26025 let body = json!({
26026 "workflow_id": "typed-1",
26027 "run_id": "run-typed-1",
26028 "workflow_type": "typed.echo",
26029 "status": "completed",
26030 "output": typed_fidelity_probe().into_json().expect("output projection"),
26031 "output_envelope": result,
26032 })
26033 .to_string();
26034 write_mock_response(stream, "200 OK", &body);
26035 return;
26036 }
26037
26038 let (status, body) = match path {
26039 "/api/cluster/info" => ("200 OK", r#"{"limits":{"max_payload_bytes":2097152}}"#),
26040 "/api/health" => ("200 OK", r#"{"status":"ok"}"#),
26041 "/api/workflows" => (
26042 "201 Created",
26043 r#"{"workflow_id":"wf-start-options","run_id":"run-start-options","workflow_type":"rust.timeout"}"#,
26044 ),
26045 "/api/worker/register" if behavior.decline_registration => (
26046 "200 OK",
26047 r#"{"worker_id":"declined-worker","registered":false}"#,
26048 ),
26049 "/api/worker/register" if behavior.waiting_query_worker => (
26050 "200 OK",
26051 r#"{"worker_id":"rust-snapshot-worker","registered":true,"heartbeat_interval_seconds":1}"#,
26052 ),
26053 "/api/worker/register" => (
26054 "200 OK",
26055 r#"{"worker_id":"mock-worker","registered":true,"heartbeat_interval_seconds":3600}"#,
26056 ),
26057 "/api/worker/heartbeat" => ("200 OK", "{}"),
26058 "/api/worker/activity-tasks/poll"
26059 if behavior.cancelled_activity && request_number == 1 =>
26060 {
26061 (
26062 "200 OK",
26063 r#"{"task":{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","activity_type":"cancel-aware","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"rust-cancel-worker"}}"#,
26064 )
26065 }
26066 "/api/worker/activity-tasks/poll"
26067 if behavior.long_poll_capacity_responses_per_path > 0
26068 && request_number
26069 == behavior
26070 .long_poll_capacity_responses_per_path
26071 .saturating_add(1) =>
26072 {
26073 (
26074 "200 OK",
26075 r#"{"task":{"task_id":"capacity-activity","activity_attempt_id":"capacity-attempt","activity_type":"capacity.activity","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"capacity-worker"}}"#,
26076 )
26077 }
26078 "/api/worker/activity-tasks/poll" | "/api/worker/workflow-tasks/poll" => {
26079 ("200 OK", r#"{"task":null}"#)
26080 }
26081 "/api/worker/query-tasks/poll"
26082 if behavior.reject_query_completion && request_number == 1 =>
26083 {
26084 (
26085 "200 OK",
26086 r#"{"task":{"query_task_id":"query-late","query_task_attempt":1,"lease_owner":"late-worker","workflow_id":"counter-late","run_id":"run-late","workflow_type":"counter","query_name":"current","payload_codec":"avro","workflow_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"query_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"run_status":"running"}}"#,
26087 )
26088 }
26089 "/api/worker/query-tasks/poll" => ("200 OK", r#"{"task":null}"#),
26090 "/api/worker/query-tasks/query-capture/complete"
26091 | "/api/worker/query-tasks/query-capture/fail" => ("200 OK", "{}"),
26092 "/api/worker/activity-tasks/activity-cancel/heartbeat" => (
26093 "200 OK",
26094 r#"{"activity_attempt_id":"attempt-cancel","cancel_requested":true,"can_continue":false,"reason":"run_cancelled","run_closed_reason":"cancelled","heartbeat_recorded":false}"#,
26095 ),
26096 "/api/worker/activity-tasks/activity-cancel/complete" => (
26097 "409 Conflict",
26098 r#"{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","reason":"run_cancelled","cancel_requested":true,"can_continue":false,"run_closed_reason":"cancelled"}"#,
26099 ),
26100 "/api/worker/activity-tasks/activity-typed/complete"
26101 | "/api/worker/activity-tasks/activity-typed/fail"
26102 | "/api/worker/activity-tasks/capacity-activity/complete"
26103 | "/api/workflows/typed-1/signal/changed" => ("200 OK", "{}"),
26104 "/api/workflows/counter-1/query/current" => (
26105 "200 OK",
26106 r#"{"workflow_id":"counter-1","query_name":"current","result":{"count":8},"result_envelope":{"codec":"avro","blob":"wwHioz3/VYAiNw4CCmNvdW50BBAA"}}"#,
26107 ),
26108 "/api/workflows/counter-1/query/missing" => (
26109 "404 Not Found",
26110 r#"{"workflow_id":"counter-1","query_name":"missing","reason":"rejected_unknown_query","message":"unknown query"}"#,
26111 ),
26112 "/api/workflows/wf-lifecycle/cancel" => (
26113 "200 OK",
26114 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","reason":"cleanup requested","command_status":"accepted"}"#,
26115 ),
26116 "/api/workflows/wf-lifecycle/terminate" => (
26117 "200 OK",
26118 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","reason":"forced stop","command_status":"accepted"}"#,
26119 ),
26120 "/api/workflows/wf-lifecycle/runs/run-current/cancel" => (
26121 "200 OK",
26122 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","command_status":"accepted"}"#,
26123 ),
26124 "/api/workflows/wf-lifecycle/runs/run-current/terminate" => (
26125 "200 OK",
26126 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","command_status":"accepted"}"#,
26127 ),
26128 "/api/workflows/wf-lifecycle/runs/run-failed/redrive" => (
26129 "202 Accepted",
26130 r#"{"workflow_id":"wf-lifecycle","continued_from_run_id":"run-failed","run_id":"run-successor","outcome":"redriven","command_status":"accepted","resume_step_sequence":2}"#,
26131 ),
26132 "/api/workflows/wf-lifecycle/runs/run-failed-existing/redrive" => (
26133 "200 OK",
26134 r#"{"workflow_id":"wf-lifecycle","continued_from_run_id":"run-failed-existing","run_id":"run-successor","outcome":"redriven","command_status":"accepted","resume_step_sequence":2}"#,
26135 ),
26136 "/api/workflows/wf-lifecycle/runs/run-completed/redrive" => (
26137 "409 Conflict",
26138 r#"{"workflow_id":"wf-lifecycle","run_id":"run-completed","reason":"run_not_failed","message":"Run is not failed."}"#,
26139 ),
26140 "/api/workflows/wf-lifecycle/runs/run-stale/cancel"
26141 | "/api/workflows/wf-lifecycle/runs/run-stale/terminate" => (
26142 "409 Conflict",
26143 r#"{"workflow_id":"wf-lifecycle","run_id":"run-stale","reason":"historical_run_command_rejected","target_scope":"run","message":"Commands cannot target historical runs."}"#,
26144 ),
26145 "/api/workflows/wf-failed" | "/api/workflows/wf-failed/runs/run-failed" => (
26146 "200 OK",
26147 r#"{"workflow_id":"wf-failed","run_id":"run-failed","status":"failed","closed_reason":"failed","error":"payment failed","failure":{"message":"payment failed","failure_category":"application","exception_type":"PaymentError","exception_class":"billing::PaymentError","non_retryable":true,"exception":{"type":"PaymentError","class":"billing::PaymentError","message":"payment failed"},"failures":[{"id":"failure-17","failure_category":"application"}]}}"#,
26148 ),
26149 "/api/workflows/wf-cancelled" => (
26150 "200 OK",
26151 r#"{"workflow_id":"wf-cancelled","run_id":"run-cancelled","status":"cancelled","closed_reason":"cancelled","reason":"cleanup requested"}"#,
26152 ),
26153 "/api/workflows/wf-terminated" => (
26154 "200 OK",
26155 r#"{"workflow_id":"wf-terminated","run_id":"run-terminated","status":"terminated","closed_reason":"terminated","reason":"forced stop"}"#,
26156 ),
26157 "/api/workflows/wf-timed-out" => (
26158 "200 OK",
26159 r#"{"workflow_id":"wf-timed-out","run_id":"run-timed-out","status":"failed","closed_reason":"timed_out","reason":"run_timeout"}"#,
26160 ),
26161 "/api/workflows/wf-waiting" | "/api/workflows/wf-waiting/runs/run-waiting" => (
26162 "200 OK",
26163 r#"{"workflow_id":"wf-waiting","run_id":"run-waiting","status":"waiting"}"#,
26164 ),
26165 "/api/workflows/wf-selected" => (
26166 "200 OK",
26167 r#"{"workflow_id":"wf-selected","run_id":"run-current","status":"completed","output":"current run output"}"#,
26168 ),
26169 "/api/workflows/wf-selected/runs/run-selected" => (
26170 "200 OK",
26171 r#"{"workflow_id":"wf-selected","run_id":"run-selected","status":"cancelled","closed_reason":"cancelled","reason":"selected run cancelled"}"#,
26172 ),
26173 _ => ("404 Not Found", r#"{"message":"not found"}"#),
26174 };
26175 write_mock_response(stream, status, body);
26176 }
26177
26178 fn mock_request_is_complete(request: &[u8]) -> bool {
26179 let Some(header_end) = request
26180 .windows(4)
26181 .position(|window| window == b"\r\n\r\n")
26182 .map(|position| position + 4)
26183 else {
26184 return false;
26185 };
26186 let headers = String::from_utf8_lossy(&request[..header_end]);
26187 let content_length = headers.lines().find_map(|line| {
26188 let (name, value) = line.split_once(':')?;
26189 name.eq_ignore_ascii_case("content-length")
26190 .then(|| value.trim().parse::<usize>().ok())
26191 .flatten()
26192 });
26193
26194 request.len() >= header_end + content_length.unwrap_or(0)
26195 }
26196
26197 fn write_mock_response(stream: &mut TcpStream, status: &str, body: &str) {
26198 let response = format!(
26199 "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
26200 body.len()
26201 );
26202
26203 let _ = stream.write_all(response.as_bytes());
26204 let _ = stream.flush();
26205 }
26206
26207 fn activity_completion_backend_recovery(
26208 path: &str,
26209 _body: &str,
26210 request_number: usize,
26211 ) -> Option<(&'static str, String)> {
26212 match path {
26213 "/api/worker/activity-tasks/poll" if request_number == 1 => Some((
26214 "200 OK",
26215 r#"{"task":{"task_id":"recover-activity","activity_attempt_id":"recover-attempt","activity_type":"recover.activity","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"recover-worker"}}"#.to_string(),
26216 )),
26217 "/api/worker/activity-tasks/recover-activity/complete" if request_number == 1 => Some((
26218 "503 Service Unavailable",
26219 r#"{"reason":"backend_unavailable","operation":"complete_activity_task","outcome":"unknown","worker_id":"recover-worker","task_queue":null,"retryable":true,"retry_after_seconds":1,"task_id":"recover-activity","lease_owner":"recover-worker","activity_attempt_id":"recover-attempt"}"#.to_string(),
26220 )),
26221 "/api/worker/activity-tasks/recover-activity/complete" => Some((
26222 "200 OK",
26223 r#"{"outcome":"completed","recorded":true}"#.to_string(),
26224 )),
26225 _ => None,
26226 }
26227 }
26228
26229 fn workflow_completion_backend_recovery(
26230 path: &str,
26231 _body: &str,
26232 request_number: usize,
26233 ) -> Option<(&'static str, String)> {
26234 match path {
26235 "/api/worker/workflow-tasks/poll" if request_number == 1 => Some((
26236 "200 OK",
26237 include_str!("../tests/fixtures/worker-completion-recovery.json").to_string(),
26238 )),
26239 "/api/worker/workflow-tasks/recover-workflow/complete" if request_number == 1 => Some((
26240 "503 Service Unavailable",
26241 r#"{"reason":"backend_unavailable","operation":"complete_workflow_task","outcome":"unknown","worker_id":"recover-worker","task_queue":null,"retryable":true,"retry_after_seconds":1,"task_id":"recover-workflow","lease_owner":"recover-worker","workflow_task_attempt":3}"#.to_string(),
26242 )),
26243 "/api/worker/workflow-tasks/recover-workflow/complete" => Some((
26244 "200 OK",
26245 r#"{"outcome":"completed","recorded":true}"#.to_string(),
26246 )),
26247 _ => None,
26248 }
26249 }
26250
26251 fn activity_completion_stale_after_backend_recovery(
26252 path: &str,
26253 body: &str,
26254 request_number: usize,
26255 ) -> Option<(&'static str, String)> {
26256 if path == "/api/worker/activity-tasks/recover-activity/complete" && request_number > 1 {
26257 return Some((
26258 "409 Conflict",
26259 r#"{"task_id":"recover-activity","activity_attempt_id":"recover-attempt","lease_owner":"recover-worker","reason":"stale_attempt","outcome":"completed","recorded":false,"activity_status":"completed","attempt_status":"completed","task_status":"completed"}"#.to_string(),
26260 ));
26261 }
26262 activity_completion_backend_recovery(path, body, request_number)
26263 }
26264
26265 fn activity_completion_remains_unavailable(
26266 path: &str,
26267 body: &str,
26268 request_number: usize,
26269 ) -> Option<(&'static str, String)> {
26270 if path == "/api/worker/activity-tasks/recover-activity/complete" {
26271 return activity_completion_backend_recovery(path, body, 1);
26272 }
26273 activity_completion_backend_recovery(path, body, request_number)
26274 }
26275
26276 #[tokio::test]
26277 async fn managed_activity_retries_same_fenced_completion_after_backend_loss() {
26278 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
26279 request_override: Some(activity_completion_backend_recovery),
26280 ..MockWorkerBehavior::default()
26281 });
26282 let client = Client::builder(server.base_url())
26283 .timeout(Duration::from_secs(2))
26284 .build()
26285 .expect("client");
26286 let handler_calls = Arc::new(AtomicUsize::new(0));
26287 let mut worker = Worker::new(client, "recover-queue")
26288 .worker_id("recover-worker")
26289 .poll_timeout(Duration::from_millis(10));
26290 worker.register_activity("recover.activity", {
26291 let handler_calls = Arc::clone(&handler_calls);
26292 move |_ctx, _args| {
26293 let handler_calls = Arc::clone(&handler_calls);
26294 async move {
26295 handler_calls.fetch_add(1, Ordering::SeqCst);
26296 Ok(json!("done"))
26297 }
26298 }
26299 });
26300
26301 assert_eq!(worker.run_once().await.expect("activity settled"), 1);
26302 assert_eq!(handler_calls.load(Ordering::SeqCst), 1);
26303 let bodies = server.request_bodies("/api/worker/activity-tasks/recover-activity/complete");
26304 assert_eq!(bodies.len(), 2);
26305 assert_eq!(bodies[0], bodies[1]);
26306 }
26307
26308 #[tokio::test]
26309 async fn managed_workflow_retries_same_fenced_completion_after_backend_loss() {
26310 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
26311 request_override: Some(workflow_completion_backend_recovery),
26312 ..MockWorkerBehavior::default()
26313 });
26314 let client = Client::builder(server.base_url())
26315 .timeout(Duration::from_secs(2))
26316 .build()
26317 .expect("client");
26318 let handler_calls = Arc::new(AtomicUsize::new(0));
26319 let mut worker = Worker::new(client, "recover-queue")
26320 .worker_id("recover-worker")
26321 .poll_timeout(Duration::from_millis(10));
26322 worker.register_workflow("recover.workflow", {
26323 let handler_calls = Arc::clone(&handler_calls);
26324 move |_ctx, _args| {
26325 let handler_calls = Arc::clone(&handler_calls);
26326 async move {
26327 handler_calls.fetch_add(1, Ordering::SeqCst);
26328 Ok(json!("done"))
26329 }
26330 }
26331 });
26332
26333 assert_eq!(worker.run_once().await.expect("workflow settled"), 1);
26334 assert_eq!(handler_calls.load(Ordering::SeqCst), 1);
26335 let bodies = server.request_bodies("/api/worker/workflow-tasks/recover-workflow/complete");
26336 assert_eq!(bodies.len(), 2);
26337 assert_eq!(bodies[0], bodies[1]);
26338 }
26339
26340 #[tokio::test]
26341 async fn managed_activity_accepts_authoritative_stale_outcome_after_unknown_completion() {
26342 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
26343 request_override: Some(activity_completion_stale_after_backend_recovery),
26344 ..MockWorkerBehavior::default()
26345 });
26346 let client = Client::builder(server.base_url())
26347 .timeout(Duration::from_secs(2))
26348 .build()
26349 .expect("client");
26350 let handler_calls = Arc::new(AtomicUsize::new(0));
26351 let mut worker = Worker::new(client, "recover-queue")
26352 .worker_id("recover-worker")
26353 .poll_timeout(Duration::from_millis(10));
26354 worker.register_activity("recover.activity", {
26355 let handler_calls = Arc::clone(&handler_calls);
26356 move |_ctx, _args| {
26357 let handler_calls = Arc::clone(&handler_calls);
26358 async move {
26359 handler_calls.fetch_add(1, Ordering::SeqCst);
26360 Ok(json!("done"))
26361 }
26362 }
26363 });
26364
26365 assert_eq!(
26366 worker
26367 .run_once()
26368 .await
26369 .expect("authoritative stale outcome"),
26370 1
26371 );
26372 assert_eq!(handler_calls.load(Ordering::SeqCst), 1);
26373 let bodies = server.request_bodies("/api/worker/activity-tasks/recover-activity/complete");
26374 assert_eq!(bodies.len(), 2);
26375 assert_eq!(bodies[0], bodies[1]);
26376 }
26377
26378 #[test]
26379 fn fenced_completion_retry_rejects_mismatched_or_incomplete_identity() {
26380 let request = json!({"lease_owner":"recover-worker","activity_attempt_id":"recover-attempt","result":{"codec":"avro","blob":"value"}});
26381 let response = json!({
26382 "reason":"backend_unavailable", "operation":"complete_activity_task",
26383 "outcome":"unknown", "worker_id":"recover-worker", "task_queue":null,
26384 "retryable":true, "retry_after_seconds":1, "task_id":"recover-activity",
26385 "lease_owner":"recover-worker", "activity_attempt_id":"recover-attempt"
26386 });
26387 let path = "/worker/activity-tasks/recover-activity/complete";
26388 let error = |body: Value| Error::Http {
26389 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
26390 body: body.to_string(),
26391 };
26392 assert_eq!(
26393 worker_fenced_completion_retry_after(&error(response.clone()), path, Some(&request)),
26394 Some(Duration::from_secs(1))
26395 );
26396
26397 for (field, invalid) in [
26398 ("operation", json!("poll_activity_task")),
26399 ("outcome", json!("rejected")),
26400 ("worker_id", json!("another-worker")),
26401 ("task_queue", json!("recover-queue")),
26402 ("retryable", json!(false)),
26403 ("retry_after_seconds", json!(0)),
26404 ("task_id", json!("another-task")),
26405 ("lease_owner", json!("another-worker")),
26406 ("activity_attempt_id", json!("another-attempt")),
26407 ] {
26408 let mut invalid_response = response.clone();
26409 invalid_response[field] = invalid;
26410 assert_eq!(
26411 worker_fenced_completion_retry_after(
26412 &error(invalid_response),
26413 path,
26414 Some(&request)
26415 ),
26416 None,
26417 "must not retry mismatched {field}"
26418 );
26419 }
26420 assert_eq!(
26421 worker_fenced_completion_retry_after(&error(response), path, None),
26422 None
26423 );
26424 }
26425
26426 #[tokio::test]
26427 async fn shutdown_interrupts_fenced_completion_backend_wait() {
26428 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
26429 request_override: Some(activity_completion_remains_unavailable),
26430 ..MockWorkerBehavior::default()
26431 });
26432 let client = Client::builder(server.base_url())
26433 .timeout(Duration::from_secs(2))
26434 .build()
26435 .expect("client");
26436 let stop = Arc::new(AtomicBool::new(false));
26437 let worker = Worker::new(client, "recover-queue")
26438 .worker_id("recover-worker")
26439 .with_storage_admission(Arc::clone(&stop));
26440 let completion = tokio::spawn(async move {
26441 worker
26442 .client
26443 .complete_activity_task(
26444 "recover-activity",
26445 "recover-attempt",
26446 "recover-worker",
26447 json!("done"),
26448 DEFAULT_CODEC,
26449 )
26450 .await
26451 });
26452 let path = "/api/worker/activity-tasks/recover-activity/complete";
26453 tokio::time::timeout(Duration::from_secs(2), async {
26454 while server.request_count(path) == 0 {
26455 tokio::time::sleep(Duration::from_millis(10)).await;
26456 }
26457 })
26458 .await
26459 .expect("first completion request");
26460 stop.store(true, Ordering::SeqCst);
26461 let error = tokio::time::timeout(Duration::from_secs(1), completion)
26462 .await
26463 .expect("shutdown must interrupt retry wait")
26464 .expect("completion task")
26465 .expect_err("backend is still unavailable");
26466 assert!(matches!(error, Error::ActivityTaskRejected(rejection) if rejection.status == 503));
26467 assert_eq!(server.request_count(path), 1);
26468 }
26469}