1#![doc = include_str!("../README.md")]
2
3mod runtime_payloads;
4mod runtime_uploads;
5
6use std::{
7 any::{type_name, Any, TypeId},
8 collections::{BTreeMap, HashMap},
9 future::Future,
10 io::{self, Read},
11 pin::Pin,
12 sync::{
13 atomic::{AtomicBool, Ordering},
14 Arc, Mutex, OnceLock,
15 },
16 task::{Context as TaskContext, Poll},
17 time::{Duration, Instant, SystemTime, UNIX_EPOCH},
18};
19
20use apache_avro::{from_avro_datum, to_avro_datum, types::Value as AvroDatum, Schema};
21use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
22use chrono::DateTime;
23use futures_util::{future::OptionFuture, task::noop_waker_ref};
24use serde::{
25 de::DeserializeOwned,
26 ser::{SerializeMap, SerializeSeq},
27 Deserialize, Deserializer, Serialize, Serializer,
28};
29pub use serde_json::{json, Value};
30use sha2::{Digest, Sha256};
31use thiserror::Error;
32pub use uuid::Uuid;
33
34pub const WORKER_PROTOCOL_VERSION: &str = "1.19";
35pub const PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION: &str = "1.18";
37pub const CONTROL_PLANE_VERSION: &str = "2";
38pub const DEFAULT_CODEC: &str = "avro";
39pub const SDK_VERSION: &str = concat!("durable-workflow-rust/", env!("CARGO_PKG_VERSION"));
40pub const CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY: &str =
42 "condition_wait_occurrence_identity";
43pub const MEMO_UPSERTS_CAPABILITY: &str = "memo_upserts";
45pub const QUERY_TASKS_CAPABILITY: &str = "query_tasks";
47pub const TYPED_SEARCH_ATTRIBUTES_CAPABILITY: &str = "typed_search_attributes";
49pub const WORKFLOW_UPDATES_CAPABILITY: &str = "workflow_updates";
51pub const MESSAGE_STREAMS_CAPABILITY: &str = "message_streams";
53pub const DURABLE_SELECTION_CAPABILITY: &str = "durable_selection";
55pub const MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.15";
56pub const MESSAGE_STREAM_SIGNAL: &str = "__durable_workflow_message_stream";
57pub const MESSAGE_STREAM_SCHEMA: &str = "durable-workflow.v2.message-stream.message";
58pub const MESSAGE_STREAM_CURSOR_SCHEMA: &str = "durable-workflow.v2.message-stream.cursor";
59pub const MESSAGE_STREAM_MAX_BATCH: usize = 100;
60pub const QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
62pub const SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
64pub const MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.14";
66pub const TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.16";
68pub const CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.9";
70pub const CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.17";
72pub const DURABLE_SELECTION_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.19";
74
75pub fn worker_protocol_supports_message_streams(version: &str) -> bool {
76 let Some((major, minor)) = version.split_once('.') else {
77 return false;
78 };
79 major == "1" && minor.parse::<u64>().is_ok_and(|minor| minor >= 15)
80}
81
82fn validate_user_signal_name(signal_name: &str) -> Result<()> {
83 if signal_name == MESSAGE_STREAM_SIGNAL {
84 return Err(Error::Codec(format!(
85 "signal name {MESSAGE_STREAM_SIGNAL:?} is reserved by the workflow runtime"
86 )));
87 }
88 Ok(())
89}
90
91const MAX_LONG_POLL_TIMEOUT_SECONDS: u64 = 60;
92const WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE: &str =
93 "Workflow task waiting for scheduled history.";
94const WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE: &str = "WorkflowTaskWaitingForHistory";
95const MISSING_TASK_PAYLOAD_CODEC: &str = "\0missing-task-payload-codec";
96const NULL_TASK_PAYLOAD_CODEC: &str = "\0null-task-payload-codec";
97const NON_STRING_TASK_PAYLOAD_CODEC: &str = "\0non-string-task-payload-codec";
98const MAX_MEMO_ENTRIES: usize = 100;
99const MAX_MEMO_VALUE_SIZE_BYTES: usize = 10_240;
100const MAX_MEMO_TOTAL_SIZE_BYTES: usize = 65_536;
101
102const QUERY_TASK_FINAL_REJECTION_REASONS: &[&str] = &[
103 "lease_expired",
104 "query_task_not_found",
105 "query_task_not_leased",
106 "query_task_timed_out",
107];
108
109pub fn portable_worker_affinity_capability_manifest() -> Value {
111 json!({
112 "local_activities": {
113 "supported": false,
114 "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
115 "reason": "rust_worker_does_not_execute_record_local_activity",
116 },
117 "worker_sessions": {
118 "supported": false,
119 "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
120 "reason": "rust_worker_has_no_typed_session_lifecycle",
121 },
122 "sticky_execution": {
123 "supported": false,
124 "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
125 "reason": "rust_worker_uses_complete_durable_history_replay",
126 },
127 })
128}
129
130pub const AVRO_VALUE_SCHEMA_JSON: &str =
132 include_str!("../schema/durable_workflow.protocol.Value.v1.avsc");
133pub const AVRO_VALUE_SCHEMA_FINGERPRINT_HEX: &str = "e2a33dff55802237";
134pub const AVRO_VALUE_SCHEMA_FINGERPRINT: [u8; 8] = [0xe2, 0xa3, 0x3d, 0xff, 0x55, 0x80, 0x22, 0x37];
135const AVRO_SINGLE_OBJECT_MAGIC: [u8; 2] = [0xc3, 0x01];
136
137static AVRO_VALUE_SCHEMA: OnceLock<std::result::Result<Schema, String>> = OnceLock::new();
138static AVRO_VALUE_ORDERED_MAP_ENCODING_SCHEMA: OnceLock<std::result::Result<Schema, String>> =
139 OnceLock::new();
140
141#[derive(Clone, Copy)]
142enum RequestProtocol {
143 ControlPlane,
144 Worker(&'static str),
145}
146
147pub type Result<T> = std::result::Result<T, Error>;
148
149#[derive(Debug, Error)]
150pub enum Error {
151 #[error("transport error: {0}")]
152 Transport(#[from] reqwest::Error),
153 #[error(
154 "invalid Durable Workflow base URL: omit the SDK-owned /api suffix and pass the Server or Cloud runtime base URL; the SDK appends /api automatically"
155 )]
156 InvalidBaseUrl,
157 #[error("json error: {0}")]
158 Json(#[from] serde_json::Error),
159 #[error("http {status}: {body}")]
160 Http {
161 status: reqwest::StatusCode,
162 body: String,
163 },
164 #[error("codec error: {0}")]
165 Codec(String),
166 #[error(transparent)]
167 QueryFailed(QueryFailure),
168 #[error(transparent)]
169 Protocol(ProtocolFailure),
170 #[error(transparent)]
171 NonDeterministicReplay(ReplayFailure),
172 #[error(transparent)]
173 ChildWorkflowFailed(ChildWorkflowFailure),
174 #[error(transparent)]
175 ActivityFailed(ActivityFailure),
176 #[error(transparent)]
177 ParallelFailed(ParallelFailure),
178 #[error(transparent)]
179 SagaCompensationFailed(SagaCompensationFailure),
180 #[error(transparent)]
181 InvalidParallelGroup(ParallelGroupError),
182 #[error(transparent)]
183 DurableOperationCancelled(DurableOperationCancelled),
184 #[error(transparent)]
185 WorkflowCancellationRequested(WorkflowCancellationRequested),
186 #[error(transparent)]
187 WorkflowCommandRejected(WorkflowCommandRejection),
188 #[error(transparent)]
189 WorkflowFailed(WorkflowTerminalOutcome),
190 #[error(transparent)]
191 WorkflowCancelled(WorkflowTerminalOutcome),
192 #[error(transparent)]
193 WorkflowTerminated(WorkflowTerminalOutcome),
194 #[error(transparent)]
195 WorkflowTimedOut(WorkflowTerminalOutcome),
196 #[error(transparent)]
197 ActivityTaskRejected(ActivityTaskRejection),
198 #[error("workflow handler {0:?} is not registered")]
199 WorkflowNotRegistered(String),
200 #[error("activity handler {0:?} is not registered")]
201 ActivityNotRegistered(String),
202 #[error(
203 "{handler_kind} handler {handler_name:?} {value_kind} type {rust_type} is incompatible with the fixed Avro Value codec: {message}"
204 )]
205 HandlerType {
206 handler_kind: HandlerKind,
207 handler_name: String,
208 value_kind: HandlerValueKind,
209 rust_type: &'static str,
210 message: String,
211 },
212 #[error("workflow future yielded without emitting a durable command")]
213 WorkflowYieldedWithoutCommand,
214 #[error(
215 "workflow_stream_command_identity_missing: workflow stream authoring requires a non-empty server-provided workflow_command_id"
216 )]
217 MissingWorkflowCommandIdentity,
218 #[error("workflow state lock is poisoned")]
219 WorkflowStatePoisoned,
220 #[error("timer duration is too large for the worker protocol")]
221 TimerDurationOverflow,
222 #[error(transparent)]
223 InvalidConditionWaitOptions(#[from] ConditionWaitOptionsError),
224 #[error(transparent)]
225 InvalidSearchAttributeUpdate(#[from] SearchAttributeUpdateError),
226 #[error("operation timed out")]
227 Timeout,
228 #[error(
229 "missing {role}-plane credentials: configure ClientBuilder::{role}_token or ClientBuilder::token; a {opposite_role}-plane token cannot authorize this request"
230 )]
231 MissingRoleCredentials {
232 role: &'static str,
233 opposite_role: &'static str,
234 },
235 #[error("worker loop error: {0}")]
236 WorkerLoop(String),
237 #[error(
238 "workflow command contract for {workflow_type:?} declares update validators, but this Rust SDK cannot execute synchronous pre-accept update validation"
239 )]
240 UnsupportedUpdateValidators { workflow_type: String },
241 #[error("{primary}; worker deregistration also failed: {deregistration}")]
242 WorkerShutdown {
243 primary: Box<Error>,
244 deregistration: Box<Error>,
245 },
246 #[error("invalid child workflow options: {0}")]
247 InvalidChildWorkflowOptions(String),
248 #[error("invalid workflow memo update: {0}")]
249 InvalidMemoUpdate(String),
250 #[error(
251 "workflow_memo_updates_unavailable: the connected runtime did not advertise workflow memo update support"
252 )]
253 WorkflowMemoUpdatesUnavailable,
254 #[error(transparent)]
255 InvalidActivityOptions(ActivityOptionsError),
256 #[error(transparent)]
257 InvalidContinueAsNewOptions(#[from] ContinueAsNewOptionsError),
258 #[doc(hidden)]
259 #[error("workflow requested continue as new")]
260 ContinueAsNew(ContinueAsNewRequest),
261}
262
263#[derive(Clone, Debug, Error, PartialEq, Eq)]
265pub enum ConditionWaitOptionsError {
266 #[error("condition_key must be non-empty")]
267 EmptyKey,
268 #[error("condition_definition_fingerprint must be non-empty")]
269 EmptyPredicateIdentity,
270 #[error("condition timeout is too large for the worker protocol")]
271 TimeoutOverflow,
272}
273
274#[derive(Clone, Debug, PartialEq, Eq)]
281pub struct ConditionWaitOptions {
282 condition_key: String,
283 predicate_identity: String,
284 timeout: Option<Duration>,
285}
286
287impl ConditionWaitOptions {
288 pub fn new(condition_key: impl Into<String>, predicate_identity: impl Into<String>) -> Self {
289 Self {
290 condition_key: condition_key.into(),
291 predicate_identity: predicate_identity.into(),
292 timeout: None,
293 }
294 }
295
296 pub fn timeout(mut self, timeout: Duration) -> Self {
297 self.timeout = Some(timeout);
298 self
299 }
300
301 fn validate(
302 &self,
303 ) -> std::result::Result<ValidatedConditionWaitOptions, ConditionWaitOptionsError> {
304 let condition_key = self.condition_key.trim();
305 if condition_key.is_empty() {
306 return Err(ConditionWaitOptionsError::EmptyKey);
307 }
308 let predicate_identity = self.predicate_identity.trim();
309 if predicate_identity.is_empty() {
310 return Err(ConditionWaitOptionsError::EmptyPredicateIdentity);
311 }
312 let timeout_seconds = self
313 .timeout
314 .map(|timeout| {
315 timeout
316 .as_secs()
317 .checked_add(u64::from(timeout.subsec_nanos() > 0))
318 .ok_or(ConditionWaitOptionsError::TimeoutOverflow)
319 })
320 .transpose()?;
321
322 Ok(ValidatedConditionWaitOptions {
323 condition_key: condition_key.to_string(),
324 predicate_identity: predicate_identity.to_string(),
325 timeout_seconds,
326 })
327 }
328}
329
330#[derive(Clone, Debug, PartialEq, Eq)]
331struct ValidatedConditionWaitOptions {
332 condition_key: String,
333 predicate_identity: String,
334 timeout_seconds: Option<u64>,
335}
336
337const CONDITION_WAIT_OCCURRENCE_PREFIX: &str = "rust:condition-wait:";
338
339#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
341#[serde(rename_all = "snake_case")]
342pub enum ConditionWaitResult {
343 Satisfied,
344 TimedOut,
345}
346
347impl ConditionWaitResult {
348 pub fn is_satisfied(self) -> bool {
349 self == Self::Satisfied
350 }
351
352 pub fn is_timed_out(self) -> bool {
353 self == Self::TimedOut
354 }
355}
356
357#[doc(hidden)]
359pub fn __condition_definition_fingerprint(source: &str) -> String {
360 let mut digest = Sha256::new();
361 digest.update(b"durable-workflow-rust.wait-condition.v1\0");
362 digest.update(source.as_bytes());
363 format!("sha256:{:x}", digest.finalize())
364}
365
366#[macro_export]
372macro_rules! wait_condition {
373 ($ctx:expr, $key:expr, timeout: $timeout:expr, $predicate:expr $(,)?) => {{
374 $ctx.wait_condition(
375 $crate::ConditionWaitOptions::new(
376 $key,
377 $crate::__condition_definition_fingerprint(concat!(
378 module_path!(),
379 "\0",
380 stringify!($predicate)
381 )),
382 )
383 .timeout($timeout),
384 $predicate,
385 )
386 }};
387 ($ctx:expr, $key:expr, $predicate:expr $(,)?) => {{
388 $ctx.wait_condition(
389 $crate::ConditionWaitOptions::new(
390 $key,
391 $crate::__condition_definition_fingerprint(concat!(
392 module_path!(),
393 "\0",
394 stringify!($predicate)
395 )),
396 ),
397 $predicate,
398 )
399 }};
400}
401
402const MAX_SEARCH_ATTRIBUTES_PER_UPDATE: usize = 100;
403const MAX_SEARCH_ATTRIBUTE_KEY_LENGTH: usize = 64;
404const MAX_SEARCH_ATTRIBUTE_STRING_LENGTH: usize = 2_048;
405const MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH: usize = 255;
406const MAX_SEARCH_ATTRIBUTE_UPDATE_BYTES: usize = 65_536;
407
408#[derive(Clone, Debug, Error, PartialEq, Eq)]
410pub enum SearchAttributeUpdateError {
411 #[error("search-attribute update requires at least one attribute")]
412 Empty,
413 #[error("search attribute key {0:?} must be 1-64 URL-safe ASCII characters")]
414 InvalidKey(String),
415 #[error("search-attribute update exceeds the limit of 100 attributes")]
416 TooManyAttributes,
417 #[error("search attribute {key:?} {kind} value exceeds {limit} bytes")]
418 ValueTooLong {
419 key: String,
420 kind: &'static str,
421 limit: usize,
422 },
423 #[error(
424 "search attribute {0:?} must not contain an empty string value; use delete() to remove it"
425 )]
426 EmptyString(String),
427 #[error("search attribute {0:?} has a non-finite float value")]
428 NonFiniteFloat(String),
429 #[error("search attribute {0:?} must use an RFC 3339 datetime with an explicit timezone")]
430 InvalidDateTime(String),
431 #[error("search-attribute update exceeds the 65536-byte protocol limit")]
432 PayloadTooLarge,
433}
434
435#[derive(Clone, Debug, PartialEq)]
437pub enum SearchAttributeValue {
438 String(String),
439 Keyword(String),
440 KeywordList(Vec<String>),
441 Int(i64),
442 Float(f64),
443 Bool(bool),
444 DateTime(String),
445 Delete,
446}
447
448impl SearchAttributeValue {
449 fn type_name(&self) -> Option<&'static str> {
450 match self {
451 Self::String(_) => Some("string"),
452 Self::Keyword(_) => Some("keyword"),
453 Self::KeywordList(_) => Some("keyword_list"),
454 Self::Int(_) => Some("int"),
455 Self::Float(_) => Some("float"),
456 Self::Bool(_) => Some("bool"),
457 Self::DateTime(_) => Some("datetime"),
458 Self::Delete => None,
459 }
460 }
461
462 fn normalized(self, key: &str) -> std::result::Result<Self, SearchAttributeUpdateError> {
463 let normalize_string = |value: String, kind: &'static str, limit: usize| {
464 let value = value.trim().to_string();
465 if value.is_empty() {
466 return Err(SearchAttributeUpdateError::EmptyString(key.to_string()));
467 }
468 if value.len() > limit {
469 return Err(SearchAttributeUpdateError::ValueTooLong {
470 key: key.to_string(),
471 kind,
472 limit,
473 });
474 }
475 Ok(value)
476 };
477
478 match self {
479 Self::String(value) => Ok(Self::String(normalize_string(
480 value,
481 "string",
482 MAX_SEARCH_ATTRIBUTE_STRING_LENGTH,
483 )?)),
484 Self::Keyword(value) => Ok(Self::Keyword(normalize_string(
485 value,
486 "keyword",
487 MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH,
488 )?)),
489 Self::KeywordList(values) => {
490 let values = values
491 .into_iter()
492 .map(|value| {
493 let value = value.trim().to_string();
494 if value.len() > MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH {
495 return Err(SearchAttributeUpdateError::ValueTooLong {
496 key: key.to_string(),
497 kind: "keyword-list entry",
498 limit: MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH,
499 });
500 }
501 Ok(value)
502 })
503 .collect::<std::result::Result<Vec<_>, _>>()?;
504 Ok(Self::KeywordList(values))
505 }
506 Self::Float(value) if !value.is_finite() => {
507 Err(SearchAttributeUpdateError::NonFiniteFloat(key.to_string()))
508 }
509 Self::DateTime(value) => {
510 let value =
511 normalize_string(value, "datetime", MAX_SEARCH_ATTRIBUTE_STRING_LENGTH)?;
512 if DateTime::parse_from_rfc3339(&value).is_err() {
513 return Err(SearchAttributeUpdateError::InvalidDateTime(key.to_string()));
514 }
515 Ok(Self::DateTime(value))
516 }
517 value => Ok(value),
518 }
519 }
520
521 fn into_json(self) -> Value {
522 match self {
523 Self::String(value) | Self::Keyword(value) | Self::DateTime(value) => {
524 Value::String(value)
525 }
526 Self::KeywordList(values) => {
527 Value::Array(values.into_iter().map(Value::String).collect())
528 }
529 Self::Int(value) => json!(value),
530 Self::Float(value) => json!(value),
531 Self::Bool(value) => json!(value),
532 Self::Delete => Value::Null,
533 }
534 }
535}
536
537#[derive(Clone, Debug, Default, PartialEq)]
539pub struct SearchAttributeUpdate {
540 attributes: BTreeMap<String, SearchAttributeValue>,
541}
542
543impl SearchAttributeUpdate {
544 pub fn new() -> Self {
545 Self::default()
546 }
547
548 pub fn set(
549 mut self,
550 key: impl Into<String>,
551 value: SearchAttributeValue,
552 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
553 let key = key.into();
554 validate_search_attribute_key(&key)?;
555 if !self.attributes.contains_key(&key)
556 && self.attributes.len() >= MAX_SEARCH_ATTRIBUTES_PER_UPDATE
557 {
558 return Err(SearchAttributeUpdateError::TooManyAttributes);
559 }
560 self.attributes.insert(key.clone(), value.normalized(&key)?);
561 self.validate_size()?;
562 Ok(self)
563 }
564
565 pub fn string(
566 self,
567 key: impl Into<String>,
568 value: impl Into<String>,
569 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
570 self.set(key, SearchAttributeValue::String(value.into()))
571 }
572
573 pub fn keyword(
574 self,
575 key: impl Into<String>,
576 value: impl Into<String>,
577 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
578 self.set(key, SearchAttributeValue::Keyword(value.into()))
579 }
580
581 pub fn keyword_list<I, V>(
582 self,
583 key: impl Into<String>,
584 values: I,
585 ) -> std::result::Result<Self, SearchAttributeUpdateError>
586 where
587 I: IntoIterator<Item = V>,
588 V: Into<String>,
589 {
590 self.set(
591 key,
592 SearchAttributeValue::KeywordList(values.into_iter().map(Into::into).collect()),
593 )
594 }
595
596 pub fn int(
597 self,
598 key: impl Into<String>,
599 value: i64,
600 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
601 self.set(key, SearchAttributeValue::Int(value))
602 }
603
604 pub fn float(
605 self,
606 key: impl Into<String>,
607 value: f64,
608 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
609 self.set(key, SearchAttributeValue::Float(value))
610 }
611
612 pub fn bool(
613 self,
614 key: impl Into<String>,
615 value: bool,
616 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
617 self.set(key, SearchAttributeValue::Bool(value))
618 }
619
620 pub fn datetime(
621 self,
622 key: impl Into<String>,
623 value: impl Into<String>,
624 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
625 self.set(key, SearchAttributeValue::DateTime(value.into()))
626 }
627
628 pub fn delete(
629 self,
630 key: impl Into<String>,
631 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
632 self.set(key, SearchAttributeValue::Delete)
633 }
634
635 fn validate_size(&self) -> std::result::Result<(), SearchAttributeUpdateError> {
636 let (attributes, _) = self.clone().into_wire_parts();
637 if serde_json::to_vec(&attributes)
638 .map(|payload| payload.len() > MAX_SEARCH_ATTRIBUTE_UPDATE_BYTES)
639 .unwrap_or(true)
640 {
641 return Err(SearchAttributeUpdateError::PayloadTooLarge);
642 }
643 Ok(())
644 }
645
646 fn into_wire_parts(self) -> (Value, BTreeMap<String, String>) {
647 let mut attributes = serde_json::Map::new();
648 let mut attribute_types = BTreeMap::new();
649 for (key, value) in self.attributes {
650 if let Some(type_name) = value.type_name() {
651 attribute_types.insert(key.clone(), type_name.to_string());
652 }
653 attributes.insert(key, value.into_json());
654 }
655 (Value::Object(attributes), attribute_types)
656 }
657
658 fn validate(&self) -> std::result::Result<(), SearchAttributeUpdateError> {
659 if self.attributes.is_empty() {
660 return Err(SearchAttributeUpdateError::Empty);
661 }
662 self.validate_size()
663 }
664}
665
666fn validate_search_attribute_key(key: &str) -> std::result::Result<(), SearchAttributeUpdateError> {
667 let valid = !key.is_empty()
668 && key.len() <= MAX_SEARCH_ATTRIBUTE_KEY_LENGTH
669 && key
670 .bytes()
671 .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-' | b':'));
672 if valid {
673 Ok(())
674 } else {
675 Err(SearchAttributeUpdateError::InvalidKey(key.to_string()))
676 }
677}
678
679#[derive(Clone, Copy, Debug, PartialEq, Eq)]
681pub enum HandlerKind {
682 Workflow,
683 Activity,
684}
685
686impl std::fmt::Display for HandlerKind {
687 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
688 formatter.write_str(match self {
689 Self::Workflow => "workflow",
690 Self::Activity => "activity",
691 })
692 }
693}
694
695#[derive(Clone, Copy, Debug, PartialEq, Eq)]
697pub enum HandlerValueKind {
698 Input,
699 Result,
700}
701
702impl std::fmt::Display for HandlerValueKind {
703 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
704 formatter.write_str(match self {
705 Self::Input => "input",
706 Self::Result => "result",
707 })
708 }
709}
710
711#[derive(Clone, Copy, Debug, PartialEq, Eq)]
713pub enum WorkflowCommandKind {
714 Cancel,
715 Terminate,
716}
717
718impl WorkflowCommandKind {
719 fn as_str(self) -> &'static str {
720 match self {
721 Self::Cancel => "cancel",
722 Self::Terminate => "terminate",
723 }
724 }
725}
726
727#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
729pub struct WorkflowCommandOptions {
730 #[serde(skip_serializing_if = "Option::is_none")]
731 pub reason: Option<String>,
732 #[serde(skip_serializing_if = "Option::is_none")]
733 pub request_id: Option<String>,
734}
735
736#[derive(Clone, Debug, PartialEq, Eq)]
743pub struct WorkflowStartOptions {
744 pub execution_timeout_seconds: u64,
745 pub run_timeout_seconds: u64,
746}
747
748impl Default for WorkflowStartOptions {
749 fn default() -> Self {
750 Self {
751 execution_timeout_seconds: 3600,
752 run_timeout_seconds: 600,
753 }
754 }
755}
756
757impl WorkflowStartOptions {
758 pub fn new() -> Self {
759 Self::default()
760 }
761
762 pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
763 self.execution_timeout_seconds = seconds;
764 self
765 }
766
767 pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
768 self.run_timeout_seconds = seconds;
769 self
770 }
771
772 fn validate(&self) -> Result<()> {
773 if self.execution_timeout_seconds == 0 {
774 return Err(Error::Codec(
775 "execution_timeout_seconds must be at least 1".to_string(),
776 ));
777 }
778 if self.run_timeout_seconds == 0 {
779 return Err(Error::Codec(
780 "run_timeout_seconds must be at least 1".to_string(),
781 ));
782 }
783 if self.run_timeout_seconds > self.execution_timeout_seconds {
784 return Err(Error::Codec(
785 "run_timeout_seconds cannot exceed execution_timeout_seconds".to_string(),
786 ));
787 }
788
789 Ok(())
790 }
791}
792
793#[derive(Clone, Debug, Default, PartialEq, Eq)]
798pub struct ContinueAsNewOptions {
799 pub workflow_type: Option<String>,
800 pub task_queue: Option<String>,
801}
802
803impl ContinueAsNewOptions {
804 pub fn new() -> Self {
805 Self::default()
806 }
807
808 pub fn workflow_type(mut self, workflow_type: impl Into<String>) -> Self {
809 self.workflow_type = Some(workflow_type.into());
810 self
811 }
812
813 pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
814 self.task_queue = Some(task_queue.into());
815 self
816 }
817
818 fn validate(&self) -> std::result::Result<(), ContinueAsNewOptionsError> {
819 for (field, value) in [
820 ("workflow_type", self.workflow_type.as_deref()),
821 ("task_queue", self.task_queue.as_deref()),
822 ] {
823 if value.is_some_and(|value| value.trim().is_empty()) {
824 return Err(ContinueAsNewOptionsError {
825 field,
826 message: format!("{field} must not be empty"),
827 });
828 }
829 }
830 Ok(())
831 }
832}
833
834#[derive(Clone, Debug, Error, PartialEq, Eq)]
836#[error("invalid continue-as-new option {field}: {message}")]
837pub struct ContinueAsNewOptionsError {
838 pub field: &'static str,
839 pub message: String,
840}
841
842#[derive(Clone, Debug, Default, PartialEq, Eq)]
844pub struct WorkflowHistoryBudget {
845 pub event_count: u64,
846 pub size_bytes: Option<u64>,
847 pub continue_as_new_recommended: bool,
848 pub pressure: Option<String>,
849}
850
851#[doc(hidden)]
852#[derive(Clone, Debug)]
853pub struct ContinueAsNewRequest {
854 arguments: AvroValue,
855 options: ContinueAsNewOptions,
856}
857
858impl WorkflowCommandOptions {
859 pub fn new() -> Self {
860 Self::default()
861 }
862
863 pub fn reason(mut self, reason: impl Into<String>) -> Self {
864 self.reason = Some(reason.into());
865 self
866 }
867
868 pub fn request_id(mut self, request_id: impl Into<String>) -> Self {
869 self.request_id = Some(request_id.into());
870 self
871 }
872}
873
874#[derive(Clone, Debug, PartialEq)]
876pub struct WorkflowCommandResult {
877 pub command: WorkflowCommandKind,
878 pub workflow_id: String,
879 pub run_id: Option<String>,
880 pub outcome: Option<String>,
881 pub reason: Option<String>,
882 pub command_status: Option<String>,
883 pub raw: Value,
884}
885
886#[derive(Clone, Debug, Error)]
888#[error("workflow {command:?} rejected ({reason}, HTTP {status}): {message}")]
889pub struct WorkflowCommandRejection {
890 pub command: WorkflowCommandKind,
891 pub status: u16,
892 pub reason: String,
893 pub message: String,
894 pub workflow_id: String,
895 pub run_id: Option<String>,
896 pub target_scope: Option<String>,
897 pub body: Value,
898}
899
900#[derive(Clone, Copy, Debug, PartialEq, Eq)]
902pub enum WorkflowTerminalKind {
903 Failed,
904 Cancelled,
905 Terminated,
906 TimedOut,
907}
908
909#[derive(Clone, Debug, Error)]
915#[error("workflow {workflow_id} run {run_id:?} ended as {kind:?} ({reason})")]
916pub struct WorkflowTerminalOutcome {
917 pub kind: WorkflowTerminalKind,
918 pub workflow_id: String,
919 pub run_id: Option<String>,
920 pub reason: String,
921 pub failure_category: Option<String>,
922 pub failure_id: Option<String>,
923 pub exception_type: Option<String>,
924 pub exception_class: Option<String>,
925 pub non_retryable: Option<bool>,
926 pub message: Option<String>,
927 pub exception: Option<Value>,
928 pub raw: Value,
929}
930
931#[derive(Clone, Debug, Error)]
933#[error("activity task {operation} rejected ({reason}, HTTP {status})")]
934pub struct ActivityTaskRejection {
935 pub operation: String,
936 pub status: u16,
937 pub reason: String,
938 pub task_id: String,
939 pub activity_attempt_id: String,
940 pub cancel_requested: bool,
941 pub can_continue: Option<bool>,
942 pub run_closed_reason: Option<String>,
943 pub body: Value,
944}
945
946#[derive(Clone, Copy, Debug, PartialEq, Eq)]
948pub enum ActivityOptionsErrorKind {
949 EmptyTaskQueue,
950 EmptyRetryPolicy,
951 InvalidMaxAttempts,
952 BackoffWithoutRetryBudget,
953 TooManyBackoffIntervals,
954 InvalidBackoffCoefficient,
955 BackoffGenerationTooLarge,
956 BackoffOverflow,
957 EmptyNonRetryableErrorType,
958 TimeoutNotPositive,
959 TimeoutOverflow,
960 TimeoutOrder,
961}
962
963#[derive(Clone, Debug, Error, PartialEq, Eq)]
965#[error("invalid activity options ({kind:?}, {field:?}): {message}")]
966pub struct ActivityOptionsError {
967 pub kind: ActivityOptionsErrorKind,
968 pub field: Option<&'static str>,
969 pub message: String,
970}
971
972impl ActivityOptionsError {
973 fn new(
974 kind: ActivityOptionsErrorKind,
975 field: Option<&'static str>,
976 message: impl Into<String>,
977 ) -> Self {
978 Self {
979 kind,
980 field,
981 message: message.into(),
982 }
983 }
984}
985
986#[derive(Clone, Copy, Debug, PartialEq, Eq)]
988pub enum ActivityFailureKind {
989 Failed,
990 Cancelled,
991 TimedOut,
992}
993
994#[derive(Clone, Debug, Error)]
999#[error("activity failed ({reason}): {message}")]
1000pub struct ActivityFailure {
1001 pub kind: ActivityFailureKind,
1002 pub reason: String,
1003 pub message: String,
1004 pub activity_execution_id: Option<String>,
1005 pub activity_attempt_id: Option<String>,
1006 pub activity_type: Option<String>,
1007 pub activity_class: Option<String>,
1008 pub attempt_number: Option<u64>,
1009 pub failure_id: Option<String>,
1010 pub failure_category: Option<String>,
1011 pub timeout_kind: Option<String>,
1012 pub non_retryable: bool,
1013 pub exception_type: Option<String>,
1014 pub exception_class: Option<String>,
1015 pub code: Option<Value>,
1016 pub exception: Option<Value>,
1017}
1018
1019#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1021pub enum ChildWorkflowFailureKind {
1022 Failed,
1023 Cancelled,
1024 Terminated,
1025}
1026
1027#[derive(Clone, Debug, Error)]
1033#[error("child workflow failed ({reason}): {message}")]
1034pub struct ChildWorkflowFailure {
1035 pub kind: ChildWorkflowFailureKind,
1036 pub reason: String,
1037 pub message: String,
1038 pub parent_workflow_id: Option<String>,
1039 pub parent_workflow_run_id: Option<String>,
1040 pub child_workflow_id: Option<String>,
1041 pub child_workflow_run_id: Option<String>,
1042 pub child_workflow_type: Option<String>,
1043 pub failure_id: Option<String>,
1044 pub failure_category: Option<String>,
1045 pub exception_type: Option<String>,
1046 pub exception_class: Option<String>,
1047 pub non_retryable: bool,
1048 pub code: Option<Value>,
1049 pub exception: Option<Value>,
1050}
1051
1052#[derive(Clone, Debug, PartialEq, Eq)]
1054pub struct WorkflowIdentity {
1055 pub workflow_id: Option<String>,
1056 pub run_id: Option<String>,
1057}
1058
1059#[derive(Clone, Debug, PartialEq)]
1061pub struct ChildWorkflowResult {
1062 pub parent: WorkflowIdentity,
1063 pub child: WorkflowIdentity,
1064 pub child_workflow_type: Option<String>,
1065 pub result: Value,
1066}
1067
1068#[derive(Clone, Debug, PartialEq)]
1070pub struct ChildWorkflowAvroResult {
1071 pub parent: WorkflowIdentity,
1072 pub child: WorkflowIdentity,
1073 pub child_workflow_type: Option<String>,
1074 pub result: AvroValue,
1075}
1076
1077#[derive(Clone, Debug, Deserialize, Hash, PartialEq, Eq, Serialize)]
1079#[serde(untagged)]
1080pub enum SelectionKey {
1081 Index(usize),
1082 Name(String),
1083}
1084
1085impl From<usize> for SelectionKey {
1086 fn from(value: usize) -> Self {
1087 Self::Index(value)
1088 }
1089}
1090
1091impl From<String> for SelectionKey {
1092 fn from(value: String) -> Self {
1093 Self::Name(value)
1094 }
1095}
1096
1097impl From<&str> for SelectionKey {
1098 fn from(value: &str) -> Self {
1099 Self::Name(value.to_string())
1100 }
1101}
1102
1103#[derive(Clone, Debug, Error, PartialEq, Eq)]
1105#[error("selected {operation_kind} operation {operation_identity} was explicitly cancelled")]
1106pub struct DurableOperationCancelled {
1107 pub selection_group_id: String,
1108 pub member_key: SelectionKey,
1109 pub member_index: usize,
1110 pub operation_kind: String,
1111 pub operation_identity: String,
1112}
1113
1114#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
1120pub struct ParallelGroupMetadata {
1121 pub parallel_group_id: String,
1122 pub parallel_group_kind: String,
1123 pub parallel_group_base_sequence: u64,
1124 pub parallel_group_size: usize,
1125 pub parallel_group_index: usize,
1126 #[serde(default, skip_serializing_if = "Option::is_none")]
1127 pub parallel_group_mode: Option<String>,
1128 #[serde(default, skip_serializing_if = "Option::is_none")]
1129 pub selection_member_key: Option<SelectionKey>,
1130 #[serde(default, skip_serializing_if = "Option::is_none")]
1131 pub selection_member_index: Option<usize>,
1132 #[serde(default, skip_serializing_if = "Option::is_none")]
1133 pub selection_member_base_sequence: Option<u64>,
1134 #[serde(default, skip_serializing_if = "Option::is_none")]
1135 pub selection_member_size: Option<usize>,
1136 #[serde(default, skip_serializing_if = "Option::is_none")]
1137 pub selection_member_kind: Option<String>,
1138}
1139
1140#[derive(Clone, Debug, PartialEq)]
1142pub enum ParallelResult {
1143 Activity(Value),
1144 ChildWorkflow(ChildWorkflowResult),
1145 Timer,
1146 Signal(Vec<Value>),
1147 Condition(ConditionWaitResult),
1148 Group(Vec<ParallelResult>),
1149}
1150
1151#[derive(Clone, Debug, PartialEq)]
1153pub enum ParallelAvroResult {
1154 Activity(AvroValue),
1155 ChildWorkflow(ChildWorkflowAvroResult),
1156 Timer,
1157 Signal(Vec<AvroValue>),
1158 Condition(ConditionWaitResult),
1159 Group(Vec<ParallelAvroResult>),
1160}
1161
1162impl ParallelAvroResult {
1163 fn into_json_result(self) -> Result<ParallelResult> {
1164 match self {
1165 Self::Activity(value) => Ok(ParallelResult::Activity(value.into_json()?)),
1166 Self::ChildWorkflow(result) => Ok(ParallelResult::ChildWorkflow(ChildWorkflowResult {
1167 parent: result.parent,
1168 child: result.child,
1169 child_workflow_type: result.child_workflow_type,
1170 result: result.result.into_json()?,
1171 })),
1172 Self::Timer => Ok(ParallelResult::Timer),
1173 Self::Signal(values) => Ok(ParallelResult::Signal(
1174 values
1175 .into_iter()
1176 .map(AvroValue::into_json)
1177 .collect::<Result<Vec<_>>>()?,
1178 )),
1179 Self::Condition(result) => Ok(ParallelResult::Condition(result)),
1180 Self::Group(results) => Ok(ParallelResult::Group(
1181 results
1182 .into_iter()
1183 .map(Self::into_json_result)
1184 .collect::<Result<Vec<_>>>()?,
1185 )),
1186 }
1187 }
1188}
1189
1190#[derive(Clone, Debug, PartialEq)]
1192pub struct ParallelCompletion {
1193 pub member_path: Vec<usize>,
1194 pub result: ParallelResult,
1195}
1196
1197#[derive(Debug, Error)]
1205#[error("parallel group {group_id} member {member_path:?} failed: {cause}")]
1206pub struct ParallelFailure {
1207 pub group_id: String,
1208 pub member_path: Vec<usize>,
1209 pub group_path: Vec<ParallelGroupMetadata>,
1210 pub completed: Vec<ParallelCompletion>,
1211 #[source]
1212 pub cause: Box<Error>,
1213}
1214
1215#[derive(Clone, Debug, Error, PartialEq, Eq)]
1217#[error("invalid deterministic parallel group ({reason}): {message}")]
1218pub struct ParallelGroupError {
1219 pub reason: &'static str,
1220 pub member_path: Vec<usize>,
1221 pub message: String,
1222}
1223
1224#[derive(Clone, Debug, Error, PartialEq, Eq)]
1226#[error("workflow cancellation was requested")]
1227pub struct WorkflowCancellationRequested;
1228
1229#[derive(Debug, Error)]
1231#[error(
1232 "saga forward execution failed; compensation activity {compensation_activity_type} (registration {compensation_registration_order}) also failed: {compensation_failure}"
1233)]
1234pub struct SagaCompensationFailure {
1235 pub initiating_failure: Box<Error>,
1236 pub compensation_failure: Box<Error>,
1237 pub compensation_activity_type: String,
1238 pub compensation_registration_order: usize,
1239}
1240
1241#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
1243pub enum ParentClosePolicy {
1244 #[default]
1245 Abandon,
1246 RequestCancel,
1247 Terminate,
1248}
1249
1250impl ParentClosePolicy {
1251 fn as_str(self) -> &'static str {
1252 match self {
1253 Self::Abandon => "abandon",
1254 Self::RequestCancel => "request_cancel",
1255 Self::Terminate => "terminate",
1256 }
1257 }
1258}
1259
1260#[derive(Clone, Debug, Default, PartialEq, Eq)]
1262pub struct ChildWorkflowRetryPolicy {
1263 pub max_attempts: Option<u32>,
1264 pub backoff_seconds: Vec<u64>,
1265 pub non_retryable_error_types: Vec<String>,
1266}
1267
1268#[derive(Clone, Debug, PartialEq, Eq)]
1272pub struct ChildWorkflowOptions {
1273 pub task_queue: String,
1274 pub parent_close_policy: ParentClosePolicy,
1275 pub retry_policy: Option<ChildWorkflowRetryPolicy>,
1276 pub execution_timeout_seconds: Option<u64>,
1277 pub run_timeout_seconds: Option<u64>,
1278}
1279
1280impl ChildWorkflowOptions {
1281 pub fn new(task_queue: impl Into<String>) -> Self {
1282 Self {
1283 task_queue: task_queue.into(),
1284 parent_close_policy: ParentClosePolicy::Abandon,
1285 retry_policy: None,
1286 execution_timeout_seconds: None,
1287 run_timeout_seconds: None,
1288 }
1289 }
1290
1291 pub fn parent_close_policy(mut self, policy: ParentClosePolicy) -> Self {
1292 self.parent_close_policy = policy;
1293 self
1294 }
1295
1296 pub fn retry_policy(mut self, policy: ChildWorkflowRetryPolicy) -> Self {
1297 self.retry_policy = Some(policy);
1298 self
1299 }
1300
1301 pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
1302 self.execution_timeout_seconds = Some(seconds);
1303 self
1304 }
1305
1306 pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
1307 self.run_timeout_seconds = Some(seconds);
1308 self
1309 }
1310}
1311
1312#[derive(Clone, Debug, PartialEq, Eq)]
1314pub enum ActivityBackoff {
1315 Explicit(Vec<Duration>),
1318 Exponential {
1320 initial_interval: Duration,
1321 coefficient: u32,
1322 maximum_interval: Option<Duration>,
1323 },
1324}
1325
1326#[derive(Clone, Debug, Default, PartialEq, Eq)]
1328pub struct ActivityRetryPolicy {
1329 pub max_attempts: Option<u32>,
1330 pub backoff: Option<ActivityBackoff>,
1331 pub non_retryable_error_types: Vec<String>,
1332}
1333
1334impl ActivityRetryPolicy {
1335 pub fn new(max_attempts: u32) -> Self {
1337 Self {
1338 max_attempts: Some(max_attempts),
1339 ..Self::default()
1340 }
1341 }
1342
1343 pub fn backoff_intervals(mut self, intervals: impl IntoIterator<Item = Duration>) -> Self {
1344 self.backoff = Some(ActivityBackoff::Explicit(intervals.into_iter().collect()));
1345 self
1346 }
1347
1348 pub fn exponential_backoff(
1349 mut self,
1350 initial_interval: Duration,
1351 coefficient: u32,
1352 maximum_interval: Option<Duration>,
1353 ) -> Self {
1354 self.backoff = Some(ActivityBackoff::Exponential {
1355 initial_interval,
1356 coefficient,
1357 maximum_interval,
1358 });
1359 self
1360 }
1361
1362 pub fn non_retryable_error_type(mut self, error_type: impl Into<String>) -> Self {
1363 self.non_retryable_error_types.push(error_type.into());
1364 self
1365 }
1366
1367 pub fn non_retryable_error_types(
1368 mut self,
1369 error_types: impl IntoIterator<Item = impl Into<String>>,
1370 ) -> Self {
1371 self.non_retryable_error_types
1372 .extend(error_types.into_iter().map(Into::into));
1373 self
1374 }
1375}
1376
1377#[derive(Clone, Debug, Default, PartialEq, Eq)]
1382pub struct ActivityOptions {
1383 pub task_queue: Option<String>,
1384 pub retry_policy: Option<ActivityRetryPolicy>,
1385 pub start_to_close_timeout: Option<Duration>,
1386 pub schedule_to_start_timeout: Option<Duration>,
1387 pub schedule_to_close_timeout: Option<Duration>,
1388 pub heartbeat_timeout: Option<Duration>,
1389}
1390
1391impl ActivityOptions {
1392 pub fn new() -> Self {
1393 Self::default()
1394 }
1395
1396 pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
1397 self.task_queue = Some(task_queue.into());
1398 self
1399 }
1400
1401 pub fn retry_policy(mut self, policy: ActivityRetryPolicy) -> Self {
1402 self.retry_policy = Some(policy);
1403 self
1404 }
1405
1406 pub fn start_to_close_timeout(mut self, timeout: Duration) -> Self {
1407 self.start_to_close_timeout = Some(timeout);
1408 self
1409 }
1410
1411 pub fn schedule_to_start_timeout(mut self, timeout: Duration) -> Self {
1412 self.schedule_to_start_timeout = Some(timeout);
1413 self
1414 }
1415
1416 pub fn schedule_to_close_timeout(mut self, timeout: Duration) -> Self {
1417 self.schedule_to_close_timeout = Some(timeout);
1418 self
1419 }
1420
1421 pub fn heartbeat_timeout(mut self, timeout: Duration) -> Self {
1422 self.heartbeat_timeout = Some(timeout);
1423 self
1424 }
1425
1426 fn validate(&self) -> std::result::Result<ValidatedActivityOptions, ActivityOptionsError> {
1427 if self
1428 .task_queue
1429 .as_deref()
1430 .is_some_and(|queue| queue.trim().is_empty())
1431 {
1432 return Err(ActivityOptionsError::new(
1433 ActivityOptionsErrorKind::EmptyTaskQueue,
1434 Some("task_queue"),
1435 "task_queue must not be empty",
1436 ));
1437 }
1438
1439 for (field, value) in [
1440 ("start_to_close_timeout", self.start_to_close_timeout),
1441 ("schedule_to_start_timeout", self.schedule_to_start_timeout),
1442 ("schedule_to_close_timeout", self.schedule_to_close_timeout),
1443 ("heartbeat_timeout", self.heartbeat_timeout),
1444 ] {
1445 if value.is_some_and(|value| value.is_zero()) {
1446 return Err(ActivityOptionsError::new(
1447 ActivityOptionsErrorKind::TimeoutNotPositive,
1448 Some(field),
1449 format!("{field} must be positive"),
1450 ));
1451 }
1452 }
1453
1454 validate_timeout_order(
1455 "heartbeat_timeout",
1456 self.heartbeat_timeout,
1457 "start_to_close_timeout",
1458 self.start_to_close_timeout,
1459 )?;
1460 validate_timeout_order(
1461 "start_to_close_timeout",
1462 self.start_to_close_timeout,
1463 "schedule_to_close_timeout",
1464 self.schedule_to_close_timeout,
1465 )?;
1466 validate_timeout_order(
1467 "schedule_to_start_timeout",
1468 self.schedule_to_start_timeout,
1469 "schedule_to_close_timeout",
1470 self.schedule_to_close_timeout,
1471 )?;
1472
1473 Ok(ValidatedActivityOptions {
1474 task_queue: self.task_queue.clone(),
1475 retry_policy: self
1476 .retry_policy
1477 .as_ref()
1478 .map(validate_activity_retry_policy)
1479 .transpose()?,
1480 start_to_close_timeout: timeout_seconds(
1481 "start_to_close_timeout",
1482 self.start_to_close_timeout,
1483 )?,
1484 schedule_to_start_timeout: timeout_seconds(
1485 "schedule_to_start_timeout",
1486 self.schedule_to_start_timeout,
1487 )?,
1488 schedule_to_close_timeout: timeout_seconds(
1489 "schedule_to_close_timeout",
1490 self.schedule_to_close_timeout,
1491 )?,
1492 heartbeat_timeout: timeout_seconds("heartbeat_timeout", self.heartbeat_timeout)?,
1493 })
1494 }
1495}
1496
1497pub enum ParallelOperation {
1503 Activity {
1504 activity_type: String,
1505 options: ActivityOptions,
1506 arguments: Result<AvroValue>,
1507 },
1508 ChildWorkflow {
1509 workflow_type: String,
1510 options: ChildWorkflowOptions,
1511 arguments: Result<AvroValue>,
1512 },
1513 Timer(Duration),
1514 Signal(String),
1515 Condition {
1516 options: ConditionWaitOptions,
1517 predicate: Box<dyn Fn() -> Result<bool> + Send + 'static>,
1518 },
1519 Group(Vec<ParallelOperation>),
1520}
1521
1522impl ParallelOperation {
1523 pub fn activity<T: Serialize>(activity_type: impl Into<String>, args: T) -> Self {
1524 Self::activity_with_options(activity_type, ActivityOptions::new(), args)
1525 }
1526
1527 pub fn activity_with_options<T: Serialize>(
1528 activity_type: impl Into<String>,
1529 options: ActivityOptions,
1530 args: T,
1531 ) -> Self {
1532 Self::Activity {
1533 activity_type: activity_type.into(),
1534 options,
1535 arguments: AvroValue::from_serialize(&args),
1536 }
1537 }
1538
1539 pub fn child_workflow<T: Serialize>(
1540 workflow_type: impl Into<String>,
1541 options: ChildWorkflowOptions,
1542 args: T,
1543 ) -> Self {
1544 Self::ChildWorkflow {
1545 workflow_type: workflow_type.into(),
1546 options,
1547 arguments: AvroValue::from_serialize(&args),
1548 }
1549 }
1550
1551 pub fn timer(duration: Duration) -> Self {
1552 Self::Timer(duration)
1553 }
1554
1555 pub fn signal(signal_name: impl Into<String>) -> Self {
1556 Self::Signal(signal_name.into())
1557 }
1558
1559 pub fn condition<F>(options: ConditionWaitOptions, predicate: F) -> Self
1560 where
1561 F: Fn() -> Result<bool> + Send + 'static,
1562 {
1563 Self::Condition {
1564 options,
1565 predicate: Box::new(predicate),
1566 }
1567 }
1568
1569 pub fn group(operations: Vec<ParallelOperation>) -> Self {
1570 Self::Group(operations)
1571 }
1572}
1573
1574#[derive(Clone, Debug)]
1575struct ValidatedActivityOptions {
1576 task_queue: Option<String>,
1577 retry_policy: Option<Value>,
1578 start_to_close_timeout: Option<u64>,
1579 schedule_to_start_timeout: Option<u64>,
1580 schedule_to_close_timeout: Option<u64>,
1581 heartbeat_timeout: Option<u64>,
1582}
1583
1584fn validate_timeout_order(
1585 smaller_name: &'static str,
1586 smaller: Option<Duration>,
1587 larger_name: &'static str,
1588 larger: Option<Duration>,
1589) -> std::result::Result<(), ActivityOptionsError> {
1590 if matches!((smaller, larger), (Some(smaller), Some(larger)) if smaller > larger) {
1591 return Err(ActivityOptionsError::new(
1592 ActivityOptionsErrorKind::TimeoutOrder,
1593 Some(smaller_name),
1594 format!("{smaller_name} must be <= {larger_name}"),
1595 ));
1596 }
1597 Ok(())
1598}
1599
1600fn timeout_seconds(
1601 field: &'static str,
1602 value: Option<Duration>,
1603) -> std::result::Result<Option<u64>, ActivityOptionsError> {
1604 value
1605 .map(|value| {
1606 activity_protocol_seconds(value).ok_or_else(|| {
1607 ActivityOptionsError::new(
1608 ActivityOptionsErrorKind::TimeoutOverflow,
1609 Some(field),
1610 format!("{field} is too large for the worker protocol"),
1611 )
1612 })
1613 })
1614 .transpose()
1615}
1616
1617fn duration_seconds_ceil(value: Duration) -> Option<u64> {
1618 value
1619 .as_secs()
1620 .checked_add(u64::from(value.subsec_nanos() > 0))
1621}
1622
1623fn activity_protocol_seconds(value: Duration) -> Option<u64> {
1624 duration_seconds_ceil(value).filter(|seconds| *seconds <= i64::MAX as u64)
1625}
1626
1627fn validate_activity_retry_policy(
1628 policy: &ActivityRetryPolicy,
1629) -> std::result::Result<Value, ActivityOptionsError> {
1630 if policy.max_attempts.is_none()
1631 && policy.backoff.is_none()
1632 && policy.non_retryable_error_types.is_empty()
1633 {
1634 return Err(ActivityOptionsError::new(
1635 ActivityOptionsErrorKind::EmptyRetryPolicy,
1636 Some("retry_policy"),
1637 "retry_policy must configure at least one field",
1638 ));
1639 }
1640 if policy.max_attempts == Some(0) {
1641 return Err(ActivityOptionsError::new(
1642 ActivityOptionsErrorKind::InvalidMaxAttempts,
1643 Some("retry_policy.max_attempts"),
1644 "max_attempts must be >= 1",
1645 ));
1646 }
1647 if policy
1648 .non_retryable_error_types
1649 .iter()
1650 .any(|error_type| error_type.trim().is_empty())
1651 {
1652 return Err(ActivityOptionsError::new(
1653 ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
1654 Some("retry_policy.non_retryable_error_types"),
1655 "non_retryable_error_types must not contain empty values",
1656 ));
1657 }
1658
1659 let backoff_seconds = match &policy.backoff {
1660 None => None,
1661 Some(backoff) => {
1662 let max_attempts = policy.max_attempts.ok_or_else(|| {
1663 ActivityOptionsError::new(
1664 ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
1665 Some("retry_policy.backoff"),
1666 "backoff requires max_attempts",
1667 )
1668 })?;
1669 let retry_count = max_attempts.saturating_sub(1) as usize;
1670 let intervals = match backoff {
1671 ActivityBackoff::Explicit(intervals) => {
1672 if intervals.len() > retry_count {
1673 return Err(ActivityOptionsError::new(
1674 ActivityOptionsErrorKind::TooManyBackoffIntervals,
1675 Some("retry_policy.backoff"),
1676 "backoff interval count must not exceed max_attempts - 1",
1677 ));
1678 }
1679 intervals.clone()
1680 }
1681 ActivityBackoff::Exponential {
1682 initial_interval,
1683 coefficient,
1684 maximum_interval,
1685 } => {
1686 if *coefficient < 1 {
1687 return Err(ActivityOptionsError::new(
1688 ActivityOptionsErrorKind::InvalidBackoffCoefficient,
1689 Some("retry_policy.backoff.coefficient"),
1690 "backoff coefficient must be >= 1",
1691 ));
1692 }
1693 if retry_count > 10_000 {
1694 return Err(ActivityOptionsError::new(
1695 ActivityOptionsErrorKind::BackoffGenerationTooLarge,
1696 Some("retry_policy.max_attempts"),
1697 "generated backoff supports at most 10000 retry intervals",
1698 ));
1699 }
1700 let mut current = *initial_interval;
1701 let mut intervals = Vec::with_capacity(retry_count);
1702 for _ in 0..retry_count {
1703 let interval = maximum_interval
1704 .map(|maximum| current.min(maximum))
1705 .unwrap_or(current);
1706 intervals.push(interval);
1707 if maximum_interval.is_some_and(|maximum| interval == maximum) {
1708 break;
1709 }
1710 current = current.checked_mul(*coefficient).ok_or_else(|| {
1711 ActivityOptionsError::new(
1712 ActivityOptionsErrorKind::BackoffOverflow,
1713 Some("retry_policy.backoff"),
1714 "generated backoff interval overflowed",
1715 )
1716 })?;
1717 }
1718 intervals
1719 }
1720 };
1721 Some(
1722 intervals
1723 .into_iter()
1724 .map(|interval| {
1725 activity_protocol_seconds(interval).ok_or_else(|| {
1726 ActivityOptionsError::new(
1727 ActivityOptionsErrorKind::BackoffOverflow,
1728 Some("retry_policy.backoff"),
1729 "backoff interval is too large for the worker protocol",
1730 )
1731 })
1732 })
1733 .collect::<std::result::Result<Vec<_>, _>>()?,
1734 )
1735 }
1736 };
1737
1738 let mut encoded = serde_json::Map::new();
1739 if let Some(max_attempts) = policy.max_attempts {
1740 encoded.insert("max_attempts".to_string(), json!(max_attempts));
1741 }
1742 if let Some(backoff_seconds) = backoff_seconds {
1743 encoded.insert("backoff_seconds".to_string(), json!(backoff_seconds));
1744 }
1745 if !policy.non_retryable_error_types.is_empty() {
1746 let mut canonical_error_types = Vec::new();
1747 for error_type in policy
1748 .non_retryable_error_types
1749 .iter()
1750 .map(|error_type| error_type.trim())
1751 {
1752 if !canonical_error_types.contains(&error_type) {
1753 canonical_error_types.push(error_type);
1754 }
1755 }
1756 encoded.insert(
1757 "non_retryable_error_types".to_string(),
1758 json!(canonical_error_types),
1759 );
1760 }
1761 Ok(Value::Object(encoded))
1762}
1763
1764#[derive(Clone, Debug, Error)]
1767#[error("non-deterministic workflow replay ({reason}) at sequence {sequence:?}: {message}")]
1768pub struct ReplayFailure {
1769 pub reason: String,
1770 pub sequence: Option<u64>,
1771 pub expected: Option<String>,
1772 pub actual: Option<String>,
1773 pub message: String,
1774}
1775
1776impl ReplayFailure {
1777 fn new(
1778 reason: impl Into<String>,
1779 sequence: Option<u64>,
1780 expected: Option<String>,
1781 actual: Option<String>,
1782 message: impl Into<String>,
1783 ) -> Self {
1784 Self {
1785 reason: reason.into(),
1786 sequence,
1787 expected,
1788 actual,
1789 message: message.into(),
1790 }
1791 }
1792}
1793
1794#[derive(Clone, Debug, Error)]
1796#[error("query failed ({reason}, HTTP {status}): {message}")]
1797pub struct QueryFailure {
1798 pub status: u16,
1799 pub reason: String,
1800 pub message: String,
1801 pub body: Value,
1802}
1803
1804#[derive(Clone, Debug, Error)]
1806#[error("protocol rejected ({reason}, HTTP {status}): {message}")]
1807pub struct ProtocolFailure {
1808 pub status: u16,
1809 pub reason: String,
1810 pub message: String,
1811 pub supported_version: Option<String>,
1812 pub requested_version: Option<String>,
1813 pub body: Value,
1814}
1815
1816#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
1817pub struct PayloadEnvelope {
1818 pub codec: String,
1819 pub blob: String,
1820}
1821
1822impl PayloadEnvelope {
1823 pub fn avro<T: Serialize>(value: &T) -> Result<Self> {
1824 encode_payload(value, DEFAULT_CODEC)
1825 }
1826
1827 pub fn avro_value(value: &AvroValue) -> Result<Self> {
1830 encode_avro_value(value)
1831 }
1832}
1833
1834#[derive(Clone, Debug)]
1836pub enum AvroValue {
1837 Null,
1838 Boolean(bool),
1839 Long(i64),
1840 Double(f64),
1841 Bytes(Vec<u8>),
1842 String(String),
1843 Array(Vec<AvroValue>),
1844 Map(BTreeMap<String, AvroValue>),
1845}
1846
1847impl PartialEq for AvroValue {
1848 fn eq(&self, other: &Self) -> bool {
1849 match (self, other) {
1850 (Self::Null, Self::Null) => true,
1851 (Self::Boolean(left), Self::Boolean(right)) => left == right,
1852 (Self::Long(left), Self::Long(right)) => left == right,
1853 (Self::Double(left), Self::Double(right)) => left.to_bits() == right.to_bits(),
1854 (Self::Bytes(left), Self::Bytes(right)) => left == right,
1855 (Self::String(left), Self::String(right)) => left == right,
1856 (Self::Array(left), Self::Array(right)) => left == right,
1857 (Self::Map(left), Self::Map(right)) => left == right,
1858 _ => false,
1859 }
1860 }
1861}
1862
1863impl AvroValue {
1864 fn from_serialize<T: Serialize>(value: &T) -> Result<Self> {
1865 Self::from_serde_value(
1866 serde_value::to_value(value).map_err(|error| {
1867 Error::Codec(format!("could not adapt value for Avro: {error}"))
1868 })?,
1869 )
1870 }
1871
1872 fn from_serde_value(value: serde_value::Value) -> Result<Self> {
1873 use serde_value::Value as SerdeValue;
1874
1875 match value {
1876 SerdeValue::Unit => Ok(Self::Null),
1877 SerdeValue::Bool(value) => Ok(Self::Boolean(value)),
1878 SerdeValue::I8(value) => Ok(Self::Long(i64::from(value))),
1879 SerdeValue::I16(value) => Ok(Self::Long(i64::from(value))),
1880 SerdeValue::I32(value) => Ok(Self::Long(i64::from(value))),
1881 SerdeValue::I64(value) => Ok(Self::Long(value)),
1882 SerdeValue::U8(value) => Ok(Self::Long(i64::from(value))),
1883 SerdeValue::U16(value) => Ok(Self::Long(i64::from(value))),
1884 SerdeValue::U32(value) => Ok(Self::Long(i64::from(value))),
1885 SerdeValue::U64(value) => i64::try_from(value).map(Self::Long).map_err(|_| {
1886 Error::Codec(
1887 "integer_overflow: Avro Value long must be within signed 64-bit range"
1888 .to_string(),
1889 )
1890 }),
1891 SerdeValue::F32(value) => Self::finite_double(f64::from(value)),
1892 SerdeValue::F64(value) => Self::finite_double(value),
1893 SerdeValue::Char(value) => Ok(Self::String(value.to_string())),
1894 SerdeValue::String(value) => Ok(Self::String(value)),
1895 SerdeValue::Bytes(value) => Ok(Self::Bytes(value)),
1896 SerdeValue::Option(None) => Ok(Self::Null),
1897 SerdeValue::Option(Some(value)) | SerdeValue::Newtype(value) => {
1898 Self::from_serde_value(*value)
1899 }
1900 SerdeValue::Seq(values) => values
1901 .into_iter()
1902 .map(Self::from_serde_value)
1903 .collect::<Result<Vec<_>>>()
1904 .map(Self::Array),
1905 SerdeValue::Map(values) => values
1906 .into_iter()
1907 .map(|(key, value)| {
1908 let SerdeValue::String(key) = key else {
1909 return Err(Error::Codec(
1910 "invalid_map_key: Avro Value map keys must be strings".to_string(),
1911 ));
1912 };
1913
1914 Ok((key, Self::from_serde_value(value)?))
1915 })
1916 .collect::<Result<BTreeMap<_, _>>>()
1917 .map(Self::Map),
1918 }
1919 }
1920
1921 fn finite_double(value: f64) -> Result<Self> {
1922 if !value.is_finite() {
1923 return Err(Error::Codec(
1924 "non_finite_float: Avro Value doubles must be finite".to_string(),
1925 ));
1926 }
1927
1928 Ok(Self::Double(value))
1929 }
1930
1931 fn into_json(self) -> Result<Value> {
1932 match self {
1933 Self::Null => Ok(Value::Null),
1934 Self::Boolean(value) => Ok(Value::Bool(value)),
1935 Self::Long(value) => Ok(Value::Number(value.into())),
1936 Self::Double(value) => serde_json::Number::from_f64(value)
1937 .map(Value::Number)
1938 .ok_or_else(|| {
1939 Error::Codec(
1940 "non_finite_float: decoded Avro Value double is not finite".to_string(),
1941 )
1942 }),
1943 Self::Bytes(value) => Ok(json!({
1944 "$type": "bytes",
1945 "base64": BASE64.encode(value),
1946 })),
1947 Self::String(value) => Ok(Value::String(value)),
1948 Self::Array(values) => values
1949 .into_iter()
1950 .map(Self::into_json)
1951 .collect::<Result<Vec<_>>>()
1952 .map(Value::Array),
1953 Self::Map(values) => values
1954 .into_iter()
1955 .map(|(key, value)| Ok((key, value.into_json()?)))
1956 .collect::<Result<serde_json::Map<_, _>>>()
1957 .map(Value::Object),
1958 }
1959 }
1960
1961 fn into_serde_value(self) -> serde_value::Value {
1962 use serde_value::Value as SerdeValue;
1963
1964 match self {
1965 Self::Null => SerdeValue::Unit,
1966 Self::Boolean(value) => SerdeValue::Bool(value),
1967 Self::Long(value) => SerdeValue::I64(value),
1968 Self::Double(value) => SerdeValue::F64(value),
1969 Self::Bytes(value) => SerdeValue::Bytes(value),
1970 Self::String(value) => SerdeValue::String(value),
1971 Self::Array(values) => {
1972 SerdeValue::Seq(values.into_iter().map(Self::into_serde_value).collect())
1973 }
1974 Self::Map(values) => SerdeValue::Map(
1975 values
1976 .into_iter()
1977 .map(|(key, value)| (SerdeValue::String(key), value.into_serde_value()))
1978 .collect(),
1979 ),
1980 }
1981 }
1982
1983 pub fn deserialize<T: DeserializeOwned>(self) -> Result<T> {
1984 self.into_serde_value().deserialize_into().map_err(|error| {
1985 Error::Codec(format!(
1986 "avro_value_type_mismatch: could not adapt decoded value: {error}"
1987 ))
1988 })
1989 }
1990}
1991
1992impl Serialize for AvroValue {
1993 fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
1994 where
1995 S: Serializer,
1996 {
1997 match self {
1998 Self::Null => serializer.serialize_unit(),
1999 Self::Boolean(value) => serializer.serialize_bool(*value),
2000 Self::Long(value) => serializer.serialize_i64(*value),
2001 Self::Double(value) => serializer.serialize_f64(*value),
2002 Self::Bytes(value) => serializer.serialize_bytes(value),
2003 Self::String(value) => serializer.serialize_str(value),
2004 Self::Array(values) => {
2005 let mut sequence = serializer.serialize_seq(Some(values.len()))?;
2006 for value in values {
2007 sequence.serialize_element(value)?;
2008 }
2009 sequence.end()
2010 }
2011 Self::Map(values) => {
2012 let mut map = serializer.serialize_map(Some(values.len()))?;
2013 for (key, value) in values {
2014 map.serialize_entry(key, value)?;
2015 }
2016 map.end()
2017 }
2018 }
2019 }
2020}
2021
2022pub fn encode_avro_value(value: &AvroValue) -> Result<PayloadEnvelope> {
2023 let datum = avro_value_to_datum(value)?;
2024 let datum = to_avro_datum(avro_value_ordered_map_encoding_schema()?, datum)
2025 .map_err(|err| Error::Codec(format!("avro_value_encode_failed: {err}")))?;
2026 let mut bytes = Vec::with_capacity(datum.len() + 10);
2027 bytes.extend_from_slice(&AVRO_SINGLE_OBJECT_MAGIC);
2028 bytes.extend_from_slice(&AVRO_VALUE_SCHEMA_FINGERPRINT);
2029 bytes.extend_from_slice(&datum);
2030 Ok(PayloadEnvelope {
2031 codec: DEFAULT_CODEC.to_string(),
2032 blob: BASE64.encode(bytes),
2033 })
2034}
2035
2036pub fn decode_avro_value(envelope: &PayloadEnvelope) -> Result<AvroValue> {
2037 if envelope.codec != DEFAULT_CODEC {
2038 return Err(unsupported_payload_codec(&envelope.codec));
2039 }
2040 decode_avro_value_blob(&envelope.blob)
2041}
2042
2043pub fn encode_payload<T: Serialize>(value: &T, codec: &str) -> Result<PayloadEnvelope> {
2044 let blob = match codec {
2045 DEFAULT_CODEC => encode_avro_value(&AvroValue::from_serialize(value)?)?.blob,
2046 other => return Err(unsupported_payload_codec(other)),
2047 };
2048
2049 Ok(PayloadEnvelope {
2050 codec: codec.to_string(),
2051 blob,
2052 })
2053}
2054
2055pub fn decode_payload<T: DeserializeOwned>(envelope: &PayloadEnvelope) -> Result<T> {
2056 match envelope.codec.as_str() {
2057 DEFAULT_CODEC => decode_avro_value(envelope)?.deserialize(),
2058 other => Err(unsupported_payload_codec(other)),
2059 }
2060}
2061
2062fn handler_type_error<T>(
2063 handler_kind: HandlerKind,
2064 handler_name: &str,
2065 value_kind: HandlerValueKind,
2066 message: impl Into<String>,
2067) -> Error {
2068 Error::HandlerType {
2069 handler_kind,
2070 handler_name: handler_name.to_string(),
2071 value_kind,
2072 rust_type: type_name::<T>(),
2073 message: message.into(),
2074 }
2075}
2076
2077fn decode_handler_input<T: DeserializeOwned>(
2078 arguments: AvroValue,
2079 handler_kind: HandlerKind,
2080 handler_name: &str,
2081) -> Result<T> {
2082 let argument = match arguments {
2083 AvroValue::Array(mut arguments) if arguments.len() == 1 => {
2084 arguments.pop().expect("one typed handler argument")
2085 }
2086 AvroValue::Array(arguments) if arguments.is_empty() => AvroValue::Null,
2087 AvroValue::Array(arguments) => {
2088 return Err(handler_type_error::<T>(
2089 handler_kind,
2090 handler_name,
2091 HandlerValueKind::Input,
2092 format!(
2093 "typed handlers accept one request value, but the task carried {} arguments",
2094 arguments.len()
2095 ),
2096 ));
2097 }
2098 argument => argument,
2099 };
2100
2101 argument.deserialize().map_err(|error| {
2102 handler_type_error::<T>(
2103 handler_kind,
2104 handler_name,
2105 HandlerValueKind::Input,
2106 error.to_string(),
2107 )
2108 })
2109}
2110
2111fn encode_handler_result<T: Serialize>(
2112 result: &T,
2113 handler_kind: HandlerKind,
2114 handler_name: &str,
2115) -> Result<AvroValue> {
2116 AvroValue::from_serialize(result).map_err(|error| {
2117 handler_type_error::<T>(
2118 handler_kind,
2119 handler_name,
2120 HandlerValueKind::Result,
2121 error.to_string(),
2122 )
2123 })
2124}
2125
2126fn decode_handler_result<T: DeserializeOwned>(
2127 result: AvroValue,
2128 handler_kind: HandlerKind,
2129 handler_name: &str,
2130) -> Result<T> {
2131 result.deserialize().map_err(|error| {
2132 handler_type_error::<T>(
2133 handler_kind,
2134 handler_name,
2135 HandlerValueKind::Result,
2136 error.to_string(),
2137 )
2138 })
2139}
2140
2141#[cfg(test)]
2142fn encode_value_envelope(value: &Value, codec: &str) -> Result<Value> {
2143 Ok(serde_json::to_value(encode_payload(value, codec)?)?)
2144}
2145
2146fn decode_wire_value(value: &Value, fallback_codec: &str) -> Result<Value> {
2147 validate_payload_codec(fallback_codec)?;
2148
2149 if value.is_null() {
2150 return Ok(Value::Null);
2151 }
2152
2153 if let Some((codec, blob)) = payload_envelope_parts(value)? {
2154 return decode_blob(blob, codec);
2155 }
2156
2157 if let Some(blob) = value.as_str() {
2158 return decode_blob(blob, fallback_codec);
2159 }
2160
2161 Err(untagged_payload_value())
2162}
2163
2164fn encode_typed_envelope(value: &AvroValue, codec: &str) -> Result<Value> {
2165 let envelope = match codec {
2166 DEFAULT_CODEC => encode_avro_value(value)?,
2167 other => return Err(unsupported_payload_codec(other)),
2168 };
2169 Ok(serde_json::to_value(envelope)?)
2170}
2171
2172fn decode_wire_avro_value(value: &Value, fallback_codec: &str) -> Result<AvroValue> {
2173 validate_payload_codec(fallback_codec)?;
2174
2175 if value.is_null() {
2176 return Ok(AvroValue::Null);
2177 }
2178
2179 if let Some((codec, blob)) = payload_envelope_parts(value)? {
2180 validate_payload_codec(codec)?;
2181 return decode_avro_value_blob(blob);
2182 }
2183
2184 if let Some(blob) = value.as_str() {
2185 return match fallback_codec {
2186 DEFAULT_CODEC => decode_avro_value_blob(blob),
2187 other => Err(unsupported_payload_codec(other)),
2188 };
2189 }
2190
2191 Err(untagged_payload_value())
2192}
2193
2194fn normalize_avro_arguments(value: AvroValue) -> AvroValue {
2195 match value {
2196 AvroValue::Null => AvroValue::Array(Vec::new()),
2197 AvroValue::Array(_) => value,
2198 other => AvroValue::Array(vec![other]),
2199 }
2200}
2201
2202fn decode_blob(blob: &str, codec: &str) -> Result<Value> {
2203 match codec {
2204 DEFAULT_CODEC => decode_avro_value_blob(blob)?.into_json(),
2205 other => Err(unsupported_payload_codec(other)),
2206 }
2207}
2208
2209fn validate_payload_codec(codec: &str) -> Result<()> {
2210 match codec {
2211 DEFAULT_CODEC => Ok(()),
2212 MISSING_TASK_PAYLOAD_CODEC => {
2213 Err(invalid_task_payload_codec("task payload_codec is missing"))
2214 }
2215 NULL_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec("task payload_codec is null")),
2216 NON_STRING_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec(
2217 "task payload_codec must be a string",
2218 )),
2219 other => Err(unsupported_payload_codec(other)),
2220 }
2221}
2222
2223fn invalid_task_payload_codec(reason: &str) -> Error {
2224 Error::Codec(format!(
2225 "unsupported_payload_codec: {reason}; Durable Workflow 2.0 requires an explicit string payload_codec=\"avro\" before worker task execution"
2226 ))
2227}
2228
2229fn payload_envelope_parts(value: &Value) -> Result<Option<(&str, &str)>> {
2230 let Some(object) = value.as_object() else {
2231 return Ok(None);
2232 };
2233 if !object.contains_key("codec") && !object.contains_key("blob") {
2234 return Ok(None);
2235 }
2236
2237 let codec = object
2238 .get("codec")
2239 .and_then(Value::as_str)
2240 .ok_or_else(invalid_payload_envelope)?;
2241 validate_payload_codec(codec)?;
2242 let blob = object
2243 .get("blob")
2244 .and_then(Value::as_str)
2245 .ok_or_else(invalid_payload_envelope)?;
2246 Ok(Some((codec, blob)))
2247}
2248
2249fn invalid_payload_envelope() -> Error {
2250 Error::Codec(
2251 "invalid_payload_envelope: durable payloads must use an object with string codec=\"avro\" and blob fields"
2252 .to_string(),
2253 )
2254}
2255
2256fn validate_workflow_task_commands(commands: &[Value]) -> Result<()> {
2257 for command in commands {
2258 let Some(command) = command.as_object() else {
2259 continue;
2260 };
2261 let Some(command_type) = command.get("type").and_then(Value::as_str) else {
2262 continue;
2263 };
2264 let Some(payload_field) = workflow_command_payload_field(command_type) else {
2265 continue;
2266 };
2267
2268 if let Some(codec) = command.get("payload_codec") {
2269 let codec = codec.as_str().ok_or_else(invalid_payload_envelope)?;
2270 validate_payload_codec(codec)?;
2271 }
2272
2273 let payload = command
2274 .get(payload_field)
2275 .ok_or_else(invalid_payload_envelope)?;
2276 if runtime_payloads::Reference::parse(payload)?.is_none() {
2277 validate_outbound_payload_envelope(payload)?;
2278 }
2279 }
2280 Ok(())
2281}
2282
2283fn workflow_completion_protocol_version(commands: &[Value]) -> &'static str {
2284 if commands.iter().any(|command| {
2285 command.get("type").and_then(Value::as_str) == Some("open_condition_wait")
2286 && command
2287 .get("condition_wait_occurrence_id")
2288 .and_then(Value::as_str)
2289 .is_some_and(|occurrence_id| !occurrence_id.is_empty())
2290 }) {
2291 CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
2292 } else if commands.iter().any(|command| {
2293 command.get("type").and_then(Value::as_str) == Some("upsert_search_attributes")
2294 && command.get("attribute_types").is_some()
2295 }) {
2296 TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION
2297 } else if commands
2298 .iter()
2299 .any(|command| command.get("type").and_then(Value::as_str) == Some("upsert_memo"))
2300 {
2301 MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION
2302 } else if commands
2303 .iter()
2304 .any(|command| command.get("type").and_then(Value::as_str) == Some("open_condition_wait"))
2305 {
2306 CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION
2307 } else if commands.iter().any(|command| {
2308 command.get("type").and_then(Value::as_str) == Some("upsert_search_attributes")
2309 }) {
2310 SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION
2311 } else {
2312 WORKER_PROTOCOL_VERSION
2313 }
2314}
2315
2316fn workflow_completion_protocol_version_with_message_streams(
2317 commands: &[Value],
2318 has_message_stream_metadata: bool,
2319) -> &'static str {
2320 let command_protocol = workflow_completion_protocol_version(commands);
2321 if has_message_stream_metadata && !worker_protocol_supports_message_streams(command_protocol) {
2322 MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION
2323 } else {
2324 command_protocol
2325 }
2326}
2327
2328fn workflow_command_payload_field(command_type: &str) -> Option<&'static str> {
2329 match command_type {
2330 "complete_workflow" | "complete_update" | "record_side_effect" => Some("result"),
2331 "schedule_activity" | "start_child_workflow" | "continue_as_new" => Some("arguments"),
2332 "start_service_operation" => Some("request_payload"),
2333 "upsert_memo" => Some("entries"),
2334 _ => None,
2335 }
2336}
2337
2338fn validate_outbound_payload_envelope(value: &Value) -> Result<()> {
2339 let Some((codec, blob)) = payload_envelope_parts(value)? else {
2340 return Err(untagged_payload_value());
2341 };
2342 validate_payload_codec(codec)?;
2343 decode_avro_value_blob(blob)?;
2344 Ok(())
2345}
2346
2347fn unsupported_payload_codec(codec: &str) -> Error {
2348 Error::Codec(format!(
2349 "unsupported_payload_codec: workflow payload codec {codec:?} is not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
2350 ))
2351}
2352
2353fn untagged_payload_value() -> Error {
2354 Error::Codec(
2355 "unsupported_payload_codec: untagged durable payload values are not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
2356 .to_string(),
2357 )
2358}
2359
2360fn decode_avro_value_blob(blob: &str) -> Result<AvroValue> {
2361 let bytes = BASE64.decode(blob).map_err(|err| {
2362 Error::Codec(format!(
2363 "invalid_payload_framing: expected strict base64 Avro single-object bytes: {err}"
2364 ))
2365 })?;
2366
2367 if serde_json::from_slice::<Value>(&bytes).is_ok() {
2368 return Err(unsupported_payload_codec("json"));
2369 }
2370
2371 if bytes.len() < 10 || bytes[..2] != AVRO_SINGLE_OBJECT_MAGIC {
2372 return Err(Error::Codec(
2373 "invalid_payload_framing: expected Avro single-object magic c301".to_string(),
2374 ));
2375 }
2376
2377 let fingerprint: [u8; 8] = bytes[2..10]
2378 .try_into()
2379 .map_err(|_| Error::Codec("invalid Avro fingerprint length".to_string()))?;
2380 if fingerprint != AVRO_VALUE_SCHEMA_FINGERPRINT {
2381 return Err(Error::Codec(format!(
2382 "unsupported_payload_schema: unknown CRC-64-AVRO fingerprint {}",
2383 fingerprint
2384 .iter()
2385 .map(|byte| format!("{byte:02x}"))
2386 .collect::<String>()
2387 )));
2388 }
2389
2390 let mut datum_reader = StrictAvroDatumReader::new(&bytes[10..]);
2391 let datum = from_avro_datum(avro_value_schema()?, &mut datum_reader, None);
2395 if datum_reader.truncated {
2396 return Err(Error::Codec(
2397 "invalid_payload_framing: truncated Avro Value datum".to_string(),
2398 ));
2399 }
2400 let datum = datum.map_err(|err| {
2401 Error::Codec(format!(
2402 "invalid_payload_framing: malformed Avro Value datum: {err}"
2403 ))
2404 })?;
2405 if datum_reader.remaining() != 0 {
2406 return Err(Error::Codec(format!(
2407 "invalid_payload_framing: {} trailing bytes after Avro Value datum",
2408 datum_reader.remaining()
2409 )));
2410 }
2411 avro_value_from_datum(datum)
2412}
2413
2414struct StrictAvroDatumReader<'a> {
2415 bytes: &'a [u8],
2416 offset: usize,
2417 truncated: bool,
2418}
2419
2420impl<'a> StrictAvroDatumReader<'a> {
2421 fn new(bytes: &'a [u8]) -> Self {
2422 Self {
2423 bytes,
2424 offset: 0,
2425 truncated: false,
2426 }
2427 }
2428
2429 fn remaining(&self) -> usize {
2430 self.bytes.len() - self.offset
2431 }
2432}
2433
2434impl Read for StrictAvroDatumReader<'_> {
2435 fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
2436 let count = buffer.len().min(self.remaining());
2437 buffer[..count].copy_from_slice(&self.bytes[self.offset..self.offset + count]);
2438 self.offset += count;
2439 if count < buffer.len() {
2440 self.truncated = true;
2441 }
2442
2443 Ok(count)
2444 }
2445}
2446
2447fn avro_value_to_datum(value: &AvroValue) -> Result<AvroDatum> {
2448 let branch = match value {
2449 AvroValue::Null => AvroDatum::Union(0, Box::new(AvroDatum::Null)),
2450 AvroValue::Boolean(value) => AvroDatum::Union(
2451 1,
2452 Box::new(AvroDatum::Record(vec![(
2453 "boolean".to_string(),
2454 AvroDatum::Boolean(*value),
2455 )])),
2456 ),
2457 AvroValue::Long(value) => AvroDatum::Union(
2458 2,
2459 Box::new(AvroDatum::Record(vec![(
2460 "long".to_string(),
2461 AvroDatum::Long(*value),
2462 )])),
2463 ),
2464 AvroValue::Double(value) => {
2465 if !value.is_finite() {
2466 return Err(Error::Codec(
2467 "non_finite_float: Avro Value doubles must be finite".to_string(),
2468 ));
2469 }
2470 AvroDatum::Union(
2471 3,
2472 Box::new(AvroDatum::Record(vec![(
2473 "double".to_string(),
2474 AvroDatum::Double(*value),
2475 )])),
2476 )
2477 }
2478 AvroValue::Bytes(value) => AvroDatum::Union(
2479 4,
2480 Box::new(AvroDatum::Record(vec![(
2481 "bytes".to_string(),
2482 AvroDatum::Bytes(value.clone()),
2483 )])),
2484 ),
2485 AvroValue::String(value) => AvroDatum::Union(
2486 5,
2487 Box::new(AvroDatum::Record(vec![(
2488 "string".to_string(),
2489 AvroDatum::String(value.clone()),
2490 )])),
2491 ),
2492 AvroValue::Array(values) => AvroDatum::Union(
2493 6,
2494 Box::new(AvroDatum::Record(vec![(
2495 "items".to_string(),
2496 AvroDatum::Array(
2497 values
2498 .iter()
2499 .map(avro_value_to_datum)
2500 .collect::<Result<Vec<_>>>()?,
2501 ),
2502 )])),
2503 ),
2504 AvroValue::Map(values) => AvroDatum::Union(
2505 7,
2506 Box::new(AvroDatum::Record(vec![(
2507 "entries".to_string(),
2508 AvroDatum::Array(
2509 values
2510 .iter()
2511 .map(|(key, value)| {
2512 Ok(AvroDatum::Record(vec![
2513 ("key".to_string(), AvroDatum::String(key.clone())),
2514 ("value".to_string(), avro_value_to_datum(value)?),
2515 ]))
2516 })
2517 .collect::<Result<Vec<_>>>()?,
2518 ),
2519 )])),
2520 ),
2521 };
2522 Ok(AvroDatum::Record(vec![("value".to_string(), branch)]))
2523}
2524
2525fn avro_value_from_datum(datum: AvroDatum) -> Result<AvroValue> {
2526 let AvroDatum::Record(mut outer) = datum else {
2527 return Err(Error::Codec(
2528 "invalid_payload_framing: datum is not a Value record".to_string(),
2529 ));
2530 };
2531 let (_, branch) = outer
2532 .pop()
2533 .filter(|(name, _)| name == "value")
2534 .ok_or_else(|| Error::Codec("invalid_payload_framing: Value field missing".to_string()))?;
2535 let AvroDatum::Union(_, branch) = branch else {
2536 return Err(Error::Codec(
2537 "invalid_payload_framing: invalid Value union".to_string(),
2538 ));
2539 };
2540 match *branch {
2541 AvroDatum::Null => Ok(AvroValue::Null),
2542 AvroDatum::Record(mut fields) => {
2543 let (name, value) = fields.pop().ok_or_else(|| {
2544 Error::Codec("invalid_payload_framing: empty Value branch".to_string())
2545 })?;
2546 match (name.as_str(), value) {
2547 ("boolean", AvroDatum::Boolean(value)) => Ok(AvroValue::Boolean(value)),
2548 ("long", AvroDatum::Long(value)) => Ok(AvroValue::Long(value)),
2549 ("double", AvroDatum::Double(value)) if value.is_finite() => {
2550 Ok(AvroValue::Double(value))
2551 }
2552 ("bytes", AvroDatum::Bytes(value)) => Ok(AvroValue::Bytes(value)),
2553 ("string", AvroDatum::String(value)) => Ok(AvroValue::String(value)),
2554 ("items", AvroDatum::Array(values)) => values
2555 .into_iter()
2556 .map(avro_value_from_datum)
2557 .collect::<Result<Vec<_>>>()
2558 .map(AvroValue::Array),
2559 ("entries", AvroDatum::Map(values)) => values
2560 .into_iter()
2561 .map(|(key, value)| Ok((key, avro_value_from_datum(value)?)))
2562 .collect::<Result<BTreeMap<_, _>>>()
2563 .map(AvroValue::Map),
2564 _ => Err(Error::Codec(
2565 "invalid_payload_framing: unknown Value branch".to_string(),
2566 )),
2567 }
2568 }
2569 _ => Err(Error::Codec(
2570 "invalid_payload_framing: invalid Value branch".to_string(),
2571 )),
2572 }
2573}
2574
2575fn avro_value_schema() -> Result<&'static Schema> {
2576 match AVRO_VALUE_SCHEMA.get_or_init(|| {
2577 Schema::parse_str(AVRO_VALUE_SCHEMA_JSON)
2578 .map_err(|err| format!("could not parse Avro Value schema: {err}"))
2579 }) {
2580 Ok(schema) => Ok(schema),
2581 Err(message) => Err(Error::Codec(message.clone())),
2582 }
2583}
2584
2585fn avro_value_ordered_map_encoding_schema() -> Result<&'static Schema> {
2586 match AVRO_VALUE_ORDERED_MAP_ENCODING_SCHEMA.get_or_init(|| {
2587 let mut schema: Value = serde_json::from_str(AVRO_VALUE_SCHEMA_JSON)
2592 .map_err(|err| format!("could not read packaged Avro Value schema: {err}"))?;
2593 let entries_schema = schema
2594 .pointer_mut("/fields/0/type/7/fields/0/type")
2595 .ok_or_else(|| "packaged Avro Value map schema is missing".to_string())?;
2596 if *entries_schema != json!({"type": "map", "values": "Value"}) {
2597 return Err("packaged Avro Value map schema changed unexpectedly".to_string());
2598 }
2599 *entries_schema = json!({
2600 "type": "array",
2601 "items": {
2602 "type": "record",
2603 "name": "MapEntry",
2604 "fields": [
2605 {"name": "key", "type": "string"},
2606 {"name": "value", "type": "Value"}
2607 ]
2608 }
2609 });
2610 Schema::parse_str(&schema.to_string())
2611 .map_err(|err| format!("could not parse ordered-map Avro Value schema: {err}"))
2612 }) {
2613 Ok(schema) => Ok(schema),
2614 Err(message) => Err(Error::Codec(message.clone())),
2615 }
2616}
2617
2618#[derive(Clone, Debug)]
2619pub struct Client {
2620 http: reqwest::Client,
2621 base_url: String,
2622 token: Option<String>,
2623 control_token: Option<String>,
2624 worker_token: Option<String>,
2625 namespace: String,
2626 max_external_payload_bytes: usize,
2627 worker_storage_admission: Option<WorkerStorageAdmission>,
2628 runtime_upload_policy: Arc<Mutex<runtime_uploads::PolicyCache>>,
2629}
2630
2631impl Client {
2632 pub fn new(base_url: impl Into<String>) -> Result<Self> {
2633 Self::builder(base_url).build()
2634 }
2635
2636 pub fn builder(base_url: impl Into<String>) -> ClientBuilder {
2637 ClientBuilder {
2638 base_url: base_url.into(),
2639 token: None,
2640 control_token: None,
2641 worker_token: None,
2642 namespace: "default".to_string(),
2643 timeout: Duration::from_secs(60),
2644 max_external_payload_bytes: 64 * 1024 * 1024,
2645 }
2646 }
2647
2648 pub async fn health(&self) -> Result<Value> {
2649 self.request_json(
2650 reqwest::Method::GET,
2651 "/health",
2652 RequestProtocol::ControlPlane,
2653 Option::<&Value>::None,
2654 )
2655 .await
2656 }
2657
2658 pub async fn cluster_info(&self) -> Result<Value> {
2659 self.request_json(
2660 reqwest::Method::GET,
2661 "/cluster/info",
2662 RequestProtocol::ControlPlane,
2663 Option::<&Value>::None,
2664 )
2665 .await
2666 }
2667
2668 pub async fn start_workflow<T: Serialize>(
2669 &self,
2670 workflow_type: &str,
2671 task_queue: &str,
2672 workflow_id: &str,
2673 input: T,
2674 ) -> Result<WorkflowHandle> {
2675 self.start_workflow_with_options(
2676 workflow_type,
2677 task_queue,
2678 workflow_id,
2679 WorkflowStartOptions::default(),
2680 input,
2681 )
2682 .await
2683 }
2684
2685 pub async fn start_workflow_with_options<T: Serialize>(
2688 &self,
2689 workflow_type: &str,
2690 task_queue: &str,
2691 workflow_id: &str,
2692 options: WorkflowStartOptions,
2693 input: T,
2694 ) -> Result<WorkflowHandle> {
2695 options.validate()?;
2696 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2697 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2698 let body = json!({
2699 "workflow_id": workflow_id,
2700 "workflow_type": workflow_type,
2701 "task_queue": task_queue,
2702 "input": input_envelope,
2703 "execution_timeout_seconds": options.execution_timeout_seconds,
2704 "run_timeout_seconds": options.run_timeout_seconds
2705 });
2706
2707 let data: Value = self
2708 .request_json(
2709 reqwest::Method::POST,
2710 "/workflows",
2711 RequestProtocol::ControlPlane,
2712 Some(&body),
2713 )
2714 .await?;
2715
2716 Ok(WorkflowHandle {
2717 client: self.clone(),
2718 workflow_id: data
2719 .get("workflow_id")
2720 .and_then(Value::as_str)
2721 .unwrap_or(workflow_id)
2722 .to_string(),
2723 run_id: data
2724 .get("run_id")
2725 .and_then(Value::as_str)
2726 .map(str::to_string),
2727 workflow_type: data
2728 .get("workflow_type")
2729 .and_then(Value::as_str)
2730 .unwrap_or(workflow_type)
2731 .to_string(),
2732 })
2733 }
2734
2735 pub async fn signal_workflow<T: Serialize>(
2736 &self,
2737 workflow_id: &str,
2738 signal_name: &str,
2739 input: T,
2740 ) -> Result<Value> {
2741 self.signal_workflow_target(workflow_id, None, signal_name, input)
2742 .await
2743 }
2744
2745 pub async fn append_message_stream<T: Serialize>(
2747 &self,
2748 workflow_id: &str,
2749 stream_name: &str,
2750 message_id: &str,
2751 input: T,
2752 ) -> Result<Value> {
2753 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2754 let body = json!({
2755 "message_id": message_id,
2756 "input": encode_typed_envelope(&input, DEFAULT_CODEC)?
2757 });
2758 self.request_json(
2759 reqwest::Method::POST,
2760 &format!("/workflows/{workflow_id}/message-streams/{stream_name}/messages"),
2761 RequestProtocol::ControlPlane,
2762 Some(&body),
2763 )
2764 .await
2765 }
2766
2767 pub async fn signal_workflow_run<T: Serialize>(
2769 &self,
2770 workflow_id: &str,
2771 run_id: &str,
2772 signal_name: &str,
2773 input: T,
2774 ) -> Result<Value> {
2775 self.signal_workflow_target(workflow_id, Some(run_id), signal_name, input)
2776 .await
2777 }
2778
2779 async fn signal_workflow_target<T: Serialize>(
2780 &self,
2781 workflow_id: &str,
2782 run_id: Option<&str>,
2783 signal_name: &str,
2784 input: T,
2785 ) -> Result<Value> {
2786 validate_user_signal_name(signal_name)?;
2787 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2788 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2789 let body = json!({
2790 "input": input_envelope
2791 });
2792 let path = match run_id {
2793 Some(run_id) => {
2794 format!("/workflows/{workflow_id}/runs/{run_id}/signal/{signal_name}")
2795 }
2796 None => format!("/workflows/{workflow_id}/signal/{signal_name}"),
2797 };
2798 self.request_json(
2799 reqwest::Method::POST,
2800 &path,
2801 RequestProtocol::ControlPlane,
2802 Some(&body),
2803 )
2804 .await
2805 }
2806
2807 pub async fn cancel_workflow(
2809 &self,
2810 workflow_id: &str,
2811 options: WorkflowCommandOptions,
2812 ) -> Result<WorkflowCommandResult> {
2813 self.workflow_command(workflow_id, None, WorkflowCommandKind::Cancel, options)
2814 .await
2815 }
2816
2817 pub async fn cancel_workflow_run(
2819 &self,
2820 workflow_id: &str,
2821 run_id: &str,
2822 options: WorkflowCommandOptions,
2823 ) -> Result<WorkflowCommandResult> {
2824 self.workflow_command(
2825 workflow_id,
2826 Some(run_id),
2827 WorkflowCommandKind::Cancel,
2828 options,
2829 )
2830 .await
2831 }
2832
2833 pub async fn terminate_workflow(
2835 &self,
2836 workflow_id: &str,
2837 options: WorkflowCommandOptions,
2838 ) -> Result<WorkflowCommandResult> {
2839 self.workflow_command(workflow_id, None, WorkflowCommandKind::Terminate, options)
2840 .await
2841 }
2842
2843 pub async fn terminate_workflow_run(
2845 &self,
2846 workflow_id: &str,
2847 run_id: &str,
2848 options: WorkflowCommandOptions,
2849 ) -> Result<WorkflowCommandResult> {
2850 self.workflow_command(
2851 workflow_id,
2852 Some(run_id),
2853 WorkflowCommandKind::Terminate,
2854 options,
2855 )
2856 .await
2857 }
2858
2859 async fn workflow_command(
2860 &self,
2861 workflow_id: &str,
2862 run_id: Option<&str>,
2863 command: WorkflowCommandKind,
2864 options: WorkflowCommandOptions,
2865 ) -> Result<WorkflowCommandResult> {
2866 let path = match run_id {
2867 Some(run_id) => format!(
2868 "/workflows/{workflow_id}/runs/{run_id}/{}",
2869 command.as_str()
2870 ),
2871 None => format!("/workflows/{workflow_id}/{}", command.as_str()),
2872 };
2873 let data = match self
2874 .request_json(
2875 reqwest::Method::POST,
2876 &path,
2877 RequestProtocol::ControlPlane,
2878 Some(&options),
2879 )
2880 .await
2881 {
2882 Ok(data) => data,
2883 Err(Error::Http { status, body }) => {
2884 return Err(Error::WorkflowCommandRejected(workflow_command_rejection(
2885 command,
2886 status,
2887 body,
2888 workflow_id,
2889 run_id,
2890 )));
2891 }
2892 Err(error) => return Err(error),
2893 };
2894
2895 Ok(workflow_command_result(command, data, workflow_id, run_id))
2896 }
2897
2898 pub async fn query_workflow<T: Serialize>(
2904 &self,
2905 workflow_id: &str,
2906 query_name: &str,
2907 input: T,
2908 ) -> Result<Value> {
2909 self.query_workflow_target(workflow_id, None, query_name, input)
2910 .await
2911 }
2912
2913 pub async fn query_workflow_run<T: Serialize>(
2915 &self,
2916 workflow_id: &str,
2917 run_id: &str,
2918 query_name: &str,
2919 input: T,
2920 ) -> Result<Value> {
2921 self.query_workflow_target(workflow_id, Some(run_id), query_name, input)
2922 .await
2923 }
2924
2925 pub async fn query_workflow_avro_value<T: Serialize>(
2927 &self,
2928 workflow_id: &str,
2929 query_name: &str,
2930 input: T,
2931 ) -> Result<AvroValue> {
2932 self.query_workflow_avro_value_target(workflow_id, None, query_name, input)
2933 .await
2934 }
2935
2936 pub async fn query_workflow_run_avro_value<T: Serialize>(
2938 &self,
2939 workflow_id: &str,
2940 run_id: &str,
2941 query_name: &str,
2942 input: T,
2943 ) -> Result<AvroValue> {
2944 self.query_workflow_avro_value_target(workflow_id, Some(run_id), query_name, input)
2945 .await
2946 }
2947
2948 async fn query_workflow_avro_value_target<T: Serialize>(
2949 &self,
2950 workflow_id: &str,
2951 run_id: Option<&str>,
2952 query_name: &str,
2953 input: T,
2954 ) -> Result<AvroValue> {
2955 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2956 let body = json!({"input": encode_typed_envelope(&input, DEFAULT_CODEC)?});
2957 let path = match run_id {
2958 Some(run_id) => {
2959 format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
2960 }
2961 None => format!("/workflows/{workflow_id}/query/{query_name}"),
2962 };
2963 let response: Value = match self
2964 .request_json(
2965 reqwest::Method::POST,
2966 &path,
2967 RequestProtocol::ControlPlane,
2968 Some(&body),
2969 )
2970 .await
2971 {
2972 Ok(response) => response,
2973 Err(Error::Http { status, body }) => {
2974 return Err(Error::QueryFailed(query_failure(status, body)));
2975 }
2976 Err(error) => return Err(error),
2977 };
2978
2979 let envelope = response
2980 .get("result_envelope")
2981 .filter(|envelope| !envelope.is_null())
2982 .ok_or_else(|| {
2983 Error::Codec(
2984 "missing_payload_envelope: typed query result requires result_envelope"
2985 .to_string(),
2986 )
2987 })?;
2988 decode_wire_avro_value(envelope, DEFAULT_CODEC)
2989 }
2990
2991 async fn query_workflow_target<T: Serialize>(
2992 &self,
2993 workflow_id: &str,
2994 run_id: Option<&str>,
2995 query_name: &str,
2996 input: T,
2997 ) -> Result<Value> {
2998 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2999 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
3000 let body = json!({
3001 "input": input_envelope
3002 });
3003 let path = match run_id {
3004 Some(run_id) => {
3005 format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
3006 }
3007 None => format!("/workflows/{workflow_id}/query/{query_name}"),
3008 };
3009 let response: Value = match self
3010 .request_json(
3011 reqwest::Method::POST,
3012 &path,
3013 RequestProtocol::ControlPlane,
3014 Some(&body),
3015 )
3016 .await
3017 {
3018 Ok(response) => response,
3019 Err(Error::Http { status, body }) => {
3020 return Err(Error::QueryFailed(query_failure(status, body)));
3021 }
3022 Err(error) => return Err(error),
3023 };
3024
3025 if let Some(envelope) = response
3026 .get("result_envelope")
3027 .filter(|envelope| !envelope.is_null())
3028 {
3029 return decode_wire_value(envelope, DEFAULT_CODEC);
3030 }
3031
3032 Ok(response.get("result").cloned().unwrap_or(Value::Null))
3033 }
3034
3035 pub async fn update_workflow<T: Serialize>(
3037 &self,
3038 workflow_id: &str,
3039 update_name: &str,
3040 input: T,
3041 request_id: Option<&str>,
3042 ) -> Result<Value> {
3043 let response = self
3044 .update_workflow_response(workflow_id, update_name, input, request_id)
3045 .await?;
3046 if let Some(envelope) = response
3047 .get("result_envelope")
3048 .filter(|envelope| !envelope.is_null())
3049 {
3050 return decode_wire_value(envelope, DEFAULT_CODEC);
3051 }
3052 Ok(response.get("result").cloned().unwrap_or(response))
3053 }
3054
3055 pub async fn update_workflow_avro_value<T: Serialize>(
3057 &self,
3058 workflow_id: &str,
3059 update_name: &str,
3060 input: T,
3061 request_id: Option<&str>,
3062 ) -> Result<AvroValue> {
3063 let response = self
3064 .update_workflow_response(workflow_id, update_name, input, request_id)
3065 .await?;
3066 let envelope = response
3067 .get("result_envelope")
3068 .filter(|envelope| !envelope.is_null())
3069 .ok_or_else(|| {
3070 Error::Codec(
3071 "missing_payload_envelope: typed update result requires result_envelope"
3072 .to_string(),
3073 )
3074 })?;
3075 decode_wire_avro_value(envelope, DEFAULT_CODEC)
3076 }
3077
3078 async fn update_workflow_response<T: Serialize>(
3079 &self,
3080 workflow_id: &str,
3081 update_name: &str,
3082 input: T,
3083 request_id: Option<&str>,
3084 ) -> Result<Value> {
3085 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
3086 let mut body = json!({
3087 "input": encode_typed_envelope(&input, DEFAULT_CODEC)?,
3088 "wait_for": "completed",
3089 });
3090 if let Some(request_id) = request_id {
3091 body["request_id"] = json!(request_id);
3092 }
3093 self.request_json(
3094 reqwest::Method::POST,
3095 &format!("/workflows/{workflow_id}/update/{update_name}"),
3096 RequestProtocol::ControlPlane,
3097 Some(&body),
3098 )
3099 .await
3100 }
3101
3102 pub async fn describe_workflow(&self, workflow_id: &str) -> Result<WorkflowDescription> {
3103 let path = format!("/workflows/{workflow_id}");
3104 let mut data: WorkflowDescription = self
3105 .request_json(
3106 reqwest::Method::GET,
3107 &path,
3108 RequestProtocol::ControlPlane,
3109 Option::<&Value>::None,
3110 )
3111 .await?;
3112 data.decode_payloads()?;
3113 Ok(data)
3114 }
3115
3116 pub async fn describe_workflow_run(
3118 &self,
3119 workflow_id: &str,
3120 run_id: &str,
3121 ) -> Result<WorkflowDescription> {
3122 let path = format!("/workflows/{workflow_id}/runs/{run_id}");
3123 let mut data: WorkflowDescription = self
3124 .request_json(
3125 reqwest::Method::GET,
3126 &path,
3127 RequestProtocol::ControlPlane,
3128 Option::<&Value>::None,
3129 )
3130 .await?;
3131 data.decode_payloads()?;
3132 Ok(data)
3133 }
3134
3135 fn workflow_stream_path(workflow_id: &str, run_id: &str, stream_name: Option<&str>) -> String {
3136 let mut path = format!(
3137 "/workflows/{}/runs/{}/streams",
3138 percent_encode_path_segment(workflow_id),
3139 percent_encode_path_segment(run_id),
3140 );
3141 if let Some(stream_name) = stream_name {
3142 path.push('/');
3143 path.push_str(&percent_encode_path_segment(stream_name));
3144 }
3145 path
3146 }
3147
3148 pub async fn list_workflow_streams(
3150 &self,
3151 workflow_id: &str,
3152 run_id: &str,
3153 ) -> Result<Vec<WorkflowStreamDescription>> {
3154 let response: WorkflowStreamListResponse = self
3155 .request_json(
3156 reqwest::Method::GET,
3157 &Self::workflow_stream_path(workflow_id, run_id, None),
3158 RequestProtocol::ControlPlane,
3159 Option::<&Value>::None,
3160 )
3161 .await?;
3162 Ok(response.streams)
3163 }
3164
3165 pub async fn describe_workflow_stream(
3167 &self,
3168 workflow_id: &str,
3169 run_id: &str,
3170 stream_name: &str,
3171 ) -> Result<WorkflowStreamDescription> {
3172 let response: WorkflowStreamDescriptionResponse = self
3173 .request_json(
3174 reqwest::Method::GET,
3175 &Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3176 RequestProtocol::ControlPlane,
3177 Option::<&Value>::None,
3178 )
3179 .await?;
3180 Ok(response.stream)
3181 }
3182
3183 pub async fn subscribe_workflow_stream(
3190 &self,
3191 workflow_id: &str,
3192 run_id: &str,
3193 stream_name: &str,
3194 from_offset: u64,
3195 max_items: usize,
3196 wait: Duration,
3197 ) -> Result<WorkflowStreamPage> {
3198 let max_items = max_items.clamp(1, 500);
3199 let wait_seconds = wait.as_secs().min(MAX_LONG_POLL_TIMEOUT_SECONDS);
3200 let path = format!(
3201 "{}/items?from={from_offset}&max_items={max_items}&wait_seconds={wait_seconds}",
3202 Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3203 );
3204 let response: WorkflowStreamPageResponse = self
3205 .request_json_with_timeout(
3206 reqwest::Method::GET,
3207 &path,
3208 RequestProtocol::ControlPlane,
3209 Option::<&Value>::None,
3210 Duration::from_secs(wait_seconds.saturating_add(5).max(5)),
3211 )
3212 .await?;
3213
3214 let items = response
3215 .items
3216 .into_iter()
3217 .map(|raw| {
3218 let offset = raw.get("offset").and_then(Value::as_u64).unwrap_or(0);
3219 let envelope = raw.get("payload").cloned();
3220 let payload = envelope
3221 .as_ref()
3222 .filter(|value| value.get("blob").is_some())
3223 .map(|value| decode_wire_avro_value(value, DEFAULT_CODEC))
3224 .transpose()?
3225 .map(AvroValue::into_json)
3226 .transpose()?;
3227 Ok(WorkflowStreamItem {
3228 offset,
3229 payload,
3230 payload_envelope: envelope,
3231 payload_reference: raw
3232 .get("payload_reference")
3233 .and_then(Value::as_str)
3234 .map(str::to_string),
3235 payload_codec: raw
3236 .get("payload_codec")
3237 .and_then(Value::as_str)
3238 .map(str::to_string),
3239 idempotency_key: raw
3240 .get("idempotency_key")
3241 .and_then(Value::as_str)
3242 .map(str::to_string),
3243 item_type: raw
3244 .get("item_type")
3245 .and_then(Value::as_str)
3246 .map(str::to_string),
3247 content_type: raw
3248 .get("content_type")
3249 .and_then(Value::as_str)
3250 .map(str::to_string),
3251 origin: raw
3252 .get("origin")
3253 .and_then(Value::as_str)
3254 .map(str::to_string),
3255 origin_reference: raw
3256 .get("origin_reference")
3257 .and_then(Value::as_str)
3258 .map(str::to_string),
3259 emitted_at: raw
3260 .get("emitted_at")
3261 .and_then(Value::as_str)
3262 .map(str::to_string),
3263 raw,
3264 })
3265 })
3266 .collect::<Result<Vec<_>>>()?;
3267 Ok(WorkflowStreamPage {
3268 stream: response.stream,
3269 items,
3270 next_offset: response.next_offset,
3271 terminal: response.terminal,
3272 })
3273 }
3274
3275 pub async fn append_workflow_stream(
3277 &self,
3278 workflow_id: &str,
3279 run_id: &str,
3280 stream_name: &str,
3281 items: &[WorkflowStreamAppendItem],
3282 max_pending_items: Option<u64>,
3283 ) -> Result<WorkflowStreamAppendResult> {
3284 if items.is_empty() {
3285 return Err(Error::Codec(
3286 "workflow_stream_items_empty: append requires at least one item".to_string(),
3287 ));
3288 }
3289 let mut body = json!({
3290 "items": items
3291 .iter()
3292 .map(|item| item.wire_value(None))
3293 .collect::<Vec<_>>(),
3294 });
3295 if let Some(max_pending_items) = max_pending_items {
3296 if max_pending_items == 0 {
3297 return Err(Error::Codec(
3298 "workflow_stream_pending_limit_invalid: max_pending_items must be positive"
3299 .to_string(),
3300 ));
3301 }
3302 body["max_pending_items"] = json!(max_pending_items);
3303 }
3304 let response: WorkflowStreamAppendResponse = self
3305 .request_json(
3306 reqwest::Method::POST,
3307 &format!(
3308 "{}/items",
3309 Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3310 ),
3311 RequestProtocol::ControlPlane,
3312 Some(&body),
3313 )
3314 .await?;
3315 Ok(WorkflowStreamAppendResult {
3316 stream: response.stream,
3317 accepted_offsets: response.accepted_offsets,
3318 accepted: response.accepted,
3319 deduped: response.deduped,
3320 })
3321 }
3322
3323 pub async fn close_workflow_stream(
3325 &self,
3326 workflow_id: &str,
3327 run_id: &str,
3328 stream_name: &str,
3329 error_reason: Option<&str>,
3330 retention_seconds: Option<u64>,
3331 ) -> Result<WorkflowStreamDescription> {
3332 let mut body = json!({});
3333 if let Some(error_reason) = error_reason {
3334 body["error_reason"] = json!(error_reason);
3335 }
3336 if let Some(retention_seconds) = retention_seconds {
3337 if retention_seconds == 0 {
3338 return Err(Error::Codec(
3339 "workflow_stream_retention_invalid: retention_seconds must be positive"
3340 .to_string(),
3341 ));
3342 }
3343 body["retention_seconds"] = json!(retention_seconds);
3344 }
3345 let response: WorkflowStreamDescriptionResponse = self
3346 .request_json(
3347 reqwest::Method::POST,
3348 &format!(
3349 "{}/close",
3350 Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3351 ),
3352 RequestProtocol::ControlPlane,
3353 Some(&body),
3354 )
3355 .await?;
3356 Ok(response.stream)
3357 }
3358
3359 pub async fn register_worker(
3360 &self,
3361 worker_id: &str,
3362 task_queue: &str,
3363 supported_workflow_types: Vec<String>,
3364 supported_activity_types: Vec<String>,
3365 max_concurrent_workflow_tasks: usize,
3366 max_concurrent_activity_tasks: usize,
3367 ) -> Result<RegisterWorkerResponse> {
3368 self.register_worker_with_capabilities(
3369 worker_id,
3370 task_queue,
3371 supported_workflow_types,
3372 supported_activity_types,
3373 max_concurrent_workflow_tasks,
3374 max_concurrent_activity_tasks,
3375 Vec::new(),
3376 )
3377 .await
3378 }
3379
3380 pub async fn register_worker_with_capabilities(
3382 &self,
3383 worker_id: &str,
3384 task_queue: &str,
3385 supported_workflow_types: Vec<String>,
3386 supported_activity_types: Vec<String>,
3387 max_concurrent_workflow_tasks: usize,
3388 max_concurrent_activity_tasks: usize,
3389 capabilities: Vec<String>,
3390 ) -> Result<RegisterWorkerResponse> {
3391 self.register_worker_with_command_contracts(
3392 worker_id,
3393 task_queue,
3394 supported_workflow_types,
3395 supported_activity_types,
3396 max_concurrent_workflow_tasks,
3397 max_concurrent_activity_tasks,
3398 capabilities,
3399 Value::Object(serde_json::Map::new()),
3400 )
3401 .await
3402 }
3403
3404 #[allow(clippy::too_many_arguments)]
3411 pub async fn register_worker_with_command_contracts(
3412 &self,
3413 worker_id: &str,
3414 task_queue: &str,
3415 supported_workflow_types: Vec<String>,
3416 supported_activity_types: Vec<String>,
3417 max_concurrent_workflow_tasks: usize,
3418 max_concurrent_activity_tasks: usize,
3419 capabilities: Vec<String>,
3420 workflow_command_contracts: Value,
3421 ) -> Result<RegisterWorkerResponse> {
3422 if let Some(contracts) = workflow_command_contracts.as_object() {
3423 for (workflow_type, contract) in contracts {
3424 let Some(update_validators) = contract.get("update_validators") else {
3425 continue;
3426 };
3427 if !update_validators
3428 .as_array()
3429 .is_some_and(|validators| validators.is_empty())
3430 {
3431 return Err(Error::UnsupportedUpdateValidators {
3432 workflow_type: workflow_type.clone(),
3433 });
3434 }
3435 }
3436 }
3437
3438 let mut body = json!({
3439 "worker_id": worker_id,
3440 "task_queue": task_queue,
3441 "runtime": "rust",
3442 "sdk_version": SDK_VERSION,
3443 "supported_workflow_types": supported_workflow_types,
3444 "supported_activity_types": supported_activity_types,
3445 "capabilities": capabilities,
3446 "capability_manifest": portable_worker_affinity_capability_manifest(),
3447 "max_concurrent_workflow_tasks": max_concurrent_workflow_tasks,
3448 "max_concurrent_activity_tasks": max_concurrent_activity_tasks
3449 });
3450 if workflow_command_contracts
3451 .as_object()
3452 .is_some_and(|contracts| !contracts.is_empty())
3453 {
3454 body["workflow_command_contracts"] = workflow_command_contracts;
3455 }
3456
3457 self.request_json(
3458 reqwest::Method::POST,
3459 "/worker/register",
3460 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3461 Some(&body),
3462 )
3463 .await
3464 }
3465
3466 pub async fn deregister_worker_registration(
3472 &self,
3473 worker_id: &str,
3474 ) -> Result<WorkerDeregistrationEnvelope> {
3475 let path = format!(
3476 "/worker/registrations/{}",
3477 percent_encode_path_segment(worker_id)
3478 );
3479 self.request_json(
3480 reqwest::Method::DELETE,
3481 &path,
3482 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3483 Option::<&Value>::None,
3484 )
3485 .await
3486 }
3487
3488 pub async fn poll_query_task(
3490 &self,
3491 worker_id: &str,
3492 task_queue: &str,
3493 timeout: Duration,
3494 ) -> Result<Option<QueryTask>> {
3495 Ok(self
3496 .poll_query_task_response(worker_id, task_queue, timeout)
3497 .await?
3498 .task)
3499 }
3500
3501 pub async fn poll_query_task_response(
3503 &self,
3504 worker_id: &str,
3505 task_queue: &str,
3506 timeout: Duration,
3507 ) -> Result<PollQueryTaskResponse> {
3508 let poll_request_id = unique_request_id("rust-query-poll");
3509 self.poll_query_task_response_with_request_id(
3510 worker_id,
3511 task_queue,
3512 timeout,
3513 &poll_request_id,
3514 1,
3515 )
3516 .await
3517 }
3518
3519 async fn poll_query_task_response_with_request_id(
3520 &self,
3521 worker_id: &str,
3522 task_queue: &str,
3523 timeout: Duration,
3524 poll_request_id: &str,
3525 transport_retries: usize,
3526 ) -> Result<PollQueryTaskResponse> {
3527 let timeout_seconds = long_poll_timeout_seconds(timeout);
3528 let body = json!({
3529 "worker_id": worker_id,
3530 "task_queue": task_queue,
3531 "poll_request_id": poll_request_id,
3532 "timeout_seconds": timeout_seconds,
3533 });
3534 self.poll_request_json(
3535 "/worker/query-tasks/poll",
3536 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3537 &body,
3538 timeout + Duration::from_secs(5),
3539 transport_retries,
3540 )
3541 .await
3542 }
3543
3544 pub async fn complete_query_task<T: Serialize>(
3546 &self,
3547 query_task_id: &str,
3548 lease_owner: &str,
3549 query_task_attempt: u64,
3550 result: T,
3551 codec: &str,
3552 ) -> Result<Value> {
3553 let typed_result = AvroValue::from_serialize(&result)?;
3554 let result_envelope = encode_typed_envelope(&typed_result, codec)?;
3555 self.complete_query_task_with_envelope(
3556 query_task_id,
3557 lease_owner,
3558 query_task_attempt,
3559 typed_result.into_json()?,
3560 result_envelope,
3561 )
3562 .await
3563 }
3564
3565 async fn complete_query_task_with_envelope(
3566 &self,
3567 query_task_id: &str,
3568 lease_owner: &str,
3569 query_task_attempt: u64,
3570 result: Value,
3571 result_envelope: Value,
3572 ) -> Result<Value> {
3573 let body = json!({
3574 "lease_owner": lease_owner,
3575 "query_task_attempt": query_task_attempt,
3576 "result": result,
3577 "result_envelope": result_envelope,
3578 });
3579 let path = format!("/worker/query-tasks/{query_task_id}/complete");
3580 let response = self
3581 .request_json(
3582 reqwest::Method::POST,
3583 &path,
3584 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3585 Some(&body),
3586 )
3587 .await;
3588 query_task_response(response)
3589 }
3590
3591 pub async fn fail_query_task(
3593 &self,
3594 query_task_id: &str,
3595 lease_owner: &str,
3596 query_task_attempt: u64,
3597 message: impl Into<String>,
3598 reason: impl Into<String>,
3599 failure_type: impl Into<String>,
3600 ) -> Result<Value> {
3601 let body = json!({
3602 "lease_owner": lease_owner,
3603 "query_task_attempt": query_task_attempt,
3604 "failure": {
3605 "message": message.into(),
3606 "reason": reason.into(),
3607 "type": failure_type.into(),
3608 }
3609 });
3610 let path = format!("/worker/query-tasks/{query_task_id}/fail");
3611 let response = self
3612 .request_json(
3613 reqwest::Method::POST,
3614 &path,
3615 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3616 Some(&body),
3617 )
3618 .await;
3619 query_task_response(response)
3620 }
3621
3622 pub async fn heartbeat_worker(
3623 &self,
3624 worker_id: &str,
3625 workflow_available: usize,
3626 activity_available: usize,
3627 ) -> Result<Value> {
3628 let body = json!({
3629 "worker_id": worker_id,
3630 "task_slots": {
3631 "workflow_available": workflow_available,
3632 "activity_available": activity_available
3633 },
3634 "process_metrics": {
3635 "process_id": std::process::id(),
3636 "process_uptime_seconds": 0
3637 }
3638 });
3639
3640 self.request_json(
3641 reqwest::Method::POST,
3642 "/worker/heartbeat",
3643 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3644 Some(&body),
3645 )
3646 .await
3647 }
3648
3649 pub async fn poll_workflow_task(
3650 &self,
3651 worker_id: &str,
3652 task_queue: &str,
3653 timeout: Duration,
3654 ) -> Result<Option<WorkflowTask>> {
3655 Ok(self
3656 .poll_workflow_task_response(worker_id, task_queue, timeout)
3657 .await?
3658 .task)
3659 }
3660
3661 pub async fn poll_workflow_task_response(
3662 &self,
3663 worker_id: &str,
3664 task_queue: &str,
3665 timeout: Duration,
3666 ) -> Result<PollWorkflowTaskResponse> {
3667 let poll_request_id = unique_request_id("rust-workflow-poll");
3668 self.poll_workflow_task_response_with_request_id(
3669 worker_id,
3670 task_queue,
3671 timeout,
3672 &poll_request_id,
3673 1,
3674 )
3675 .await
3676 }
3677
3678 async fn poll_workflow_task_response_with_request_id(
3679 &self,
3680 worker_id: &str,
3681 task_queue: &str,
3682 timeout: Duration,
3683 poll_request_id: &str,
3684 transport_retries: usize,
3685 ) -> Result<PollWorkflowTaskResponse> {
3686 let body = json!({
3687 "worker_id": worker_id,
3688 "task_queue": task_queue,
3689 "poll_request_id": poll_request_id,
3690 "timeout_seconds": long_poll_timeout_seconds(timeout),
3691 });
3692 let mut data: PollWorkflowTaskResponse = self
3693 .poll_request_json(
3694 "/worker/workflow-tasks/poll",
3695 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3696 &body,
3697 timeout + Duration::from_secs(5),
3698 transport_retries,
3699 )
3700 .await?;
3701
3702 if let Some(task) = data.task.as_mut() {
3703 self.fetch_remaining_workflow_history(worker_id, task)
3704 .await?;
3705 }
3706
3707 Ok(data)
3708 }
3709
3710 async fn fetch_remaining_workflow_history(
3711 &self,
3712 worker_id: &str,
3713 task: &mut WorkflowTask,
3714 ) -> Result<()> {
3715 let mut next_token = task.next_history_page_token.clone();
3716
3717 while let Some(token) = next_token.take().filter(|token| !token.is_empty()) {
3718 let lease_owner = task
3719 .lease_owner
3720 .clone()
3721 .unwrap_or_else(|| worker_id.to_string());
3722 let page = self
3723 .workflow_task_history_page(
3724 &task.task_id,
3725 &lease_owner,
3726 task.workflow_task_attempt,
3727 &token,
3728 )
3729 .await?;
3730
3731 task.append_history_page(page);
3732
3733 if task.next_history_page_token.as_deref() == Some(token.as_str()) {
3734 return Err(Error::Codec(
3735 "workflow history pagination returned the same page token".to_string(),
3736 ));
3737 }
3738
3739 next_token = task.next_history_page_token.clone();
3740 }
3741
3742 Ok(())
3743 }
3744
3745 async fn workflow_task_history_page(
3746 &self,
3747 task_id: &str,
3748 lease_owner: &str,
3749 workflow_task_attempt: u64,
3750 next_history_page_token: &str,
3751 ) -> Result<WorkflowTaskHistoryPage> {
3752 let body = json!({
3753 "lease_owner": lease_owner,
3754 "workflow_task_attempt": workflow_task_attempt,
3755 "next_history_page_token": next_history_page_token
3756 });
3757 let path = format!("/worker/workflow-tasks/{task_id}/history");
3758
3759 self.request_json(
3760 reqwest::Method::POST,
3761 &path,
3762 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3763 Some(&body),
3764 )
3765 .await
3766 }
3767
3768 pub async fn complete_workflow_task(
3769 &self,
3770 task_id: &str,
3771 lease_owner: &str,
3772 workflow_task_attempt: u64,
3773 commands: Vec<Value>,
3774 ) -> Result<Value> {
3775 self.complete_workflow_task_with_message_streams(
3776 task_id,
3777 lease_owner,
3778 workflow_task_attempt,
3779 commands,
3780 Vec::new(),
3781 Vec::new(),
3782 )
3783 .await
3784 }
3785
3786 async fn complete_workflow_task_with_message_streams(
3787 &self,
3788 task_id: &str,
3789 lease_owner: &str,
3790 workflow_task_attempt: u64,
3791 commands: Vec<Value>,
3792 message_stream_cursors: Vec<Value>,
3793 message_stream_waits: Vec<Value>,
3794 ) -> Result<Value> {
3795 validate_workflow_task_commands(&commands)?;
3796 let has_message_stream_metadata =
3797 !message_stream_cursors.is_empty() || !message_stream_waits.is_empty();
3798 if has_message_stream_metadata
3799 && !worker_protocol_supports_message_streams(WORKER_PROTOCOL_VERSION)
3800 {
3801 return Err(Error::Codec(
3802 "message_streams_unavailable: message stream completion metadata requires worker protocol 1.15 or newer"
3803 .to_string(),
3804 ));
3805 }
3806 let protocol_version = workflow_completion_protocol_version_with_message_streams(
3807 &commands,
3808 has_message_stream_metadata,
3809 );
3810 let mut body = json!({
3811 "lease_owner": lease_owner,
3812 "workflow_task_attempt": workflow_task_attempt,
3813 "commands": commands
3814 });
3815 if !message_stream_cursors.is_empty() {
3816 body["message_stream_cursors"] = Value::Array(message_stream_cursors);
3817 }
3818 if !message_stream_waits.is_empty() {
3819 body["message_stream_waits"] = Value::Array(message_stream_waits);
3820 }
3821 let path = format!("/worker/workflow-tasks/{task_id}/complete");
3822 self.request_json(
3823 reqwest::Method::POST,
3824 &path,
3825 RequestProtocol::Worker(protocol_version),
3826 Some(&body),
3827 )
3828 .await
3829 }
3830
3831 pub async fn fail_workflow_task(
3832 &self,
3833 task_id: &str,
3834 lease_owner: &str,
3835 workflow_task_attempt: u64,
3836 message: impl Into<String>,
3837 ) -> Result<Value> {
3838 self.fail_workflow_task_with_type(
3839 task_id,
3840 lease_owner,
3841 workflow_task_attempt,
3842 message,
3843 "RustWorkflowTaskFailure",
3844 )
3845 .await
3846 }
3847
3848 async fn fail_workflow_task_with_type(
3849 &self,
3850 task_id: &str,
3851 lease_owner: &str,
3852 workflow_task_attempt: u64,
3853 message: impl Into<String>,
3854 failure_type: &str,
3855 ) -> Result<Value> {
3856 let body = json!({
3857 "lease_owner": lease_owner,
3858 "workflow_task_attempt": workflow_task_attempt,
3859 "failure": {
3860 "message": message.into(),
3861 "type": failure_type
3862 }
3863 });
3864 let path = format!("/worker/workflow-tasks/{task_id}/fail");
3865 self.request_json(
3866 reqwest::Method::POST,
3867 &path,
3868 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3869 Some(&body),
3870 )
3871 .await
3872 }
3873
3874 pub async fn poll_activity_task(
3875 &self,
3876 worker_id: &str,
3877 task_queue: &str,
3878 timeout: Duration,
3879 ) -> Result<Option<ActivityTask>> {
3880 Ok(self
3881 .poll_activity_task_response(worker_id, task_queue, timeout)
3882 .await?
3883 .task)
3884 }
3885
3886 pub async fn poll_activity_task_response(
3888 &self,
3889 worker_id: &str,
3890 task_queue: &str,
3891 timeout: Duration,
3892 ) -> Result<PollActivityTaskResponse> {
3893 let poll_request_id = unique_request_id("rust-activity-poll");
3894 self.poll_activity_task_response_with_request_id(
3895 worker_id,
3896 task_queue,
3897 timeout,
3898 &poll_request_id,
3899 1,
3900 )
3901 .await
3902 }
3903
3904 async fn poll_activity_task_response_with_request_id(
3905 &self,
3906 worker_id: &str,
3907 task_queue: &str,
3908 timeout: Duration,
3909 poll_request_id: &str,
3910 transport_retries: usize,
3911 ) -> Result<PollActivityTaskResponse> {
3912 let body = json!({
3913 "worker_id": worker_id,
3914 "task_queue": task_queue,
3915 "poll_request_id": poll_request_id,
3916 "timeout_seconds": long_poll_timeout_seconds(timeout),
3917 });
3918 let data: PollActivityTaskResponse = self
3919 .poll_request_json(
3920 "/worker/activity-tasks/poll",
3921 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3922 &body,
3923 timeout + Duration::from_secs(5),
3924 transport_retries,
3925 )
3926 .await?;
3927 Ok(data)
3928 }
3929
3930 pub async fn complete_activity_task<T: Serialize>(
3931 &self,
3932 task_id: &str,
3933 activity_attempt_id: &str,
3934 lease_owner: &str,
3935 result: T,
3936 codec: &str,
3937 ) -> Result<Value> {
3938 let result = encode_typed_envelope(&AvroValue::from_serialize(&result)?, codec)?;
3939 let body = json!({
3940 "activity_attempt_id": activity_attempt_id,
3941 "lease_owner": lease_owner,
3942 "result": result
3943 });
3944 let path = format!("/worker/activity-tasks/{task_id}/complete");
3945 activity_task_response(
3946 self.request_json(
3947 reqwest::Method::POST,
3948 &path,
3949 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3950 Some(&body),
3951 )
3952 .await,
3953 "complete",
3954 task_id,
3955 activity_attempt_id,
3956 )
3957 }
3958
3959 pub async fn fail_activity_task(
3960 &self,
3961 task_id: &str,
3962 activity_attempt_id: &str,
3963 lease_owner: &str,
3964 message: impl Into<String>,
3965 non_retryable: bool,
3966 ) -> Result<Value> {
3967 let body = json!({
3968 "activity_attempt_id": activity_attempt_id,
3969 "lease_owner": lease_owner,
3970 "failure": {
3971 "message": message.into(),
3972 "type": "RustActivityFailure",
3973 "non_retryable": non_retryable
3974 }
3975 });
3976 let path = format!("/worker/activity-tasks/{task_id}/fail");
3977 activity_task_response(
3978 self.request_json(
3979 reqwest::Method::POST,
3980 &path,
3981 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3982 Some(&body),
3983 )
3984 .await,
3985 "fail",
3986 task_id,
3987 activity_attempt_id,
3988 )
3989 }
3990
3991 pub async fn heartbeat_activity_task<T: Serialize>(
3992 &self,
3993 task_id: &str,
3994 activity_attempt_id: &str,
3995 lease_owner: &str,
3996 details: T,
3997 ) -> Result<ActivityHeartbeatResponse> {
3998 let details = encode_typed_envelope(&AvroValue::from_serialize(&details)?, DEFAULT_CODEC)?;
3999 let body = json!({
4000 "activity_attempt_id": activity_attempt_id,
4001 "lease_owner": lease_owner,
4002 "details": details
4003 });
4004 let path = format!("/worker/activity-tasks/{task_id}/heartbeat");
4005 activity_task_response(
4006 self.request_json(
4007 reqwest::Method::POST,
4008 &path,
4009 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
4010 Some(&body),
4011 )
4012 .await,
4013 "heartbeat",
4014 task_id,
4015 activity_attempt_id,
4016 )
4017 }
4018
4019 async fn request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
4020 &self,
4021 method: reqwest::Method,
4022 path: &str,
4023 protocol: RequestProtocol,
4024 body: Option<&B>,
4025 ) -> Result<T> {
4026 self.request_json_with_timeout(method, path, protocol, body, Duration::from_secs(60))
4027 .await
4028 }
4029
4030 async fn request_json_with_timeout<T: DeserializeOwned, B: Serialize + ?Sized>(
4031 &self,
4032 method: reqwest::Method,
4033 path: &str,
4034 protocol: RequestProtocol,
4035 body: Option<&B>,
4036 timeout: Duration,
4037 ) -> Result<T> {
4038 let auth_token = self.auth_token(protocol)?;
4039 let mut request = self
4040 .http
4041 .request(method.clone(), format!("{}/api{}", self.base_url, path))
4042 .timeout(timeout)
4043 .header(reqwest::header::ACCEPT, "application/json")
4044 .header(reqwest::header::CONTENT_TYPE, "application/json")
4045 .header("X-Namespace", &self.namespace);
4046
4047 match protocol {
4048 RequestProtocol::Worker(version) => {
4049 request = request.header("X-Durable-Workflow-Protocol-Version", version);
4050 }
4051 RequestProtocol::ControlPlane => {
4052 request = request.header(
4053 "X-Durable-Workflow-Control-Plane-Version",
4054 CONTROL_PLANE_VERSION,
4055 );
4056 }
4057 }
4058
4059 if let Some(token) = auth_token {
4060 request = request.bearer_auth(token);
4061 }
4062
4063 if let Some(body) = body {
4064 let mut body = serde_json::to_value(body)?;
4065 if matches!(
4066 method,
4067 reqwest::Method::POST | reqwest::Method::PUT | reqwest::Method::PATCH
4068 ) {
4069 self.externalize_runtime_payloads(&mut body, path, protocol)
4070 .await?;
4071 }
4072 request = request.json(&body);
4073 }
4074
4075 let request = request.build()?;
4076 let poll_request_id = path.ends_with("/poll").then(|| {
4077 request
4078 .body()
4079 .and_then(reqwest::Body::as_bytes)
4080 .and_then(|body| serde_json::from_slice::<Value>(body).ok())
4081 .and_then(|body| body.get("poll_request_id")?.as_str().map(str::to_owned))
4082 .unwrap_or_default()
4083 });
4084 let mut storage_retries = 0_usize;
4085
4086 loop {
4087 let response = self
4089 .http
4090 .execute(request.try_clone().ok_or_else(|| {
4091 Error::WorkerLoop("worker request body cannot be retried".to_string())
4092 })?)
4093 .await?;
4094 let status = response.status();
4095 let bytes = response.bytes().await?;
4096
4097 if !status.is_success() {
4098 let body = String::from_utf8_lossy(&bytes).to_string();
4099 if let Some(protocol) = protocol_failure(status, &body) {
4100 return Err(Error::Protocol(protocol));
4101 }
4102 let error = Error::Http { status, body };
4103 if self
4104 .wait_for_storage_admission(
4105 &error,
4106 protocol,
4107 poll_request_id.as_deref(),
4108 &mut storage_retries,
4109 )
4110 .await
4111 {
4112 continue;
4113 }
4114 return Err(error);
4115 }
4116
4117 if bytes.is_empty() {
4118 return Ok(serde_json::from_value(Value::Null)?);
4119 }
4120
4121 let mut value: Value = serde_json::from_slice(&bytes)?;
4122 self.resolve_runtime_payloads(&mut value, path, protocol)
4123 .await?;
4124 return Ok(serde_json::from_value(value)?);
4125 }
4126 }
4127
4128 async fn wait_for_storage_admission(
4129 &self,
4130 error: &Error,
4131 protocol: RequestProtocol,
4132 poll_request_id: Option<&str>,
4133 retries: &mut usize,
4134 ) -> bool {
4135 let Some(admission) = self
4136 .worker_storage_admission
4137 .as_ref()
4138 .filter(|_| matches!(protocol, RequestProtocol::Worker(_)))
4139 else {
4140 return false;
4141 };
4142 let Some(advertised_delay) = worker_storage_admission_retry_after(error, poll_request_id)
4143 else {
4144 return false;
4145 };
4146 *retries = retries.saturating_add(1);
4147 let delay = worker_retry_delay(admission.policy, *retries)
4148 .max(advertised_delay)
4149 .min(admission.policy.max_backoff.max(Duration::from_millis(1)));
4150 let deadline = tokio::time::Instant::now() + delay;
4151 loop {
4152 if admission.stop.load(Ordering::SeqCst) {
4153 return false;
4154 }
4155 let remaining = deadline.saturating_duration_since(tokio::time::Instant::now());
4156 if remaining.is_zero() {
4157 return true;
4158 }
4159 tokio::time::sleep(remaining.min(Duration::from_millis(100))).await;
4160 }
4161 }
4162
4163 async fn poll_request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
4164 &self,
4165 path: &str,
4166 protocol: RequestProtocol,
4167 body: &B,
4168 timeout: Duration,
4169 max_retries: usize,
4170 ) -> Result<T> {
4171 let mut retries = 0;
4172
4173 loop {
4174 let response = self
4175 .request_json_with_timeout(
4176 reqwest::Method::POST,
4177 path,
4178 protocol,
4179 Some(body),
4180 timeout,
4181 )
4182 .await;
4183
4184 match response {
4185 Err(Error::Transport(_)) if retries < max_retries => retries += 1,
4186 response => return worker_poll_response(response),
4187 }
4188 }
4189 }
4190
4191 fn auth_token(&self, protocol: RequestProtocol) -> Result<Option<&str>> {
4192 match protocol {
4193 RequestProtocol::Worker(_) => {
4194 if let Some(token) = self.worker_token.as_deref().or(self.token.as_deref()) {
4195 return Ok(Some(token));
4196 }
4197 if self.control_token.is_some() {
4198 return Err(Error::MissingRoleCredentials {
4199 role: "worker",
4200 opposite_role: "control",
4201 });
4202 }
4203 Ok(None)
4204 }
4205 RequestProtocol::ControlPlane => {
4206 if let Some(token) = self.control_token.as_deref().or(self.token.as_deref()) {
4207 return Ok(Some(token));
4208 }
4209 if self.worker_token.is_some() {
4210 return Err(Error::MissingRoleCredentials {
4211 role: "control",
4212 opposite_role: "worker",
4213 });
4214 }
4215 Ok(None)
4216 }
4217 }
4218 }
4219}
4220
4221fn query_failure(status: reqwest::StatusCode, raw_body: String) -> QueryFailure {
4222 let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
4223 let reason = body
4224 .get("reason")
4225 .and_then(Value::as_str)
4226 .unwrap_or("query_rejected")
4227 .to_string();
4228 let message = body
4229 .get("message")
4230 .or_else(|| body.get("error"))
4231 .and_then(Value::as_str)
4232 .unwrap_or("workflow query was rejected")
4233 .to_string();
4234
4235 QueryFailure {
4236 status: status.as_u16(),
4237 reason,
4238 message,
4239 body,
4240 }
4241}
4242
4243fn workflow_command_result(
4244 command: WorkflowCommandKind,
4245 data: Value,
4246 workflow_id: &str,
4247 run_id: Option<&str>,
4248) -> WorkflowCommandResult {
4249 WorkflowCommandResult {
4250 command,
4251 workflow_id: data
4252 .get("workflow_id")
4253 .and_then(Value::as_str)
4254 .unwrap_or(workflow_id)
4255 .to_string(),
4256 run_id: data
4257 .get("run_id")
4258 .and_then(Value::as_str)
4259 .or(run_id)
4260 .map(str::to_string),
4261 outcome: data
4262 .get("outcome")
4263 .and_then(Value::as_str)
4264 .map(str::to_string),
4265 reason: data
4266 .get("reason")
4267 .and_then(Value::as_str)
4268 .map(str::to_string),
4269 command_status: data
4270 .get("command_status")
4271 .and_then(Value::as_str)
4272 .map(str::to_string),
4273 raw: data,
4274 }
4275}
4276
4277fn workflow_command_rejection(
4278 command: WorkflowCommandKind,
4279 status: reqwest::StatusCode,
4280 raw_body: String,
4281 workflow_id: &str,
4282 run_id: Option<&str>,
4283) -> WorkflowCommandRejection {
4284 let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
4285 WorkflowCommandRejection {
4286 command,
4287 status: status.as_u16(),
4288 reason: body
4289 .get("reason")
4290 .and_then(Value::as_str)
4291 .unwrap_or("workflow_command_rejected")
4292 .to_string(),
4293 message: body
4294 .get("message")
4295 .or_else(|| body.get("error"))
4296 .and_then(Value::as_str)
4297 .unwrap_or("workflow lifecycle command was rejected")
4298 .to_string(),
4299 workflow_id: body
4300 .get("workflow_id")
4301 .and_then(Value::as_str)
4302 .unwrap_or(workflow_id)
4303 .to_string(),
4304 run_id: body
4305 .get("run_id")
4306 .and_then(Value::as_str)
4307 .or(run_id)
4308 .map(str::to_string),
4309 target_scope: body
4310 .get("target_scope")
4311 .and_then(Value::as_str)
4312 .map(str::to_string),
4313 body,
4314 }
4315}
4316
4317fn query_task_response(response: Result<Value>) -> Result<Value> {
4318 match response {
4319 Err(Error::Http { status, body }) => Err(Error::QueryFailed(query_failure(status, body))),
4320 response => response,
4321 }
4322}
4323
4324fn worker_poll_response<T: DeserializeOwned>(response: Result<T>) -> Result<T> {
4325 match response {
4326 Err(Error::Http { status, body })
4327 if status == reqwest::StatusCode::CONFLICT && worker_poll_body_is_stop(&body) =>
4328 {
4329 Ok(serde_json::from_str(&body)?)
4330 }
4331 response => response,
4332 }
4333}
4334
4335fn worker_poll_body_is_stop(body: &str) -> bool {
4336 serde_json::from_str::<Value>(body)
4337 .ok()
4338 .is_some_and(|body| {
4339 worker_poll_is_stop(
4340 body.get("poll_status").and_then(Value::as_str),
4341 body.get("reason").and_then(Value::as_str),
4342 )
4343 })
4344}
4345
4346fn worker_poll_is_stop(poll_status: Option<&str>, reason: Option<&str>) -> bool {
4347 matches!(poll_status, Some("draining" | "stopped"))
4348 || matches!(reason, Some("worker_draining" | "worker_stopped"))
4349}
4350
4351fn query_task_rejection_is_final(error: &Error) -> bool {
4352 matches!(
4353 error,
4354 Error::QueryFailed(failure)
4355 if QUERY_TASK_FINAL_REJECTION_REASONS.contains(&failure.reason.as_str())
4356 )
4357}
4358
4359fn activity_task_response<T>(
4360 response: Result<T>,
4361 operation: &str,
4362 task_id: &str,
4363 activity_attempt_id: &str,
4364) -> Result<T> {
4365 match response {
4366 Err(Error::Http { status, body }) => {
4367 let body = serde_json::from_str(&body).unwrap_or_else(|_| json!({"message": body}));
4368 Err(Error::ActivityTaskRejected(ActivityTaskRejection {
4369 operation: operation.to_string(),
4370 status: status.as_u16(),
4371 reason: body
4372 .get("reason")
4373 .and_then(Value::as_str)
4374 .unwrap_or("activity_task_rejected")
4375 .to_string(),
4376 task_id: body
4377 .get("task_id")
4378 .and_then(Value::as_str)
4379 .unwrap_or(task_id)
4380 .to_string(),
4381 activity_attempt_id: body
4382 .get("activity_attempt_id")
4383 .and_then(Value::as_str)
4384 .unwrap_or(activity_attempt_id)
4385 .to_string(),
4386 cancel_requested: body
4387 .get("cancel_requested")
4388 .and_then(Value::as_bool)
4389 .unwrap_or(false),
4390 can_continue: body.get("can_continue").and_then(Value::as_bool),
4391 run_closed_reason: body
4392 .get("run_closed_reason")
4393 .and_then(Value::as_str)
4394 .map(str::to_string),
4395 body,
4396 }))
4397 }
4398 response => response,
4399 }
4400}
4401
4402fn activity_task_rejection_is_final(error: &Error) -> bool {
4403 matches!(
4404 error,
4405 Error::ActivityTaskRejected(rejection)
4406 if matches!(
4407 rejection.reason.as_str(),
4408 "run_cancelled"
4409 | "run_terminated"
4410 | "attempt_closed"
4411 | "stale_attempt"
4412 | "activity_cancelled"
4413 | "task_cancelled"
4414 | "run_closed"
4415 | "activity_not_running"
4416 | "attempt_not_found"
4417 )
4418 )
4419}
4420
4421fn workflow_task_completion_is_terminal_timeout(
4422 error: &Error,
4423 task_id: &str,
4424 workflow_task_attempt: u64,
4425 run_id: Option<&str>,
4426) -> bool {
4427 let Error::Http { status, body } = error else {
4428 return false;
4429 };
4430 if *status != reqwest::StatusCode::CONFLICT {
4431 return false;
4432 }
4433
4434 let Some(run_id) = run_id else {
4435 return false;
4436 };
4437 let Ok(body) = serde_json::from_str::<Value>(body) else {
4438 return false;
4439 };
4440
4441 body.get("recorded").and_then(Value::as_bool) == Some(false)
4442 && body.get("reason").and_then(Value::as_str) == Some("run_timed_out")
4443 && body.get("run_status").and_then(Value::as_str) == Some("failed")
4444 && body.get("run_id").and_then(Value::as_str) == Some(run_id)
4445 && body.get("task_id").and_then(Value::as_str) == Some(task_id)
4446 && body.get("workflow_task_attempt").and_then(Value::as_u64) == Some(workflow_task_attempt)
4447}
4448
4449fn protocol_failure(status: reqwest::StatusCode, raw_body: &str) -> Option<ProtocolFailure> {
4450 let body: Value = serde_json::from_str(raw_body).ok()?;
4451 let reason = body.get("reason")?.as_str()?;
4452 if !matches!(
4453 reason,
4454 "missing_protocol_version"
4455 | "unsupported_protocol_version"
4456 | "missing_control_plane_version"
4457 | "unsupported_control_plane_version"
4458 ) {
4459 return None;
4460 }
4461
4462 Some(ProtocolFailure {
4463 status: status.as_u16(),
4464 reason: reason.to_string(),
4465 message: body
4466 .get("message")
4467 .or_else(|| body.get("error"))
4468 .and_then(Value::as_str)
4469 .unwrap_or("protocol version rejected")
4470 .to_string(),
4471 supported_version: body
4472 .get("supported_version")
4473 .and_then(Value::as_str)
4474 .map(str::to_string),
4475 requested_version: body
4476 .get("requested_version")
4477 .and_then(Value::as_str)
4478 .map(str::to_string),
4479 body,
4480 })
4481}
4482
4483fn long_poll_timeout_seconds(timeout: Duration) -> u64 {
4484 timeout
4485 .as_secs()
4486 .saturating_add(u64::from(timeout.subsec_nanos() > 0))
4487 .min(MAX_LONG_POLL_TIMEOUT_SECONDS)
4488}
4489
4490fn worker_operation_is_retryable(error: &Error) -> bool {
4491 if worker_poll_capacity_retry_after(error).is_some()
4492 || worker_storage_admission_body(error).is_some()
4493 || worker_operation_is_explicitly_non_retryable(error)
4494 {
4495 return false;
4496 }
4497
4498 match error {
4499 Error::Transport(error) => {
4500 error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()
4501 }
4502 Error::Http { status, .. } => {
4503 matches!(
4504 *status,
4505 reqwest::StatusCode::REQUEST_TIMEOUT | reqwest::StatusCode::TOO_MANY_REQUESTS
4506 ) || status.is_server_error()
4507 }
4508 _ => false,
4509 }
4510}
4511
4512fn worker_storage_admission_body(error: &Error) -> Option<Value> {
4513 let body: Value = match error {
4514 Error::Http { body, .. } => serde_json::from_str(body).ok()?,
4515 Error::ActivityTaskRejected(rejection) => rejection.body.clone(),
4516 _ => return None,
4517 };
4518 matches!(
4519 body.get("reason").and_then(Value::as_str),
4520 Some("storage_pressure" | "storage_admission_unavailable")
4521 )
4522 .then_some(body)
4523}
4524
4525fn worker_storage_admission_retry_after(
4526 error: &Error,
4527 poll_request_id: Option<&str>,
4528) -> Option<Duration> {
4529 let Error::Http { status, .. } = error else {
4530 return None;
4531 };
4532 let body = worker_storage_admission_body(error)?;
4533 let delay = body.get("retry_after_seconds")?.as_u64()?;
4534 if *status != reqwest::StatusCode::SERVICE_UNAVAILABLE
4535 || delay == 0
4536 || body.get("retryable") != Some(&Value::Bool(true))
4537 || !matches!(body.get("storage_state")?.as_str()?, "draining" | "fenced")
4538 || (body["reason"] == "storage_admission_unavailable" && body["storage_state"] != "fenced")
4539 || body
4540 .get("request_admitted")
4541 .is_some_and(|admitted| admitted != &Value::Bool(false))
4542 {
4543 return None;
4544 }
4545 match poll_request_id {
4546 Some(id) => {
4547 if id.is_empty()
4548 || body.get("task") != Some(&Value::Null)
4549 || body.get("poll_request_id").and_then(Value::as_str) != Some(id)
4550 || body.get("poll_status") != body.get("reason")
4551 || body.get("retry_same_poll_request_id") != Some(&Value::Bool(true))
4552 || body.get("claim_admitted") != Some(&Value::Bool(false))
4553 {
4554 return None;
4555 }
4556 }
4557 None if body.get("request_admitted") != Some(&Value::Bool(false)) => return None,
4558 None => {}
4559 }
4560 Some(Duration::from_secs(delay))
4561}
4562
4563fn worker_operation_is_explicitly_non_retryable(error: &Error) -> bool {
4564 let Error::Http { body, .. } = error else {
4565 return false;
4566 };
4567
4568 serde_json::from_str::<Value>(body)
4569 .ok()
4570 .and_then(|body| body.get("retryable").and_then(Value::as_bool))
4571 == Some(false)
4572}
4573
4574fn worker_poll_capacity_retry_after(error: &Error) -> Option<Duration> {
4575 let Error::Http { status, body } = error else {
4576 return None;
4577 };
4578 if *status != reqwest::StatusCode::TOO_MANY_REQUESTS {
4579 return None;
4580 }
4581
4582 let body = serde_json::from_str::<Value>(body).ok()?;
4583 let capacity_exhausted = body.get("poll_status").and_then(Value::as_str)
4584 == Some("long_poll_capacity_exhausted")
4585 || body.get("reason").and_then(Value::as_str) == Some("long_poll_capacity_exhausted");
4586 if !capacity_exhausted || body.get("retryable").and_then(Value::as_bool) != Some(true) {
4587 return None;
4588 }
4589
4590 Some(Duration::from_secs(
4591 body.get("retry_after_seconds")
4592 .and_then(Value::as_u64)
4593 .unwrap_or_default(),
4594 ))
4595}
4596
4597fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration {
4598 let exponent = retry.saturating_sub(1).min(31) as u32;
4599 policy
4600 .initial_backoff
4601 .saturating_mul(1_u32 << exponent)
4602 .min(policy.max_backoff)
4603}
4604
4605#[derive(Debug)]
4606pub struct ClientBuilder {
4607 base_url: String,
4608 token: Option<String>,
4609 control_token: Option<String>,
4610 worker_token: Option<String>,
4611 namespace: String,
4612 timeout: Duration,
4613 max_external_payload_bytes: usize,
4614}
4615
4616impl ClientBuilder {
4617 pub fn token(mut self, token: Option<String>) -> Self {
4618 self.token = token;
4619 self
4620 }
4621
4622 pub fn control_token(mut self, token: Option<String>) -> Self {
4623 self.control_token = token;
4624 self
4625 }
4626
4627 pub fn worker_token(mut self, token: Option<String>) -> Self {
4628 self.worker_token = token;
4629 self
4630 }
4631
4632 pub fn namespace(mut self, namespace: impl Into<String>) -> Self {
4633 self.namespace = namespace.into();
4634 self
4635 }
4636
4637 pub fn timeout(mut self, timeout: Duration) -> Self {
4638 self.timeout = timeout;
4639 self
4640 }
4641
4642 pub fn max_external_payload_bytes(mut self, bytes: usize) -> Self {
4645 self.max_external_payload_bytes = bytes;
4646 self
4647 }
4648
4649 pub fn build(self) -> Result<Client> {
4650 let base_url = self.base_url.trim_end_matches('/').to_string();
4651 let has_sdk_api_suffix = reqwest::Url::parse(&base_url)
4652 .map(|url| url.path().trim_end_matches('/').ends_with("/api"))
4653 .unwrap_or_else(|_| base_url.ends_with("/api"));
4654
4655 if has_sdk_api_suffix {
4656 return Err(Error::InvalidBaseUrl);
4657 }
4658
4659 Ok(Client {
4660 http: reqwest::Client::builder()
4661 .timeout(self.timeout)
4662 .redirect(reqwest::redirect::Policy::none())
4663 .build()?,
4664 base_url,
4665 token: self.token,
4666 control_token: self.control_token,
4667 worker_token: self.worker_token,
4668 namespace: self.namespace,
4669 max_external_payload_bytes: self.max_external_payload_bytes,
4670 worker_storage_admission: None,
4671 runtime_upload_policy: Arc::new(Mutex::new([None, None])),
4672 })
4673 }
4674}
4675
4676#[derive(Clone, Debug)]
4677pub struct WorkflowHandle {
4678 client: Client,
4679 pub workflow_id: String,
4680 pub run_id: Option<String>,
4681 pub workflow_type: String,
4682}
4683
4684impl WorkflowHandle {
4685 pub async fn describe(&self) -> Result<WorkflowDescription> {
4687 self.client.describe_workflow(&self.workflow_id).await
4688 }
4689
4690 pub async fn describe_selected_run(&self) -> Result<WorkflowDescription> {
4692 let run_id = self.run_id.as_deref().ok_or_else(|| {
4693 Error::Codec("run_id is required for selected-run description".to_string())
4694 })?;
4695 self.client
4696 .describe_workflow_run(&self.workflow_id, run_id)
4697 .await
4698 }
4699
4700 pub async fn signal<T: Serialize>(&self, signal_name: &str, input: T) -> Result<Value> {
4701 self.client
4702 .signal_workflow(&self.workflow_id, signal_name, input)
4703 .await
4704 }
4705
4706 pub async fn append_message<T: Serialize>(
4707 &self,
4708 stream_name: &str,
4709 message_id: &str,
4710 input: T,
4711 ) -> Result<Value> {
4712 self.client
4713 .append_message_stream(&self.workflow_id, stream_name, message_id, input)
4714 .await
4715 }
4716
4717 pub async fn signal_selected_run<T: Serialize>(
4719 &self,
4720 signal_name: &str,
4721 input: T,
4722 ) -> Result<Value> {
4723 let run_id = self.run_id.as_deref().ok_or_else(|| {
4724 Error::Codec("run_id is required for selected-run signaling".to_string())
4725 })?;
4726 self.client
4727 .signal_workflow_run(&self.workflow_id, run_id, signal_name, input)
4728 .await
4729 }
4730
4731 pub async fn cancel(&self, options: WorkflowCommandOptions) -> Result<WorkflowCommandResult> {
4733 self.client
4734 .cancel_workflow(&self.workflow_id, options)
4735 .await
4736 }
4737
4738 pub async fn cancel_selected_run(
4740 &self,
4741 options: WorkflowCommandOptions,
4742 ) -> Result<WorkflowCommandResult> {
4743 let run_id = self.run_id.as_deref().ok_or_else(|| {
4744 Error::Codec("run_id is required for selected-run cancellation".to_string())
4745 })?;
4746 self.client
4747 .cancel_workflow_run(&self.workflow_id, run_id, options)
4748 .await
4749 }
4750
4751 pub async fn terminate(
4753 &self,
4754 options: WorkflowCommandOptions,
4755 ) -> Result<WorkflowCommandResult> {
4756 self.client
4757 .terminate_workflow(&self.workflow_id, options)
4758 .await
4759 }
4760
4761 pub async fn terminate_selected_run(
4763 &self,
4764 options: WorkflowCommandOptions,
4765 ) -> Result<WorkflowCommandResult> {
4766 let run_id = self.run_id.as_deref().ok_or_else(|| {
4767 Error::Codec("run_id is required for selected-run termination".to_string())
4768 })?;
4769 self.client
4770 .terminate_workflow_run(&self.workflow_id, run_id, options)
4771 .await
4772 }
4773
4774 pub async fn query<T: Serialize>(&self, query_name: &str, input: T) -> Result<Value> {
4776 self.client
4777 .query_workflow(&self.workflow_id, query_name, input)
4778 .await
4779 }
4780
4781 pub async fn query_avro_value<T: Serialize>(
4782 &self,
4783 query_name: &str,
4784 input: T,
4785 ) -> Result<AvroValue> {
4786 self.client
4787 .query_workflow_avro_value(&self.workflow_id, query_name, input)
4788 .await
4789 }
4790
4791 pub async fn update<T: Serialize>(
4792 &self,
4793 update_name: &str,
4794 input: T,
4795 request_id: Option<&str>,
4796 ) -> Result<Value> {
4797 self.client
4798 .update_workflow(&self.workflow_id, update_name, input, request_id)
4799 .await
4800 }
4801
4802 pub async fn update_avro_value<T: Serialize>(
4803 &self,
4804 update_name: &str,
4805 input: T,
4806 request_id: Option<&str>,
4807 ) -> Result<AvroValue> {
4808 self.client
4809 .update_workflow_avro_value(&self.workflow_id, update_name, input, request_id)
4810 .await
4811 }
4812
4813 pub async fn query_selected_run<T: Serialize>(
4815 &self,
4816 query_name: &str,
4817 input: T,
4818 ) -> Result<Value> {
4819 let run_id = self
4820 .run_id
4821 .as_deref()
4822 .ok_or_else(|| Error::Codec("run_id is required for selected-run query".to_string()))?;
4823 self.client
4824 .query_workflow_run(&self.workflow_id, run_id, query_name, input)
4825 .await
4826 }
4827
4828 pub async fn result(&self, options: WorkflowResultOptions) -> Result<Value> {
4830 self.result_target(options, None).await
4831 }
4832
4833 pub async fn result_avro_value(&self, options: WorkflowResultOptions) -> Result<AvroValue> {
4835 self.result_avro_value_target(options, None).await
4836 }
4837
4838 pub async fn result_typed<T: DeserializeOwned>(
4840 &self,
4841 options: WorkflowResultOptions,
4842 ) -> Result<T> {
4843 let result = self.result_avro_value(options).await?;
4844 decode_handler_result(result, HandlerKind::Workflow, &self.workflow_type)
4845 }
4846
4847 pub async fn result_selected_run(&self, options: WorkflowResultOptions) -> Result<Value> {
4849 let run_id = self.run_id.as_deref().ok_or_else(|| {
4850 Error::Codec("run_id is required for selected-run result".to_string())
4851 })?;
4852 self.result_target(options, Some(run_id)).await
4853 }
4854
4855 pub async fn result_selected_run_avro_value(
4857 &self,
4858 options: WorkflowResultOptions,
4859 ) -> Result<AvroValue> {
4860 let run_id = self.run_id.as_deref().ok_or_else(|| {
4861 Error::Codec("run_id is required for selected-run result".to_string())
4862 })?;
4863 self.result_avro_value_target(options, Some(run_id)).await
4864 }
4865
4866 pub async fn result_selected_run_typed<T: DeserializeOwned>(
4868 &self,
4869 options: WorkflowResultOptions,
4870 ) -> Result<T> {
4871 let result = self.result_selected_run_avro_value(options).await?;
4872 decode_handler_result(result, HandlerKind::Workflow, &self.workflow_type)
4873 }
4874
4875 async fn result_avro_value_target(
4876 &self,
4877 options: WorkflowResultOptions,
4878 selected_run_id: Option<&str>,
4879 ) -> Result<AvroValue> {
4880 let started = Instant::now();
4881
4882 loop {
4883 let description = match selected_run_id {
4884 Some(run_id) => {
4885 self.client
4886 .describe_workflow_run(&self.workflow_id, run_id)
4887 .await?
4888 }
4889 None => self.describe().await?,
4890 };
4891 if description.is_completed() {
4892 return description.output_avro_value.ok_or_else(|| {
4893 Error::Codec(
4894 "missing_payload_envelope: typed workflow result requires output_envelope"
4895 .to_string(),
4896 )
4897 });
4898 }
4899 if description.is_terminal() {
4900 let outcome =
4901 workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
4902 return Err(match outcome.kind {
4903 WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
4904 WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
4905 WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
4906 WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
4907 });
4908 }
4909 if started.elapsed() >= options.timeout {
4910 return Err(Error::Timeout);
4911 }
4912 tokio::time::sleep(options.poll_interval).await;
4913 }
4914 }
4915
4916 async fn result_target(
4917 &self,
4918 options: WorkflowResultOptions,
4919 selected_run_id: Option<&str>,
4920 ) -> Result<Value> {
4921 let started = Instant::now();
4922
4923 loop {
4924 let description = match selected_run_id {
4925 Some(run_id) => {
4926 self.client
4927 .describe_workflow_run(&self.workflow_id, run_id)
4928 .await?
4929 }
4930 None => self.describe().await?,
4931 };
4932 if description.is_completed() {
4933 return Ok(description.output.unwrap_or(Value::Null));
4934 }
4935
4936 if description.is_terminal() {
4937 let outcome =
4938 workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
4939 return Err(match outcome.kind {
4940 WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
4941 WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
4942 WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
4943 WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
4944 });
4945 }
4946
4947 if started.elapsed() >= options.timeout {
4948 return Err(Error::WorkflowTimedOut(WorkflowTerminalOutcome {
4949 kind: WorkflowTerminalKind::TimedOut,
4950 workflow_id: description
4951 .workflow_id
4952 .clone()
4953 .unwrap_or_else(|| self.workflow_id.clone()),
4954 run_id: description
4955 .run_id
4956 .clone()
4957 .or_else(|| selected_run_id.map(str::to_string)),
4958 reason: "result_wait_timeout".to_string(),
4959 failure_category: Some("client_timeout".to_string()),
4960 failure_id: None,
4961 exception_type: None,
4962 exception_class: None,
4963 non_retryable: None,
4964 message: Some(format!(
4965 "workflow result was not terminal within {:?}",
4966 options.timeout
4967 )),
4968 exception: None,
4969 raw: description.raw_value(),
4970 }));
4971 }
4972
4973 tokio::time::sleep(options.poll_interval).await;
4974 }
4975 }
4976}
4977
4978#[derive(Clone, Copy, Debug)]
4979pub struct WorkflowResultOptions {
4980 pub poll_interval: Duration,
4981 pub timeout: Duration,
4982}
4983
4984impl Default for WorkflowResultOptions {
4985 fn default() -> Self {
4986 Self {
4987 poll_interval: Duration::from_millis(500),
4988 timeout: Duration::from_secs(30),
4989 }
4990 }
4991}
4992
4993#[derive(Clone, Debug, Deserialize)]
4994pub struct WorkflowDescription {
4995 pub workflow_id: Option<String>,
4996 pub run_id: Option<String>,
4997 pub workflow_type: Option<String>,
4998 pub status: Option<String>,
4999 #[serde(default)]
5000 pub closed_reason: Option<String>,
5001 #[serde(default)]
5002 pub error: Option<String>,
5003 #[serde(default)]
5004 pub failure: Option<Value>,
5005 #[serde(default)]
5006 pub exception: Option<Value>,
5007 #[serde(default)]
5008 pub failures: Vec<Value>,
5009 #[serde(default)]
5010 pub output: Option<Value>,
5011 #[serde(default)]
5012 pub output_envelope: Option<Value>,
5013 #[serde(skip)]
5014 pub output_avro_value: Option<AvroValue>,
5015 #[serde(flatten)]
5016 pub raw: HashMap<String, Value>,
5017}
5018
5019#[derive(Clone, Debug, Deserialize)]
5021pub struct WorkflowStreamDescription {
5022 pub stream_name: String,
5023 pub status: String,
5024 pub last_offset: i64,
5025 pub total_items: u64,
5026 pub pending_items: u64,
5027 #[serde(default)]
5028 pub opened_at: Option<String>,
5029 #[serde(default)]
5030 pub last_appended_at: Option<String>,
5031 #[serde(default)]
5032 pub closed_at: Option<String>,
5033 #[serde(default)]
5034 pub error_reason: Option<String>,
5035 #[serde(default)]
5036 pub retention_seconds: Option<u64>,
5037 #[serde(flatten)]
5038 pub raw: HashMap<String, Value>,
5039}
5040
5041impl WorkflowStreamDescription {
5042 pub fn is_terminal(&self) -> bool {
5043 matches!(self.status.as_str(), "closed" | "errored")
5044 }
5045}
5046
5047#[derive(Clone, Debug, Default)]
5049pub struct WorkflowStreamAppendItem {
5050 pub payload_envelope: Option<Value>,
5051 pub payload_reference: Option<String>,
5052 pub item_type: Option<String>,
5053 pub content_type: Option<String>,
5054 pub idempotency_key: Option<String>,
5055}
5056
5057impl WorkflowStreamAppendItem {
5058 pub fn new<T: Serialize>(payload: T) -> Result<Self> {
5060 let value = AvroValue::from_serialize(&payload)?;
5061 Ok(Self {
5062 payload_envelope: Some(encode_typed_envelope(&value, DEFAULT_CODEC)?),
5063 ..Self::default()
5064 })
5065 }
5066
5067 pub fn from_reference(reference: impl Into<String>) -> Self {
5069 Self {
5070 payload_reference: Some(reference.into()),
5071 ..Self::default()
5072 }
5073 }
5074
5075 pub fn item_type(mut self, item_type: impl Into<String>) -> Self {
5076 self.item_type = Some(item_type.into());
5077 self
5078 }
5079
5080 pub fn content_type(mut self, content_type: impl Into<String>) -> Self {
5081 self.content_type = Some(content_type.into());
5082 self
5083 }
5084
5085 pub fn idempotency_key(mut self, idempotency_key: impl Into<String>) -> Self {
5086 self.idempotency_key = Some(idempotency_key.into());
5087 self
5088 }
5089
5090 fn wire_value(&self, derived_idempotency_key: Option<String>) -> Value {
5091 let mut item = serde_json::Map::new();
5092 if let Some(payload) = &self.payload_envelope {
5093 item.insert("payload".to_string(), payload.clone());
5094 item.insert("payload_codec".to_string(), json!(DEFAULT_CODEC));
5095 }
5096 if let Some(reference) = &self.payload_reference {
5097 item.insert("payload_reference".to_string(), json!(reference));
5098 }
5099 if let Some(item_type) = &self.item_type {
5100 item.insert("item_type".to_string(), json!(item_type));
5101 }
5102 if let Some(content_type) = &self.content_type {
5103 item.insert("content_type".to_string(), json!(content_type));
5104 }
5105 if let Some(key) = derived_idempotency_key
5106 .as_ref()
5107 .or(self.idempotency_key.as_ref())
5108 {
5109 item.insert("idempotency_key".to_string(), json!(key));
5110 }
5111 Value::Object(item)
5112 }
5113}
5114
5115#[derive(Clone, Debug)]
5117pub struct WorkflowStreamItem {
5118 pub offset: u64,
5119 pub payload: Option<Value>,
5120 pub payload_envelope: Option<Value>,
5121 pub payload_reference: Option<String>,
5122 pub payload_codec: Option<String>,
5123 pub idempotency_key: Option<String>,
5124 pub item_type: Option<String>,
5125 pub content_type: Option<String>,
5126 pub origin: Option<String>,
5127 pub origin_reference: Option<String>,
5128 pub emitted_at: Option<String>,
5129 pub raw: Value,
5130}
5131
5132#[derive(Clone, Debug)]
5134pub struct WorkflowStreamPage {
5135 pub stream: WorkflowStreamDescription,
5136 pub items: Vec<WorkflowStreamItem>,
5137 pub next_offset: u64,
5138 pub terminal: bool,
5139}
5140
5141#[derive(Clone, Debug)]
5143pub struct WorkflowStreamAppendResult {
5144 pub stream: WorkflowStreamDescription,
5145 pub accepted_offsets: Vec<u64>,
5146 pub accepted: u64,
5147 pub deduped: u64,
5148}
5149
5150#[derive(Deserialize)]
5151struct WorkflowStreamListResponse {
5152 #[serde(default)]
5153 streams: Vec<WorkflowStreamDescription>,
5154}
5155
5156#[derive(Deserialize)]
5157struct WorkflowStreamDescriptionResponse {
5158 stream: WorkflowStreamDescription,
5159}
5160
5161#[derive(Deserialize)]
5162struct WorkflowStreamPageResponse {
5163 stream: WorkflowStreamDescription,
5164 #[serde(default)]
5165 items: Vec<Value>,
5166 next_offset: u64,
5167 terminal: bool,
5168}
5169
5170#[derive(Deserialize)]
5171struct WorkflowStreamAppendResponse {
5172 stream: WorkflowStreamDescription,
5173 #[serde(default)]
5174 accepted_offsets: Vec<u64>,
5175 accepted: u64,
5176 deduped: u64,
5177}
5178
5179impl WorkflowDescription {
5180 pub fn is_completed(&self) -> bool {
5181 matches!(self.status.as_deref(), Some("completed" | "Completed"))
5182 }
5183
5184 pub fn is_terminal(&self) -> bool {
5185 matches!(
5186 self.status.as_deref(),
5187 Some(
5188 "completed"
5189 | "Completed"
5190 | "failed"
5191 | "Failed"
5192 | "cancelled"
5193 | "Cancelled"
5194 | "terminated"
5195 | "Terminated"
5196 | "timed_out"
5197 | "TimedOut",
5198 )
5199 )
5200 }
5201
5202 fn decode_payloads(&mut self) -> Result<()> {
5203 if let Some(envelope) = &self.output_envelope {
5204 let value = decode_wire_avro_value(envelope, DEFAULT_CODEC)?;
5205 self.output = Some(value.clone().into_json()?);
5206 self.output_avro_value = Some(value);
5207 }
5208
5209 Ok(())
5210 }
5211
5212 fn raw_value(&self) -> Value {
5213 let mut data = self.raw.clone();
5214 data.insert(
5215 "workflow_id".to_string(),
5216 self.workflow_id
5217 .clone()
5218 .map(Value::String)
5219 .unwrap_or(Value::Null),
5220 );
5221 data.insert(
5222 "run_id".to_string(),
5223 self.run_id
5224 .clone()
5225 .map(Value::String)
5226 .unwrap_or(Value::Null),
5227 );
5228 data.insert(
5229 "workflow_type".to_string(),
5230 self.workflow_type
5231 .clone()
5232 .map(Value::String)
5233 .unwrap_or(Value::Null),
5234 );
5235 data.insert(
5236 "status".to_string(),
5237 self.status
5238 .clone()
5239 .map(Value::String)
5240 .unwrap_or(Value::Null),
5241 );
5242 data.insert(
5243 "closed_reason".to_string(),
5244 self.closed_reason
5245 .clone()
5246 .map(Value::String)
5247 .unwrap_or(Value::Null),
5248 );
5249 if let Some(failure) = &self.failure {
5250 data.insert("failure".to_string(), failure.clone());
5251 }
5252 if let Some(exception) = &self.exception {
5253 data.insert("exception".to_string(), exception.clone());
5254 }
5255 Value::Object(data.into_iter().collect())
5256 }
5257}
5258
5259fn workflow_terminal_outcome(
5260 description: &WorkflowDescription,
5261 workflow_id: &str,
5262 run_id: Option<&str>,
5263) -> WorkflowTerminalOutcome {
5264 let terminal_kind = description
5265 .closed_reason
5266 .as_deref()
5267 .or(description.status.as_deref())
5268 .unwrap_or("failed")
5269 .to_ascii_lowercase();
5270 let kind = match terminal_kind.as_str() {
5271 "cancelled" | "canceled" => WorkflowTerminalKind::Cancelled,
5272 "terminated" => WorkflowTerminalKind::Terminated,
5273 "timed_out" | "timedout" => WorkflowTerminalKind::TimedOut,
5274 _ => WorkflowTerminalKind::Failed,
5275 };
5276 let default_reason = match kind {
5277 WorkflowTerminalKind::Failed => "workflow_failed",
5278 WorkflowTerminalKind::Cancelled => "cancelled",
5279 WorkflowTerminalKind::Terminated => "terminated",
5280 WorkflowTerminalKind::TimedOut => "timed_out",
5281 };
5282 let failure = description
5283 .failure
5284 .as_ref()
5285 .filter(|value| value.is_object());
5286 let nested_failure = failure
5287 .and_then(|value| value.get("failures"))
5288 .and_then(Value::as_array)
5289 .and_then(|failures| failures.last())
5290 .or_else(|| description.failures.last());
5291 let exception = description
5292 .exception
5293 .clone()
5294 .or_else(|| failure.and_then(|value| value.get("exception")).cloned())
5295 .or_else(|| {
5296 nested_failure
5297 .and_then(|value| value.get("exception_payload"))
5298 .cloned()
5299 });
5300 let string_field = |name: &str| {
5301 failure
5302 .and_then(|value| value.get(name))
5303 .and_then(Value::as_str)
5304 .or_else(|| {
5305 nested_failure
5306 .and_then(|value| value.get(name))
5307 .and_then(Value::as_str)
5308 })
5309 .map(str::to_string)
5310 };
5311 let exception_field = |name: &str| {
5312 exception
5313 .as_ref()
5314 .and_then(|value| value.get(name))
5315 .and_then(Value::as_str)
5316 .map(str::to_string)
5317 };
5318 let message = description
5319 .error
5320 .clone()
5321 .or_else(|| string_field("message"))
5322 .or_else(|| exception_field("message"));
5323 let reason = description
5324 .raw
5325 .get("reason")
5326 .and_then(Value::as_str)
5327 .map(str::to_string)
5328 .or_else(|| {
5329 failure
5330 .and_then(|value| value.get("reason"))
5331 .and_then(Value::as_str)
5332 .map(str::to_string)
5333 })
5334 .or_else(|| description.closed_reason.clone())
5335 .unwrap_or_else(|| default_reason.to_string());
5336 let failure_id = string_field("failure_id").or_else(|| {
5337 nested_failure
5338 .and_then(|value| value.get("id"))
5339 .and_then(Value::as_str)
5340 .map(str::to_string)
5341 });
5342
5343 WorkflowTerminalOutcome {
5344 kind,
5345 workflow_id: description
5346 .workflow_id
5347 .clone()
5348 .unwrap_or_else(|| workflow_id.to_string()),
5349 run_id: description
5350 .run_id
5351 .clone()
5352 .or_else(|| run_id.map(str::to_string)),
5353 reason,
5354 failure_category: string_field("failure_category")
5355 .or_else(|| Some(default_reason.to_string())),
5356 failure_id,
5357 exception_type: string_field("exception_type").or_else(|| exception_field("type")),
5358 exception_class: string_field("exception_class").or_else(|| exception_field("class")),
5359 non_retryable: failure
5360 .and_then(|value| value.get("non_retryable"))
5361 .and_then(Value::as_bool)
5362 .or_else(|| {
5363 nested_failure
5364 .and_then(|value| value.get("non_retryable"))
5365 .and_then(Value::as_bool)
5366 }),
5367 message,
5368 exception,
5369 raw: description.raw_value(),
5370 }
5371}
5372
5373#[derive(Clone, Debug, Deserialize)]
5374pub struct RegisterWorkerResponse {
5375 pub worker_id: String,
5376 pub registered: bool,
5377 #[serde(default)]
5378 pub heartbeat_interval_seconds: Option<u64>,
5379 #[serde(default)]
5380 pub protocol_version: Option<String>,
5381 #[serde(default)]
5382 pub server_capabilities: Option<Value>,
5383}
5384
5385#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
5387pub struct WorkerDeregistrationEnvelope {
5388 pub worker_id: String,
5389 pub outcome: String,
5390 pub recovered_workflow_task_count: u64,
5391}
5392
5393#[derive(Clone, Debug, Deserialize)]
5394pub struct PollWorkflowTaskResponse {
5395 #[serde(default)]
5396 pub task: Option<WorkflowTask>,
5397 #[serde(default)]
5398 pub poll_status: Option<String>,
5399 #[serde(default)]
5400 pub reason: Option<String>,
5401 #[serde(default)]
5402 pub protocol_version: Option<String>,
5403 #[serde(default)]
5404 pub server_capabilities: Option<Value>,
5405}
5406
5407impl PollWorkflowTaskResponse {
5408 pub fn outcome(&self) -> WorkerPollOutcome {
5410 worker_poll_outcome(
5411 self.task.is_some(),
5412 self.poll_status.as_deref(),
5413 self.reason.as_deref(),
5414 )
5415 }
5416}
5417
5418fn runtime_supports_workflow_memo_updates(capabilities: Option<&Value>) -> bool {
5419 let Some(capabilities) = capabilities.and_then(Value::as_object) else {
5420 return false;
5421 };
5422 let supported = capabilities
5423 .get("workflow_memo_updates")
5424 .and_then(Value::as_object)
5425 .and_then(|memo| memo.get("supported"))
5426 .and_then(Value::as_bool)
5427 == Some(true);
5428 let command_advertised = capabilities
5429 .get("supported_workflow_task_commands")
5430 .and_then(Value::as_array)
5431 .is_some_and(|commands| {
5432 commands
5433 .iter()
5434 .any(|command| command.as_str() == Some("upsert_memo"))
5435 });
5436 supported && command_advertised
5437}
5438
5439fn commands_use_workflow_memo_updates(commands: &[Value]) -> bool {
5440 commands
5441 .iter()
5442 .any(|command| command.get("type").and_then(Value::as_str) == Some("upsert_memo"))
5443}
5444
5445#[derive(Clone, Debug, Deserialize)]
5446pub struct PollActivityTaskResponse {
5447 #[serde(default)]
5448 pub task: Option<ActivityTask>,
5449 #[serde(default)]
5450 pub poll_status: Option<String>,
5451 #[serde(default)]
5452 pub reason: Option<String>,
5453}
5454
5455impl PollActivityTaskResponse {
5456 pub fn outcome(&self) -> WorkerPollOutcome {
5458 worker_poll_outcome(
5459 self.task.is_some(),
5460 self.poll_status.as_deref(),
5461 self.reason.as_deref(),
5462 )
5463 }
5464}
5465
5466#[derive(Clone, Debug, Deserialize)]
5467pub struct PollQueryTaskResponse {
5468 #[serde(default)]
5469 pub task: Option<QueryTask>,
5470 #[serde(default)]
5471 pub poll_status: Option<String>,
5472 #[serde(default)]
5473 pub reason: Option<String>,
5474}
5475
5476impl PollQueryTaskResponse {
5477 pub fn outcome(&self) -> WorkerPollOutcome {
5479 worker_poll_outcome(
5480 self.task.is_some(),
5481 self.poll_status.as_deref(),
5482 self.reason.as_deref(),
5483 )
5484 }
5485}
5486
5487#[derive(Clone, Debug, PartialEq, Eq)]
5489pub enum WorkerPollOutcome {
5490 Task,
5492 Idle {
5494 poll_status: Option<String>,
5495 reason: Option<String>,
5496 },
5497 Stop {
5499 poll_status: Option<String>,
5500 reason: Option<String>,
5501 },
5502}
5503
5504impl WorkerPollOutcome {
5505 pub fn should_stop(&self) -> bool {
5506 matches!(self, Self::Stop { .. })
5507 }
5508}
5509
5510fn worker_poll_outcome(
5511 has_task: bool,
5512 poll_status: Option<&str>,
5513 reason: Option<&str>,
5514) -> WorkerPollOutcome {
5515 if worker_poll_is_stop(poll_status, reason) {
5516 return WorkerPollOutcome::Stop {
5517 poll_status: poll_status.map(str::to_string),
5518 reason: reason.map(str::to_string),
5519 };
5520 }
5521
5522 if has_task {
5523 WorkerPollOutcome::Task
5524 } else {
5525 WorkerPollOutcome::Idle {
5526 poll_status: poll_status.map(str::to_string),
5527 reason: reason.map(str::to_string),
5528 }
5529 }
5530}
5531
5532#[derive(Clone, Debug, Deserialize)]
5534pub struct QueryTask {
5535 pub query_task_id: String,
5536 #[serde(default = "default_workflow_task_attempt")]
5537 pub query_task_attempt: u64,
5538 #[serde(default)]
5539 pub lease_owner: Option<String>,
5540 #[serde(default)]
5541 pub workflow_id: Option<String>,
5542 #[serde(default)]
5543 pub run_id: Option<String>,
5544 pub workflow_type: String,
5545 pub query_name: String,
5546 #[serde(
5547 default = "missing_task_payload_codec",
5548 deserialize_with = "deserialize_task_payload_codec"
5549 )]
5550 pub payload_codec: String,
5551 #[serde(default)]
5552 pub workflow_arguments: Option<Value>,
5553 #[serde(default)]
5554 pub query_arguments: Option<Value>,
5555 #[serde(default)]
5556 pub history_events: Vec<HistoryEvent>,
5557 #[serde(default)]
5558 pub history_export: Option<Value>,
5559 #[serde(default)]
5560 pub run_status: Option<String>,
5561}
5562
5563#[derive(Clone, Debug, Deserialize)]
5564pub struct WorkflowTask {
5565 pub task_id: String,
5566 #[serde(default)]
5567 pub workflow_command_id: Option<String>,
5568 #[serde(default)]
5569 pub workflow_id: Option<String>,
5570 #[serde(default)]
5571 pub run_id: Option<String>,
5572 pub workflow_type: String,
5573 #[serde(default)]
5574 pub cancel_requested: bool,
5575 #[serde(
5576 default = "missing_task_payload_codec",
5577 deserialize_with = "deserialize_task_payload_codec"
5578 )]
5579 pub payload_codec: String,
5580 #[serde(default)]
5581 pub arguments: Option<Value>,
5582 #[serde(default)]
5583 pub history_events: Vec<HistoryEvent>,
5584 #[serde(default)]
5585 pub total_history_events: Option<u64>,
5586 #[serde(default)]
5587 pub history_size_bytes: Option<u64>,
5588 #[serde(default)]
5589 pub continue_as_new_recommended: Option<bool>,
5590 #[serde(default)]
5591 pub history_budget_pressure: Option<String>,
5592 #[serde(default)]
5593 pub next_history_page_token: Option<String>,
5594 #[serde(default = "default_workflow_task_attempt")]
5595 pub workflow_task_attempt: u64,
5596 #[serde(default)]
5597 pub workflow_signal_id: Option<String>,
5598 #[serde(default)]
5599 pub signal_name: Option<String>,
5600 #[serde(default)]
5601 pub signal_arguments: Option<Value>,
5602 #[serde(default)]
5603 pub workflow_update_id: Option<String>,
5604 #[serde(default)]
5605 pub update_name: Option<String>,
5606 #[serde(default)]
5607 pub lease_owner: Option<String>,
5608}
5609
5610impl WorkflowTask {
5611 fn append_history_page(&mut self, page: WorkflowTaskHistoryPage) {
5612 self.history_events.extend(page.history_events);
5613
5614 if page.total_history_events.is_some() {
5615 self.total_history_events = page.total_history_events;
5616 }
5617
5618 self.next_history_page_token = page
5619 .next_history_page_token
5620 .filter(|token| !token.is_empty());
5621 }
5622}
5623
5624#[derive(Clone, Debug, Deserialize)]
5625struct WorkflowTaskHistoryPage {
5626 #[serde(default)]
5627 history_events: Vec<HistoryEvent>,
5628 #[serde(default)]
5629 total_history_events: Option<u64>,
5630 #[serde(default)]
5631 next_history_page_token: Option<String>,
5632}
5633
5634#[derive(Clone, Debug, Deserialize)]
5635pub struct ActivityTask {
5636 pub task_id: String,
5637 #[serde(default)]
5638 pub activity_attempt_id: Option<String>,
5639 #[serde(default)]
5640 pub attempt_id: Option<String>,
5641 pub activity_type: String,
5642 #[serde(
5643 default = "missing_task_payload_codec",
5644 deserialize_with = "deserialize_task_payload_codec"
5645 )]
5646 pub payload_codec: String,
5647 #[serde(default)]
5648 pub arguments: Option<Value>,
5649 #[serde(default = "default_attempt_number")]
5650 pub attempt_number: u64,
5651 #[serde(default)]
5652 pub lease_owner: Option<String>,
5653}
5654
5655#[derive(Clone, Debug, Deserialize)]
5656pub struct HistoryEvent {
5657 #[serde(alias = "type")]
5658 pub event_type: String,
5659 #[serde(default)]
5660 pub payload: Value,
5661 #[serde(flatten)]
5662 pub raw: HashMap<String, Value>,
5663}
5664
5665#[derive(Clone, Debug, PartialEq)]
5667pub struct QuerySignal {
5668 pub id: Option<String>,
5669 pub name: String,
5670 pub arguments: Vec<Value>,
5671 avro_arguments: Vec<AvroValue>,
5672 pub workflow_sequence: Option<u64>,
5673}
5674
5675impl QuerySignal {
5676 pub fn arguments_avro_value(&self) -> &[AvroValue] {
5678 &self.avro_arguments
5679 }
5680}
5681
5682#[derive(Clone, Debug)]
5688pub struct QueryContext {
5689 pub workflow_id: Option<String>,
5690 pub run_id: Option<String>,
5691 pub workflow_type: String,
5692 pub run_status: Option<String>,
5693 workflow_input: Value,
5694 workflow_input_avro_value: AvroValue,
5695 history_events: Arc<Vec<HistoryEvent>>,
5696 signal_events: Arc<Vec<QuerySignal>>,
5697}
5698
5699impl QueryContext {
5700 pub fn workflow_input(&self) -> &Value {
5702 &self.workflow_input
5703 }
5704
5705 pub fn workflow_input_avro_value(&self) -> &AvroValue {
5707 &self.workflow_input_avro_value
5708 }
5709
5710 pub fn history_events(&self) -> &[HistoryEvent] {
5712 self.history_events.as_slice()
5713 }
5714
5715 pub fn signal_events(&self) -> &[QuerySignal] {
5717 self.signal_events.as_slice()
5718 }
5719
5720 pub fn signals(&self, signal_name: &str) -> Vec<Vec<Value>> {
5722 self.signal_events
5723 .iter()
5724 .filter(|signal| signal.name == signal_name)
5725 .map(|signal| signal.arguments.clone())
5726 .collect()
5727 }
5728
5729 pub fn signals_avro_value(&self, signal_name: &str) -> Vec<Vec<AvroValue>> {
5731 self.signal_events
5732 .iter()
5733 .filter(|signal| signal.name == signal_name)
5734 .map(|signal| signal.avro_arguments.clone())
5735 .collect()
5736 }
5737}
5738
5739#[derive(Clone, Debug, Deserialize)]
5740pub struct ActivityHeartbeatResponse {
5741 #[serde(default)]
5742 pub cancel_requested: bool,
5743 #[serde(default)]
5744 pub heartbeat_recorded: bool,
5745 #[serde(default)]
5746 pub can_continue: Option<bool>,
5747 #[serde(default)]
5748 pub reason: Option<String>,
5749 #[serde(default)]
5750 pub run_closed_reason: Option<String>,
5751 #[serde(default)]
5752 pub run_closed_at: Option<String>,
5753 #[serde(default)]
5754 pub lease_expires_at: Option<String>,
5755 #[serde(default)]
5756 pub last_heartbeat_at: Option<String>,
5757}
5758
5759impl ActivityHeartbeatResponse {
5760 pub fn should_stop(&self) -> bool {
5762 self.cancel_requested || self.can_continue == Some(false)
5763 }
5764}
5765
5766fn missing_task_payload_codec() -> String {
5767 MISSING_TASK_PAYLOAD_CODEC.to_string()
5768}
5769
5770fn deserialize_task_payload_codec<'de, D>(deserializer: D) -> std::result::Result<String, D::Error>
5771where
5772 D: Deserializer<'de>,
5773{
5774 Ok(match Value::deserialize(deserializer)? {
5775 Value::String(codec) => codec,
5776 Value::Null => NULL_TASK_PAYLOAD_CODEC.to_string(),
5777 _ => NON_STRING_TASK_PAYLOAD_CODEC.to_string(),
5778 })
5779}
5780
5781fn default_workflow_task_attempt() -> u64 {
5782 1
5783}
5784
5785fn default_attempt_number() -> u64 {
5786 1
5787}
5788
5789type WorkflowFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5790type WorkflowHandler = Arc<dyn Fn(WorkflowContext, AvroValue) -> WorkflowFuture + Send + Sync>;
5791type ErasedWorkflowState = Arc<dyn Any + Send + Sync>;
5792type WorkflowStateSnapshot = Arc<dyn Fn() -> Result<ErasedWorkflowState> + Send + Sync>;
5793type ReplayedWorkflowHandler =
5794 Arc<dyn Fn(WorkflowContext, AvroValue) -> ReplayedWorkflowInvocation + Send + Sync>;
5795type ActivityFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5796type ActivityHandler = Arc<dyn Fn(ActivityContext, AvroValue) -> ActivityFuture + Send + Sync>;
5797type QueryFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5798type QueryHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
5799type UpdateHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
5800type ReplayedQueryHandler = Arc<
5801 dyn Fn(QueryContext, ErasedWorkflowState, AvroValue) -> std::result::Result<QueryFuture, String>
5802 + Send
5803 + Sync,
5804>;
5805type WorkerHeartbeatObserver = Arc<dyn Fn(&WorkerHeartbeatObservation) + Send + Sync>;
5806
5807struct ReplayedWorkflowInvocation {
5808 future: WorkflowFuture,
5809 snapshot: WorkflowStateSnapshot,
5810}
5811
5812#[derive(Clone)]
5813struct RegisteredWorkflow {
5814 execute: WorkflowHandler,
5815 replay: Option<ReplayedWorkflowHandler>,
5816 state_type: Option<TypeId>,
5817}
5818
5819#[derive(Debug)]
5820struct WorkflowTaskDecision {
5821 commands: Vec<Value>,
5822 message_stream_cursors: Vec<Value>,
5823 message_stream_waits: Vec<Value>,
5824}
5825
5826impl WorkflowTaskDecision {
5827 fn without_message_streams(commands: Vec<Value>) -> Self {
5828 Self {
5829 commands,
5830 message_stream_cursors: Vec::new(),
5831 message_stream_waits: Vec::new(),
5832 }
5833 }
5834}
5835
5836#[derive(Clone)]
5837enum RegisteredQuery {
5838 Snapshot(QueryHandler),
5839 Replayed {
5840 state_type: TypeId,
5841 handler: ReplayedQueryHandler,
5842 },
5843}
5844
5845#[derive(Clone, Debug)]
5846pub struct WorkerHeartbeatObservation {
5847 pub worker_id: String,
5848 pub task_queue: String,
5849 pub acknowledged_at_unix_millis: u64,
5850 pub acknowledgement: Value,
5851}
5852
5853#[derive(Clone, Copy, Debug)]
5865pub struct WorkerRetryPolicy {
5866 pub max_retries: usize,
5868 pub initial_backoff: Duration,
5870 pub max_backoff: Duration,
5872}
5873
5874impl Default for WorkerRetryPolicy {
5875 fn default() -> Self {
5876 Self {
5877 max_retries: 5,
5878 initial_backoff: Duration::from_millis(100),
5879 max_backoff: Duration::from_secs(5),
5880 }
5881 }
5882}
5883
5884#[derive(Clone, Debug)]
5885struct WorkerStorageAdmission {
5886 policy: WorkerRetryPolicy,
5887 stop: Arc<AtomicBool>,
5888}
5889
5890struct StopWorkerOnDrop(Arc<AtomicBool>);
5891
5892impl Drop for StopWorkerOnDrop {
5893 fn drop(&mut self) {
5894 self.0.store(true, Ordering::SeqCst);
5895 }
5896}
5897
5898async fn wait_for_worker_stop(stop: &AtomicBool) {
5899 while !stop.load(Ordering::SeqCst) {
5900 tokio::time::sleep(Duration::from_millis(100)).await;
5901 }
5902}
5903
5904#[derive(Clone, Copy, Debug, PartialEq, Eq)]
5905enum ManagedPollOutcome {
5906 Idle,
5907 Handled,
5908 Stop,
5909}
5910
5911#[derive(Clone)]
5912pub struct Worker {
5913 client: Client,
5914 worker_id: String,
5915 task_queue: String,
5916 workflows: HashMap<String, RegisteredWorkflow>,
5917 activities: HashMap<String, ActivityHandler>,
5918 queries: HashMap<String, HashMap<String, RegisteredQuery>>,
5919 updates: HashMap<String, HashMap<String, UpdateHandler>>,
5920 max_concurrent_workflow_tasks: usize,
5921 max_concurrent_activity_tasks: usize,
5922 poll_timeout: Duration,
5923 heartbeat_interval: Duration,
5924 retry_policy: WorkerRetryPolicy,
5925 heartbeat_observer: Option<WorkerHeartbeatObserver>,
5926}
5927
5928impl Worker {
5929 pub fn new(client: Client, task_queue: impl Into<String>) -> Self {
5930 Self {
5931 client,
5932 worker_id: default_worker_id(),
5933 task_queue: task_queue.into(),
5934 workflows: HashMap::new(),
5935 activities: HashMap::new(),
5936 queries: HashMap::new(),
5937 updates: HashMap::new(),
5938 max_concurrent_workflow_tasks: 10,
5939 max_concurrent_activity_tasks: 10,
5940 poll_timeout: Duration::from_secs(30),
5941 heartbeat_interval: Duration::from_secs(60),
5942 retry_policy: WorkerRetryPolicy::default(),
5943 heartbeat_observer: None,
5944 }
5945 }
5946
5947 pub fn worker_id(mut self, worker_id: impl Into<String>) -> Self {
5948 self.worker_id = worker_id.into();
5949 self
5950 }
5951
5952 pub fn poll_timeout(mut self, timeout: Duration) -> Self {
5953 self.poll_timeout = timeout;
5954 self
5955 }
5956
5957 pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
5958 self.heartbeat_interval = interval;
5959 self
5960 }
5961
5962 pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self {
5964 self.retry_policy = policy;
5965 self
5966 }
5967
5968 pub fn on_worker_heartbeat<F>(mut self, observer: F) -> Self
5969 where
5970 F: Fn(&WorkerHeartbeatObservation) + Send + Sync + 'static,
5971 {
5972 self.heartbeat_observer = Some(Arc::new(observer));
5973 self
5974 }
5975
5976 pub fn max_concurrent_workflow_tasks(mut self, count: usize) -> Self {
5977 self.max_concurrent_workflow_tasks = count.max(1);
5978 self
5979 }
5980
5981 pub fn max_concurrent_activity_tasks(mut self, count: usize) -> Self {
5982 self.max_concurrent_activity_tasks = count.max(1);
5983 self
5984 }
5985
5986 pub fn register_workflow<F, Fut>(&mut self, workflow_type: impl Into<String>, handler: F)
5993 where
5994 F: Fn(WorkflowContext, Value) -> Fut + Send + Sync + 'static,
5995 Fut: Future<Output = Result<Value>> + Send + 'static,
5996 {
5997 let handler = Arc::new(handler);
5998 self.workflows.insert(
5999 workflow_type.into(),
6000 RegisteredWorkflow {
6001 execute: Arc::new(move |ctx, input| {
6002 let handler = Arc::clone(&handler);
6003 Box::pin(async move {
6004 let result = handler(ctx, input.into_json()?).await?;
6005 AvroValue::from_serialize(&result)
6006 })
6007 }),
6008 replay: None,
6009 state_type: None,
6010 },
6011 );
6012 }
6013
6014 pub fn register_typed_workflow<I, O, F, Fut>(
6025 &mut self,
6026 workflow_type: impl Into<String>,
6027 handler: F,
6028 ) where
6029 I: DeserializeOwned + Send + 'static,
6030 O: Serialize + Send + 'static,
6031 F: Fn(WorkflowContext, I) -> Fut + Send + Sync + 'static,
6032 Fut: Future<Output = Result<O>> + Send + 'static,
6033 {
6034 let workflow_type = workflow_type.into();
6035 let handler_name = workflow_type.clone();
6036 let handler = Arc::new(handler);
6037 self.workflows.insert(
6038 workflow_type,
6039 RegisteredWorkflow {
6040 execute: Arc::new(move |ctx, input| {
6041 let handler = Arc::clone(&handler);
6042 let handler_name = handler_name.clone();
6043 Box::pin(async move {
6044 let input =
6045 decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
6046 let result = handler(ctx, input).await?;
6047 encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
6048 })
6049 }),
6050 replay: None,
6051 state_type: None,
6052 },
6053 );
6054 }
6055
6056 pub fn register_workflow_avro_value<F, Fut>(
6058 &mut self,
6059 workflow_type: impl Into<String>,
6060 handler: F,
6061 ) where
6062 F: Fn(WorkflowContext, AvroValue) -> Fut + Send + Sync + 'static,
6063 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6064 {
6065 self.workflows.insert(
6066 workflow_type.into(),
6067 RegisteredWorkflow {
6068 execute: Arc::new(move |ctx, input| Box::pin(handler(ctx, input))),
6069 replay: None,
6070 state_type: None,
6071 },
6072 );
6073 }
6074
6075 pub fn register_replayed_workflow<S, Factory, F, Fut>(
6083 &mut self,
6084 workflow_type: impl Into<String>,
6085 state_factory: Factory,
6086 handler: F,
6087 ) where
6088 S: Clone + Send + Sync + 'static,
6089 Factory: Fn() -> S + Send + Sync + 'static,
6090 F: Fn(WorkflowContext, Value, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
6091 Fut: Future<Output = Result<Value>> + Send + 'static,
6092 {
6093 let state_factory = Arc::new(state_factory);
6094 let handler = Arc::new(handler);
6095
6096 let execute_factory = Arc::clone(&state_factory);
6097 let execute_handler = Arc::clone(&handler);
6098 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6099 let state = WorkflowInstance::new(execute_factory());
6100 let handler = Arc::clone(&execute_handler);
6101 Box::pin(async move {
6102 let result = handler(ctx, input.into_json()?, state).await?;
6103 AvroValue::from_serialize(&result)
6104 }) as WorkflowFuture
6105 });
6106
6107 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6108 let state = WorkflowInstance::new(state_factory());
6109 let snapshot_state = state.clone();
6110 let snapshot: WorkflowStateSnapshot =
6111 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6112 let replay_handler = Arc::clone(&handler);
6113 let future = async move {
6114 let result = replay_handler(ctx, input.into_json()?, state).await?;
6115 AvroValue::from_serialize(&result)
6116 };
6117 ReplayedWorkflowInvocation {
6118 future: Box::pin(future),
6119 snapshot,
6120 }
6121 });
6122
6123 self.workflows.insert(
6124 workflow_type.into(),
6125 RegisteredWorkflow {
6126 execute,
6127 replay: Some(replay),
6128 state_type: Some(TypeId::of::<S>()),
6129 },
6130 );
6131 }
6132
6133 pub fn register_typed_replayed_workflow<I, O, S, Factory, F, Fut>(
6139 &mut self,
6140 workflow_type: impl Into<String>,
6141 state_factory: Factory,
6142 handler: F,
6143 ) where
6144 I: DeserializeOwned + Send + 'static,
6145 O: Serialize + Send + 'static,
6146 S: Clone + Send + Sync + 'static,
6147 Factory: Fn() -> S + Send + Sync + 'static,
6148 F: Fn(WorkflowContext, I, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
6149 Fut: Future<Output = Result<O>> + Send + 'static,
6150 {
6151 let workflow_type = workflow_type.into();
6152 let state_factory = Arc::new(state_factory);
6153 let handler = Arc::new(handler);
6154
6155 let execute_name = workflow_type.clone();
6156 let execute_factory = Arc::clone(&state_factory);
6157 let execute_handler = Arc::clone(&handler);
6158 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6159 let state = WorkflowInstance::new(execute_factory());
6160 let handler = Arc::clone(&execute_handler);
6161 let handler_name = execute_name.clone();
6162 Box::pin(async move {
6163 let input = decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
6164 let result = handler(ctx, input, state).await?;
6165 encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
6166 }) as WorkflowFuture
6167 });
6168
6169 let replay_name = workflow_type.clone();
6170 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6171 let state = WorkflowInstance::new(state_factory());
6172 let snapshot_state = state.clone();
6173 let snapshot: WorkflowStateSnapshot =
6174 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6175 let handler = Arc::clone(&handler);
6176 let handler_name = replay_name.clone();
6177 let future = async move {
6178 let input = decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
6179 let result = handler(ctx, input, state).await?;
6180 encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
6181 };
6182 ReplayedWorkflowInvocation {
6183 future: Box::pin(future),
6184 snapshot,
6185 }
6186 });
6187
6188 self.workflows.insert(
6189 workflow_type,
6190 RegisteredWorkflow {
6191 execute,
6192 replay: Some(replay),
6193 state_type: Some(TypeId::of::<S>()),
6194 },
6195 );
6196 }
6197
6198 pub fn register_replayed_workflow_avro_value<S, Factory, F, Fut>(
6200 &mut self,
6201 workflow_type: impl Into<String>,
6202 state_factory: Factory,
6203 handler: F,
6204 ) where
6205 S: Clone + Send + Sync + 'static,
6206 Factory: Fn() -> S + Send + Sync + 'static,
6207 F: Fn(WorkflowContext, AvroValue, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
6208 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6209 {
6210 let state_factory = Arc::new(state_factory);
6211 let handler = Arc::new(handler);
6212
6213 let execute_factory = Arc::clone(&state_factory);
6214 let execute_handler = Arc::clone(&handler);
6215 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6216 let state = WorkflowInstance::new(execute_factory());
6217 Box::pin(execute_handler(ctx, input, state)) as WorkflowFuture
6218 });
6219
6220 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6221 let state = WorkflowInstance::new(state_factory());
6222 let snapshot_state = state.clone();
6223 let snapshot: WorkflowStateSnapshot =
6224 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6225 ReplayedWorkflowInvocation {
6226 future: Box::pin(handler(ctx, input, state)),
6227 snapshot,
6228 }
6229 });
6230
6231 self.workflows.insert(
6232 workflow_type.into(),
6233 RegisteredWorkflow {
6234 execute,
6235 replay: Some(replay),
6236 state_type: Some(TypeId::of::<S>()),
6237 },
6238 );
6239 }
6240
6241 pub fn register_activity<F, Fut>(&mut self, activity_type: impl Into<String>, handler: F)
6242 where
6243 F: Fn(ActivityContext, Value) -> Fut + Send + Sync + 'static,
6244 Fut: Future<Output = Result<Value>> + Send + 'static,
6245 {
6246 let handler = Arc::new(handler);
6247 self.activities.insert(
6248 activity_type.into(),
6249 Arc::new(move |ctx, args| {
6250 let handler = Arc::clone(&handler);
6251 Box::pin(async move {
6252 let result = handler(ctx, args.into_json()?).await?;
6253 AvroValue::from_serialize(&result)
6254 })
6255 }),
6256 );
6257 }
6258
6259 pub fn register_typed_activity<I, O, F, Fut>(
6265 &mut self,
6266 activity_type: impl Into<String>,
6267 handler: F,
6268 ) where
6269 I: DeserializeOwned + Send + 'static,
6270 O: Serialize + Send + 'static,
6271 F: Fn(ActivityContext, I) -> Fut + Send + Sync + 'static,
6272 Fut: Future<Output = Result<O>> + Send + 'static,
6273 {
6274 let activity_type = activity_type.into();
6275 let handler_name = activity_type.clone();
6276 let handler = Arc::new(handler);
6277 self.activities.insert(
6278 activity_type,
6279 Arc::new(move |ctx, input| {
6280 let handler = Arc::clone(&handler);
6281 let handler_name = handler_name.clone();
6282 Box::pin(async move {
6283 let input =
6284 decode_handler_input::<I>(input, HandlerKind::Activity, &handler_name)?;
6285 let result = handler(ctx, input).await?;
6286 encode_handler_result(&result, HandlerKind::Activity, &handler_name)
6287 })
6288 }),
6289 );
6290 }
6291
6292 pub fn register_activity_avro_value<F, Fut>(
6294 &mut self,
6295 activity_type: impl Into<String>,
6296 handler: F,
6297 ) where
6298 F: Fn(ActivityContext, AvroValue) -> Fut + Send + Sync + 'static,
6299 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6300 {
6301 self.activities.insert(
6302 activity_type.into(),
6303 Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
6304 );
6305 }
6306
6307 pub fn register_query<F, Fut>(
6313 &mut self,
6314 workflow_type: impl Into<String>,
6315 query_name: impl Into<String>,
6316 handler: F,
6317 ) where
6318 F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
6319 Fut: Future<Output = Result<Value>> + Send + 'static,
6320 {
6321 let handler = Arc::new(handler);
6322 self.queries
6323 .entry(workflow_type.into())
6324 .or_default()
6325 .insert(
6326 query_name.into(),
6327 RegisteredQuery::Snapshot(Arc::new(move |ctx, args| {
6328 let handler = Arc::clone(&handler);
6329 Box::pin(async move {
6330 let result = handler(ctx, args.into_json()?).await?;
6331 AvroValue::from_serialize(&result)
6332 })
6333 })),
6334 );
6335 }
6336
6337 pub fn register_query_avro_value<F, Fut>(
6339 &mut self,
6340 workflow_type: impl Into<String>,
6341 query_name: impl Into<String>,
6342 handler: F,
6343 ) where
6344 F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
6345 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6346 {
6347 self.queries
6348 .entry(workflow_type.into())
6349 .or_default()
6350 .insert(
6351 query_name.into(),
6352 RegisteredQuery::Snapshot(Arc::new(move |ctx, args| Box::pin(handler(ctx, args)))),
6353 );
6354 }
6355
6356 pub fn register_replayed_query<S, F, Fut>(
6363 &mut self,
6364 workflow_type: impl Into<String>,
6365 query_name: impl Into<String>,
6366 handler: F,
6367 ) where
6368 S: Clone + Send + Sync + 'static,
6369 F: Fn(QueryContext, Arc<S>, Value) -> Fut + Send + Sync + 'static,
6370 Fut: Future<Output = Result<Value>> + Send + 'static,
6371 {
6372 let handler = Arc::new(handler);
6373 let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
6374 let state = state.downcast::<S>().map_err(|_| {
6375 "registered query state type does not match the replayed workflow state".to_string()
6376 })?;
6377 let handler = Arc::clone(&handler);
6378 Ok(Box::pin(async move {
6379 let result = handler(ctx, state, args.into_json()?).await?;
6380 AvroValue::from_serialize(&result)
6381 }))
6382 });
6383
6384 self.queries
6385 .entry(workflow_type.into())
6386 .or_default()
6387 .insert(
6388 query_name.into(),
6389 RegisteredQuery::Replayed {
6390 state_type: TypeId::of::<S>(),
6391 handler: erased_handler,
6392 },
6393 );
6394 }
6395
6396 pub fn register_replayed_query_avro_value<S, F, Fut>(
6398 &mut self,
6399 workflow_type: impl Into<String>,
6400 query_name: impl Into<String>,
6401 handler: F,
6402 ) where
6403 S: Clone + Send + Sync + 'static,
6404 F: Fn(QueryContext, Arc<S>, AvroValue) -> Fut + Send + Sync + 'static,
6405 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6406 {
6407 let handler = Arc::new(handler);
6408 let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
6409 let state = state.downcast::<S>().map_err(|_| {
6410 "registered query state type does not match the replayed workflow state".to_string()
6411 })?;
6412 Ok(Box::pin(handler(ctx, state, args)))
6413 });
6414
6415 self.queries
6416 .entry(workflow_type.into())
6417 .or_default()
6418 .insert(
6419 query_name.into(),
6420 RegisteredQuery::Replayed {
6421 state_type: TypeId::of::<S>(),
6422 handler: erased_handler,
6423 },
6424 );
6425 }
6426
6427 pub fn register_update<F, Fut>(
6429 &mut self,
6430 workflow_type: impl Into<String>,
6431 update_name: impl Into<String>,
6432 handler: F,
6433 ) where
6434 F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
6435 Fut: Future<Output = Result<Value>> + Send + 'static,
6436 {
6437 let handler = Arc::new(handler);
6438 self.updates
6439 .entry(workflow_type.into())
6440 .or_default()
6441 .insert(
6442 update_name.into(),
6443 Arc::new(move |ctx, args| {
6444 let handler = Arc::clone(&handler);
6445 Box::pin(async move {
6446 let result = handler(ctx, args.into_json()?).await?;
6447 AvroValue::from_serialize(&result)
6448 })
6449 }),
6450 );
6451 }
6452
6453 pub fn register_update_avro_value<F, Fut>(
6455 &mut self,
6456 workflow_type: impl Into<String>,
6457 update_name: impl Into<String>,
6458 handler: F,
6459 ) where
6460 F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
6461 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6462 {
6463 self.updates
6464 .entry(workflow_type.into())
6465 .or_default()
6466 .insert(
6467 update_name.into(),
6468 Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
6469 );
6470 }
6471
6472 pub async fn register(&self) -> Result<RegisterWorkerResponse> {
6473 let mut command_contracts = serde_json::Map::new();
6474 for workflow_type in self.workflows.keys() {
6475 let mut queries = self
6476 .queries
6477 .get(workflow_type)
6478 .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
6479 .unwrap_or_default();
6480 queries.sort();
6481 let mut updates = self
6482 .updates
6483 .get(workflow_type)
6484 .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
6485 .unwrap_or_default();
6486 updates.sort();
6487 command_contracts.insert(
6488 workflow_type.clone(),
6489 json!({
6490 "queries": queries,
6491 "query_contracts": [],
6492 "signals": [],
6493 "signal_contracts": [],
6494 "updates": updates,
6495 "update_contracts": [],
6496 "update_validators": [],
6497 }),
6498 );
6499 }
6500
6501 self.client
6502 .register_worker_with_command_contracts(
6503 &self.worker_id,
6504 &self.task_queue,
6505 self.workflows.keys().cloned().collect(),
6506 self.activities.keys().cloned().collect(),
6507 self.max_concurrent_workflow_tasks,
6508 self.max_concurrent_activity_tasks,
6509 [
6510 Some(CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY.to_string()),
6511 Some(DURABLE_SELECTION_CAPABILITY.to_string()),
6512 Some(MEMO_UPSERTS_CAPABILITY.to_string()),
6513 Some(TYPED_SEARCH_ATTRIBUTES_CAPABILITY.to_string()),
6514 (!self.queries.is_empty()).then(|| QUERY_TASKS_CAPABILITY.to_string()),
6515 (!self.updates.is_empty()).then(|| WORKFLOW_UPDATES_CAPABILITY.to_string()),
6516 worker_protocol_supports_message_streams(WORKER_PROTOCOL_VERSION)
6517 .then(|| MESSAGE_STREAMS_CAPABILITY.to_string()),
6518 ]
6519 .into_iter()
6520 .flatten()
6521 .collect(),
6522 Value::Object(command_contracts),
6523 )
6524 .await
6525 }
6526
6527 pub async fn run(&self) -> Result<()> {
6533 self.run_until(std::future::pending::<()>()).await
6534 }
6535
6536 pub async fn run_until<F>(&self, shutdown: F) -> Result<()>
6540 where
6541 F: Future<Output = ()>,
6542 {
6543 let stop = Arc::new(AtomicBool::new(false));
6544 let _stop_on_drop = StopWorkerOnDrop(Arc::clone(&stop));
6545 let worker = self.with_storage_admission(Arc::clone(&stop));
6546 let run = worker.run_with_storage_admission(Arc::clone(&stop));
6547 tokio::pin!(run);
6548 tokio::pin!(shutdown);
6549 tokio::select! {
6550 result = &mut run => result,
6551 _ = &mut shutdown => {
6552 stop.store(true, Ordering::SeqCst);
6553 run.await
6554 }
6555 }
6556 }
6557
6558 fn with_storage_admission(&self, stop: Arc<AtomicBool>) -> Self {
6559 let mut worker = self.clone();
6560 worker.client.worker_storage_admission = Some(WorkerStorageAdmission {
6561 policy: self.retry_policy,
6562 stop,
6563 });
6564 worker
6565 }
6566
6567 async fn run_with_storage_admission(&self, stop: Arc<AtomicBool>) -> Result<()> {
6568 let registration = self.register().await?;
6569 if !registration.registered {
6570 return Err(Error::WorkerLoop(format!(
6571 "worker registration for {:?} was not accepted",
6572 self.worker_id
6573 )));
6574 }
6575 let registered_worker_id = registration.worker_id.clone();
6576 let primary = self.run_registered_until(stop, registration).await;
6577 let deregistration = self
6578 .client
6579 .deregister_worker_registration(®istered_worker_id)
6580 .await;
6581
6582 match (primary, deregistration) {
6583 (Ok(()), Ok(_)) => Ok(()),
6584 (Ok(()), Err(deregistration)) => Err(deregistration),
6585 (Err(primary), Ok(_)) => Err(primary),
6586 (Err(primary), Err(deregistration)) => Err(Error::WorkerShutdown {
6587 primary: Box::new(primary),
6588 deregistration: Box::new(deregistration),
6589 }),
6590 }
6591 }
6592
6593 async fn run_registered_until(
6594 &self,
6595 stop: Arc<AtomicBool>,
6596 registration: RegisterWorkerResponse,
6597 ) -> Result<()> {
6598 let heartbeat_interval = Duration::from_secs(
6599 registration
6600 .heartbeat_interval_seconds
6601 .unwrap_or(self.heartbeat_interval.as_secs().max(1)),
6602 );
6603 let heartbeat = tokio::time::sleep(Duration::ZERO);
6609 tokio::pin!(heartbeat);
6610 let mut workflow_poller = (!self.workflows.is_empty()).then(|| {
6614 let worker = self.clone();
6615 let stop = Arc::clone(&stop);
6616 tokio::spawn(async move { worker.poll_workflows_until_stopped(stop).await })
6617 });
6618 let mut activity_poller = (!self.activities.is_empty()).then(|| {
6619 let worker = self.clone();
6620 let stop = Arc::clone(&stop);
6621 tokio::spawn(async move { worker.poll_activities_until_stopped(stop).await })
6622 });
6623 let mut query_poller = (!self.queries.is_empty()).then(|| {
6624 let worker = self.clone();
6625 let stop = Arc::clone(&stop);
6626 tokio::spawn(async move { worker.poll_queries_until_stopped(stop).await })
6627 });
6628
6629 loop {
6630 tokio::select! {
6631 _ = wait_for_worker_stop(&stop) => {
6632 stop.store(true, Ordering::SeqCst);
6633 break;
6634 }
6635 _ = &mut heartbeat => {
6636 let result = self.retry_worker_operation(|| {
6637 self.client.heartbeat_worker(
6638 &self.worker_id,
6639 self.max_concurrent_workflow_tasks,
6640 self.max_concurrent_activity_tasks,
6641 )
6642 }).await;
6643 heartbeat
6644 .as_mut()
6645 .reset(tokio::time::Instant::now() + heartbeat_interval);
6646 match result {
6647 Ok(acknowledgement) => {
6648 if let Some(observer) = &self.heartbeat_observer {
6649 observer(&WorkerHeartbeatObservation {
6650 worker_id: self.worker_id.clone(),
6651 task_queue: self.task_queue.clone(),
6652 acknowledged_at_unix_millis: SystemTime::now()
6653 .duration_since(UNIX_EPOCH)
6654 .unwrap_or_default()
6655 .as_millis()
6656 .min(u64::MAX as u128)
6657 as u64,
6658 acknowledgement,
6659 });
6660 }
6661 }
6662 Err(error) => {
6663 stop.store(true, Ordering::SeqCst);
6664 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await?;
6665 return Err(error);
6666 }
6667 }
6668 }
6669 result = OptionFuture::from(workflow_poller.as_mut()), if workflow_poller.is_some() => {
6670 workflow_poller = None;
6671 let stopped_by_server = stop.load(Ordering::SeqCst);
6672 stop.store(true, Ordering::SeqCst);
6673 let poller_result = optional_poller_result("workflow", result);
6674 let join_result =
6675 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6676 poller_result?;
6677 join_result?;
6678 if stopped_by_server {
6679 return Ok(());
6680 }
6681 return Err(Error::WorkerLoop(
6682 "workflow poller stopped unexpectedly".to_string(),
6683 ));
6684 }
6685 result = OptionFuture::from(activity_poller.as_mut()), if activity_poller.is_some() => {
6686 activity_poller = None;
6687 let stopped_by_server = stop.load(Ordering::SeqCst);
6688 stop.store(true, Ordering::SeqCst);
6689 let poller_result = optional_poller_result("activity", result);
6690 let join_result =
6691 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6692 poller_result?;
6693 join_result?;
6694 if stopped_by_server {
6695 return Ok(());
6696 }
6697 return Err(Error::WorkerLoop(
6698 "activity poller stopped unexpectedly".to_string(),
6699 ));
6700 }
6701 result = OptionFuture::from(query_poller.as_mut()), if query_poller.is_some() => {
6702 query_poller = None;
6703 let stopped_by_server = stop.load(Ordering::SeqCst);
6704 stop.store(true, Ordering::SeqCst);
6705 let poller_result = optional_poller_result("query", result);
6706 let join_result =
6707 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6708 poller_result?;
6709 join_result?;
6710 if stopped_by_server {
6711 return Ok(());
6712 }
6713 return Err(Error::WorkerLoop(
6714 "query poller stopped unexpectedly".to_string(),
6715 ));
6716 }
6717 }
6718 }
6719
6720 join_pollers(
6721 workflow_poller.take(),
6722 activity_poller.take(),
6723 query_poller.take(),
6724 )
6725 .await
6726 }
6727
6728 pub async fn run_once(&self) -> Result<usize> {
6742 let worker = self.with_storage_admission(Arc::new(AtomicBool::new(false)));
6743 let mut handled = 0;
6744 match worker.poll_workflow_once().await? {
6745 ManagedPollOutcome::Handled => handled += 1,
6746 ManagedPollOutcome::Stop => return Ok(handled),
6747 ManagedPollOutcome::Idle => {}
6748 }
6749 match worker.poll_activity_once().await? {
6750 ManagedPollOutcome::Handled => handled += 1,
6751 ManagedPollOutcome::Stop => return Ok(handled),
6752 ManagedPollOutcome::Idle => {}
6753 }
6754 if !self.queries.is_empty() {
6755 match worker.poll_query_once().await? {
6756 ManagedPollOutcome::Handled => handled += 1,
6757 ManagedPollOutcome::Stop => return Ok(handled),
6758 ManagedPollOutcome::Idle => {}
6759 }
6760 }
6761 Ok(handled)
6762 }
6763
6764 async fn poll_workflow_once(&self) -> Result<ManagedPollOutcome> {
6765 let poll_request_id = unique_request_id("rust-workflow-poll");
6766 let response = self
6767 .retry_worker_operation(|| {
6768 self.client.poll_workflow_task_response_with_request_id(
6769 &self.worker_id,
6770 &self.task_queue,
6771 self.poll_timeout,
6772 &poll_request_id,
6773 0,
6774 )
6775 })
6776 .await;
6777 let Some(response) = self.settle_worker_poll_response(response).await? else {
6778 return Ok(ManagedPollOutcome::Idle);
6779 };
6780 if response.outcome().should_stop() {
6781 return Ok(ManagedPollOutcome::Stop);
6782 }
6783 let memo_updates_supported =
6784 runtime_supports_workflow_memo_updates(response.server_capabilities.as_ref());
6785 let Some(task) = response.task else {
6786 return Ok(ManagedPollOutcome::Idle);
6787 };
6788
6789 let task_id = task.task_id.clone();
6790 let attempt = task.workflow_task_attempt;
6791 let run_id = task.run_id.clone();
6792 let lease_owner = task
6793 .lease_owner
6794 .clone()
6795 .unwrap_or_else(|| self.worker_id.clone());
6796
6797 match self.execute_workflow_task_decision(task) {
6798 Ok(decision)
6799 if commands_use_workflow_memo_updates(&decision.commands)
6800 && !memo_updates_supported =>
6801 {
6802 self.client
6803 .fail_workflow_task(
6804 &task_id,
6805 &lease_owner,
6806 attempt,
6807 Error::WorkflowMemoUpdatesUnavailable.to_string(),
6808 )
6809 .await?;
6810 }
6811 Ok(decision) if decision.commands.is_empty() => {
6812 self.client
6818 .fail_workflow_task_with_type(
6819 &task_id,
6820 &lease_owner,
6821 attempt,
6822 WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE,
6823 WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE,
6824 )
6825 .await?;
6826 }
6827 Ok(decision) => {
6828 let completion = self
6829 .client
6830 .complete_workflow_task_with_message_streams(
6831 &task_id,
6832 &lease_owner,
6833 attempt,
6834 decision.commands,
6835 decision.message_stream_cursors,
6836 decision.message_stream_waits,
6837 )
6838 .await;
6839 if let Err(error) = completion {
6840 if !workflow_task_completion_is_terminal_timeout(
6841 &error,
6842 &task_id,
6843 attempt,
6844 run_id.as_deref(),
6845 ) {
6846 return Err(error);
6847 }
6848 }
6849 }
6850 Err(error) => {
6851 self.client
6852 .fail_workflow_task(&task_id, &lease_owner, attempt, error.to_string())
6853 .await?;
6854 }
6855 }
6856
6857 Ok(ManagedPollOutcome::Handled)
6858 }
6859
6860 async fn poll_workflows_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
6861 while !stop.load(Ordering::SeqCst) {
6862 if self.poll_workflow_once().await? == ManagedPollOutcome::Stop {
6863 stop.store(true, Ordering::SeqCst);
6864 break;
6865 }
6866 }
6867
6868 Ok(())
6869 }
6870
6871 async fn poll_activity_once(&self) -> Result<ManagedPollOutcome> {
6872 let poll_request_id = unique_request_id("rust-activity-poll");
6873 let response = self
6874 .retry_worker_operation(|| {
6875 self.client.poll_activity_task_response_with_request_id(
6876 &self.worker_id,
6877 &self.task_queue,
6878 self.poll_timeout,
6879 &poll_request_id,
6880 0,
6881 )
6882 })
6883 .await;
6884 let Some(response) = self.settle_worker_poll_response(response).await? else {
6885 return Ok(ManagedPollOutcome::Idle);
6886 };
6887 if response.outcome().should_stop() {
6888 return Ok(ManagedPollOutcome::Stop);
6889 }
6890 let Some(task) = response.task else {
6891 return Ok(ManagedPollOutcome::Idle);
6892 };
6893
6894 let task_id = task.task_id.clone();
6895 let attempt_id = task
6896 .activity_attempt_id
6897 .clone()
6898 .or(task.attempt_id.clone())
6899 .unwrap_or_default();
6900 let lease_owner = task
6901 .lease_owner
6902 .clone()
6903 .unwrap_or_else(|| self.worker_id.clone());
6904 let codec = task.payload_codec.clone();
6905 let result = self.execute_activity_task(task).await;
6906 match result {
6907 Err(error) if worker_storage_admission_body(&error).is_some() => return Err(error),
6908 Ok(value) => {
6909 let completion = self
6910 .client
6911 .complete_activity_task(&task_id, &attempt_id, &lease_owner, value, &codec)
6912 .await;
6913 if let Err(error) = completion {
6914 if !activity_task_rejection_is_final(&error) {
6915 return Err(error);
6916 }
6917 }
6918 }
6919 Err(error) => {
6920 let failure = self
6921 .client
6922 .fail_activity_task(
6923 &task_id,
6924 &attempt_id,
6925 &lease_owner,
6926 error.to_string(),
6927 false,
6928 )
6929 .await;
6930 if let Err(error) = failure {
6931 if !activity_task_rejection_is_final(&error) {
6932 return Err(error);
6933 }
6934 }
6935 }
6936 }
6937
6938 Ok(ManagedPollOutcome::Handled)
6939 }
6940
6941 async fn poll_activities_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
6942 while !stop.load(Ordering::SeqCst) {
6943 if self.poll_activity_once().await? == ManagedPollOutcome::Stop {
6944 stop.store(true, Ordering::SeqCst);
6945 break;
6946 }
6947 }
6948
6949 Ok(())
6950 }
6951
6952 async fn poll_query_once(&self) -> Result<ManagedPollOutcome> {
6953 let poll_request_id = unique_request_id("rust-query-poll");
6954 let response = self
6955 .retry_worker_operation(|| {
6956 self.client.poll_query_task_response_with_request_id(
6957 &self.worker_id,
6958 &self.task_queue,
6959 self.poll_timeout,
6960 &poll_request_id,
6961 0,
6962 )
6963 })
6964 .await;
6965 let Some(response) = self.settle_worker_poll_response(response).await? else {
6966 return Ok(ManagedPollOutcome::Idle);
6967 };
6968 if response.outcome().should_stop() {
6969 return Ok(ManagedPollOutcome::Stop);
6970 }
6971 let Some(task) = response.task else {
6972 return Ok(ManagedPollOutcome::Idle);
6973 };
6974
6975 let query_task_id = task.query_task_id.clone();
6976 let attempt = task.query_task_attempt;
6977 let lease_owner = task
6978 .lease_owner
6979 .clone()
6980 .unwrap_or_else(|| self.worker_id.clone());
6981 let codec = task.payload_codec.clone();
6982
6983 match self.execute_query_task(task).await {
6984 Ok(value) => {
6985 let result_envelope = match encode_typed_envelope(&value, &codec) {
6986 Ok(result_envelope) => result_envelope,
6987 Err(error) => {
6988 let failure = self
6989 .client
6990 .fail_query_task(
6991 &query_task_id,
6992 &lease_owner,
6993 attempt,
6994 error.to_string(),
6995 "query_result_encode_failed",
6996 "QueryResultEncodeFailed",
6997 )
6998 .await;
6999 if let Err(error) = failure {
7000 if !query_task_rejection_is_final(&error) {
7001 return Err(error);
7002 }
7003 }
7004 return Ok(ManagedPollOutcome::Handled);
7005 }
7006 };
7007
7008 if let Err(error) = self
7009 .client
7010 .complete_query_task_with_envelope(
7011 &query_task_id,
7012 &lease_owner,
7013 attempt,
7014 value.clone().into_json()?,
7015 result_envelope,
7016 )
7017 .await
7018 {
7019 if !query_task_rejection_is_final(&error) {
7020 return Err(error);
7021 }
7022 }
7023 }
7024 Err(failure) => {
7025 let result = self
7026 .client
7027 .fail_query_task(
7028 &query_task_id,
7029 &lease_owner,
7030 attempt,
7031 failure.message,
7032 failure.reason,
7033 failure.failure_type,
7034 )
7035 .await;
7036 if let Err(error) = result {
7037 if !query_task_rejection_is_final(&error) {
7038 return Err(error);
7039 }
7040 }
7041 }
7042 }
7043
7044 Ok(ManagedPollOutcome::Handled)
7045 }
7046
7047 async fn poll_queries_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
7048 while !stop.load(Ordering::SeqCst) {
7049 if self.poll_query_once().await? == ManagedPollOutcome::Stop {
7050 stop.store(true, Ordering::SeqCst);
7051 break;
7052 }
7053 }
7054
7055 Ok(())
7056 }
7057
7058 async fn retry_worker_operation<T, F, Fut>(&self, mut operation: F) -> Result<T>
7059 where
7060 F: FnMut() -> Fut,
7061 Fut: Future<Output = Result<T>>,
7062 {
7063 let mut retries = 0;
7064
7065 loop {
7066 match operation().await {
7067 Err(error)
7068 if worker_operation_is_retryable(&error)
7069 && retries < self.retry_policy.max_retries =>
7070 {
7071 retries += 1;
7072 tokio::time::sleep(worker_retry_delay(self.retry_policy, retries)).await;
7073 }
7074 result => return result,
7075 }
7076 }
7077 }
7078
7079 async fn settle_worker_poll_response<T>(&self, response: Result<T>) -> Result<Option<T>> {
7080 match response {
7081 Ok(response) => Ok(Some(response)),
7082 Err(error) => {
7083 let Some(advertised_delay) = worker_poll_capacity_retry_after(&error) else {
7084 return Err(error);
7085 };
7086 let minimum_delay = self
7087 .retry_policy
7088 .initial_backoff
7089 .max(Duration::from_millis(1));
7090 let maximum_delay = self.retry_policy.max_backoff.max(minimum_delay);
7091 tokio::time::sleep(advertised_delay.max(minimum_delay).min(maximum_delay)).await;
7092 Ok(None)
7093 }
7094 }
7095 }
7096
7097 async fn execute_query_task(
7098 &self,
7099 mut task: QueryTask,
7100 ) -> std::result::Result<AvroValue, QueryTaskExecutionFailure> {
7101 validate_query_task_payloads(&task).map_err(|error| {
7102 QueryTaskExecutionFailure::new(
7103 "query_payload_decode_failed",
7104 error.to_string(),
7105 "QueryPayloadDecodeFailed",
7106 )
7107 })?;
7108
7109 if !self.workflows.contains_key(&task.workflow_type) {
7110 return Err(QueryTaskExecutionFailure::new(
7111 "query_workflow_type_not_registered",
7112 format!("no workflow registered for type {:?}", task.workflow_type),
7113 "WorkflowTypeNotRegistered",
7114 ));
7115 }
7116
7117 let Some(handlers) = self.queries.get(&task.workflow_type) else {
7118 return Err(QueryTaskExecutionFailure::new(
7119 "query_handler_unavailable",
7120 format!(
7121 "query handlers are unavailable for workflow type {:?}",
7122 task.workflow_type
7123 ),
7124 "QueryHandlerUnavailable",
7125 ));
7126 };
7127 let Some(query) = handlers.get(&task.query_name) else {
7128 return Err(QueryTaskExecutionFailure::new(
7129 "rejected_unknown_query",
7130 format!("unknown query {:?}", task.query_name),
7131 "QueryFailed",
7132 ));
7133 };
7134
7135 let args = decode_task_avro_arguments(task.query_arguments.as_ref(), &task.payload_codec)
7136 .map_err(|error| {
7137 QueryTaskExecutionFailure::new(
7138 "query_payload_decode_failed",
7139 format!("cannot decode query arguments: {error}"),
7140 "QueryPayloadDecodeFailed",
7141 )
7142 })?;
7143 let workflow_input_typed =
7144 decode_task_avro_arguments(task.workflow_arguments.as_ref(), &task.payload_codec)
7145 .map_err(|error| {
7146 QueryTaskExecutionFailure::new(
7147 "query_workflow_state_unavailable",
7148 format!("cannot decode workflow start input: {error}"),
7149 "QueryWorkflowStateUnavailable",
7150 )
7151 })?;
7152 let workflow_input = workflow_input_typed.clone().into_json().map_err(|error| {
7153 QueryTaskExecutionFailure::new(
7154 "query_workflow_state_unavailable",
7155 format!("cannot project workflow start input: {error}"),
7156 "QueryWorkflowStateUnavailable",
7157 )
7158 })?;
7159 hydrate_query_history_from_export(&mut task).map_err(|error| {
7160 QueryTaskExecutionFailure::new(
7161 "query_workflow_state_unavailable",
7162 format!("cannot restore query history snapshot: {error}"),
7163 "QueryWorkflowStateUnavailable",
7164 )
7165 })?;
7166 enrich_query_history_from_export(&mut task).map_err(|error| {
7167 QueryTaskExecutionFailure::new(
7168 "query_workflow_state_unavailable",
7169 format!("cannot restore compact query history payloads: {error}"),
7170 "QueryWorkflowStateUnavailable",
7171 )
7172 })?;
7173 let signal_events = query_signal_events(&task).map_err(|error| {
7174 QueryTaskExecutionFailure::new(
7175 "query_workflow_state_unavailable",
7176 format!("cannot decode committed workflow signals: {error}"),
7177 "QueryWorkflowStateUnavailable",
7178 )
7179 })?;
7180 let history_events = Arc::new(std::mem::take(&mut task.history_events));
7181 let context = QueryContext {
7182 workflow_id: task.workflow_id,
7183 run_id: task.run_id,
7184 workflow_type: task.workflow_type.clone(),
7185 run_status: task.run_status,
7186 workflow_input,
7187 workflow_input_avro_value: workflow_input_typed.clone(),
7188 history_events: Arc::clone(&history_events),
7189 signal_events: Arc::new(signal_events),
7190 };
7191
7192 let future = match query {
7193 RegisteredQuery::Snapshot(handler) => handler(context, args),
7194 RegisteredQuery::Replayed {
7195 state_type,
7196 handler,
7197 } => {
7198 let workflow = self
7199 .workflows
7200 .get(&task.workflow_type)
7201 .expect("workflow registration was checked above");
7202 if workflow.state_type != Some(*state_type) {
7203 return Err(QueryTaskExecutionFailure::new(
7204 "query_workflow_state_unavailable",
7205 "replayed query state type does not match its workflow registration",
7206 "QueryWorkflowStateUnavailable",
7207 ));
7208 }
7209 let replay = workflow.replay.as_ref().ok_or_else(|| {
7210 QueryTaskExecutionFailure::new(
7211 "query_workflow_state_unavailable",
7212 format!(
7213 "workflow type {:?} is not registered for instance-state replay",
7214 task.workflow_type
7215 ),
7216 "QueryWorkflowStateUnavailable",
7217 )
7218 })?;
7219 let workflow_state = Arc::new(Mutex::new(
7220 WorkflowState::new_with_identity(
7221 history_events.as_ref().clone(),
7222 context.workflow_id.clone(),
7223 context.run_id.clone(),
7224 self.task_queue.clone(),
7225 task.payload_codec,
7226 None,
7227 )
7228 .map_err(|error| {
7229 QueryTaskExecutionFailure::new(
7230 "query_workflow_state_unavailable",
7231 format!("workflow replay failed before query: {error}"),
7232 "QueryWorkflowStateUnavailable",
7233 )
7234 })?,
7235 ));
7236 let workflow_context = WorkflowContext {
7237 state: workflow_state,
7238 };
7239 let mut invocation = replay(workflow_context.clone(), workflow_input_typed.clone());
7240 let mut cx = TaskContext::from_waker(noop_waker_ref());
7241 match invocation.future.as_mut().poll(&mut cx) {
7242 Poll::Ready(Ok(_)) => {
7243 workflow_context
7244 .ensure_history_consumed()
7245 .map_err(|error| {
7246 QueryTaskExecutionFailure::new(
7247 "query_workflow_state_unavailable",
7248 format!("workflow replay failed before query: {error}"),
7249 "QueryWorkflowStateUnavailable",
7250 )
7251 })?;
7252 }
7253 Poll::Ready(Err(error)) => {
7254 return Err(QueryTaskExecutionFailure::new(
7255 "query_workflow_state_unavailable",
7256 format!("workflow replay failed before query: {error}"),
7257 "QueryWorkflowStateUnavailable",
7258 ));
7259 }
7260 Poll::Pending => {
7261 let commands = workflow_context.take_commands().map_err(|error| {
7262 QueryTaskExecutionFailure::new(
7263 "query_workflow_state_unavailable",
7264 format!("workflow replay failed before query: {error}"),
7265 "QueryWorkflowStateUnavailable",
7266 )
7267 })?;
7268 if commands.is_empty()
7269 && !workflow_context
7270 .matched_recorded_pending()
7271 .map_err(|error| {
7272 QueryTaskExecutionFailure::new(
7273 "query_workflow_state_unavailable",
7274 format!("workflow replay failed before query: {error}"),
7275 "QueryWorkflowStateUnavailable",
7276 )
7277 })?
7278 {
7279 return Err(QueryTaskExecutionFailure::new(
7280 "query_workflow_state_unavailable",
7281 "workflow replay yielded without a durable command",
7282 "QueryWorkflowStateUnavailable",
7283 ));
7284 }
7285 }
7286 }
7287 let state = (invocation.snapshot)().map_err(|error| {
7288 QueryTaskExecutionFailure::new(
7289 "query_workflow_state_unavailable",
7290 format!("cannot snapshot replayed workflow state: {error}"),
7291 "QueryWorkflowStateUnavailable",
7292 )
7293 })?;
7294 handler(context, state, args).map_err(|message| {
7295 QueryTaskExecutionFailure::new(
7296 "query_workflow_state_unavailable",
7297 message,
7298 "QueryWorkflowStateUnavailable",
7299 )
7300 })?
7301 }
7302 };
7303
7304 future.await.map_err(|error| {
7305 QueryTaskExecutionFailure::new("query_rejected", error.to_string(), "QueryFailed")
7306 })
7307 }
7308
7309 #[cfg(test)]
7310 fn execute_workflow_task(&self, task: WorkflowTask) -> Result<Vec<Value>> {
7311 Ok(self.execute_workflow_task_decision(task)?.commands)
7312 }
7313
7314 fn execute_workflow_task_decision(&self, task: WorkflowTask) -> Result<WorkflowTaskDecision> {
7315 validate_workflow_task_payloads(&task)?;
7316
7317 if let Some(update_id) = task
7318 .workflow_update_id
7319 .as_deref()
7320 .filter(|update_id| !update_id.is_empty())
7321 {
7322 return self
7323 .execute_update_task(&task, update_id)
7324 .map(WorkflowTaskDecision::without_message_streams);
7325 }
7326
7327 let workflow = self
7328 .workflows
7329 .get(&task.workflow_type)
7330 .ok_or_else(|| Error::WorkflowNotRegistered(task.workflow_type.clone()))?;
7331 let input = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
7332 let resume_signal = decode_resume_signal(&task)?;
7333 let history_budget = WorkflowHistoryBudget {
7334 event_count: task
7335 .total_history_events
7336 .unwrap_or_else(|| u64::try_from(task.history_events.len()).unwrap_or(u64::MAX)),
7337 size_bytes: task.history_size_bytes,
7338 continue_as_new_recommended: task.continue_as_new_recommended.unwrap_or(false),
7339 pressure: task.history_budget_pressure.clone(),
7340 };
7341 let workflow_command_identity = task
7342 .workflow_command_id
7343 .clone()
7344 .filter(|identity| !identity.is_empty())
7345 .unwrap_or_default();
7346 let mut workflow_state = WorkflowState::new_with_identity(
7347 task.history_events,
7348 task.workflow_id,
7349 task.run_id,
7350 self.task_queue.clone(),
7351 task.payload_codec.clone(),
7352 resume_signal,
7353 )?;
7354 workflow_state.history_budget = history_budget;
7355 workflow_state.workflow_command_identity = workflow_command_identity;
7356 workflow_state.cancel_requested = task.cancel_requested;
7357 let state = Arc::new(Mutex::new(workflow_state));
7358 let ctx = WorkflowContext { state };
7359 let mut future = (workflow.execute)(ctx.clone(), input);
7360 let mut cx = TaskContext::from_waker(noop_waker_ref());
7361
7362 match future.as_mut().poll(&mut cx) {
7363 Poll::Ready(Ok(result)) => {
7364 ctx.ensure_history_consumed()?;
7365 let result = encode_typed_envelope(&result, &task.payload_codec)?;
7366 let mut commands = ctx.take_commands()?;
7367 commands.push(json!({
7368 "type": "complete_workflow",
7369 "result": result
7370 }));
7371 self.message_stream_decision(&ctx, commands)
7372 }
7373 Poll::Ready(Err(error)) => {
7374 if let Error::ContinueAsNew(request) = error {
7375 let mut commands = ctx.take_commands()?;
7376 if let Some(command) = ctx.continue_as_new_command(request)? {
7377 commands.push(command);
7378 }
7379 ctx.ensure_history_consumed()?;
7380 return self.message_stream_decision(&ctx, commands);
7381 }
7382 if workflow_task_integrity_error(&error) {
7383 return Err(error);
7388 }
7389 ctx.ensure_history_consumed()?;
7392 let mut commands = ctx.take_commands()?;
7393 commands.push(workflow_failure_command(&error));
7394 self.message_stream_decision(&ctx, commands)
7395 }
7396 Poll::Pending => {
7397 let commands = ctx.take_commands()?;
7398 if commands.is_empty() && !ctx.matched_recorded_pending()? {
7399 Err(Error::WorkflowYieldedWithoutCommand)
7400 } else {
7401 self.message_stream_decision(&ctx, commands)
7402 }
7403 }
7404 }
7405 }
7406
7407 fn message_stream_decision(
7408 &self,
7409 ctx: &WorkflowContext,
7410 commands: Vec<Value>,
7411 ) -> Result<WorkflowTaskDecision> {
7412 let (message_stream_cursors, message_stream_waits) = ctx.message_stream_metadata()?;
7413 Ok(WorkflowTaskDecision {
7414 commands,
7415 message_stream_cursors,
7416 message_stream_waits,
7417 })
7418 }
7419
7420 fn execute_update_task(&self, task: &WorkflowTask, update_id: &str) -> Result<Vec<Value>> {
7421 if !self.workflows.contains_key(&task.workflow_type) {
7422 return Err(Error::WorkflowNotRegistered(task.workflow_type.clone()));
7423 }
7424
7425 let accepted = task.history_events.iter().rev().find_map(|event| {
7426 (event.event_type == "UpdateAccepted"
7427 && event.payload.get("update_id").and_then(Value::as_str) == Some(update_id))
7428 .then_some(&event.payload)
7429 });
7430 let update_name = accepted
7431 .and_then(|payload| payload.get("update_name"))
7432 .and_then(Value::as_str)
7433 .or(task.update_name.as_deref())
7434 .unwrap_or_default();
7435 let Some(handler) = self
7436 .updates
7437 .get(&task.workflow_type)
7438 .and_then(|handlers| handlers.get(update_name))
7439 else {
7440 return Ok(vec![json!({
7441 "type": "fail_update",
7442 "update_id": update_id,
7443 "message": format!(
7444 "no update handler is registered for {}.{update_name}",
7445 task.workflow_type
7446 ),
7447 "exception_type": "UnknownUpdate",
7448 "non_retryable": true,
7449 })]);
7450 };
7451 let arguments = accepted
7452 .and_then(|payload| payload.get("arguments"))
7453 .or(task.arguments.as_ref());
7454 let arguments = decode_task_avro_arguments(arguments, &task.payload_codec)?;
7455 let context = QueryContext {
7456 workflow_id: task.workflow_id.clone(),
7457 run_id: task.run_id.clone(),
7458 workflow_type: task.workflow_type.clone(),
7459 run_status: Some("running".to_string()),
7460 workflow_input: Value::Null,
7461 workflow_input_avro_value: AvroValue::Null,
7462 history_events: Arc::new(task.history_events.clone()),
7463 signal_events: Arc::new(Vec::new()),
7464 };
7465 let mut future = handler(context, arguments);
7466 let mut cx = TaskContext::from_waker(noop_waker_ref());
7467
7468 match future.as_mut().poll(&mut cx) {
7469 Poll::Ready(Ok(result)) => Ok(vec![json!({
7470 "type": "complete_update",
7471 "update_id": update_id,
7472 "result": encode_typed_envelope(&result, &task.payload_codec)?,
7473 })]),
7474 Poll::Ready(Err(error)) => Ok(vec![json!({
7475 "type": "fail_update",
7476 "update_id": update_id,
7477 "message": error.to_string(),
7478 "exception_type": "UpdateFailed",
7479 "non_retryable": true,
7480 })]),
7481 Poll::Pending => Err(Error::WorkflowYieldedWithoutCommand),
7482 }
7483 }
7484
7485 async fn execute_activity_task(&self, task: ActivityTask) -> Result<AvroValue> {
7486 validate_activity_task_payloads(&task)?;
7487
7488 let handler = self
7489 .activities
7490 .get(&task.activity_type)
7491 .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?;
7492 let args = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
7493 let attempt_id = task
7494 .activity_attempt_id
7495 .clone()
7496 .or(task.attempt_id.clone())
7497 .unwrap_or_default();
7498 let lease_owner = task
7499 .lease_owner
7500 .clone()
7501 .unwrap_or_else(|| self.worker_id.clone());
7502 let ctx = ActivityContext {
7503 client: self.client.clone(),
7504 task_id: task.task_id,
7505 activity_attempt_id: attempt_id,
7506 lease_owner,
7507 activity_type: task.activity_type,
7508 attempt_number: task.attempt_number,
7509 task_queue: self.task_queue.clone(),
7510 worker_id: self.worker_id.clone(),
7511 };
7512
7513 handler(ctx, args).await
7514 }
7515}
7516
7517fn poller_result(
7518 kind: &str,
7519 result: std::result::Result<Result<()>, tokio::task::JoinError>,
7520) -> Result<()> {
7521 match result {
7522 Ok(result) => result,
7523 Err(error) => Err(Error::WorkerLoop(format!(
7524 "{kind} poller join error: {error}"
7525 ))),
7526 }
7527}
7528
7529fn optional_poller_result(
7530 kind: &str,
7531 result: Option<std::result::Result<Result<()>, tokio::task::JoinError>>,
7532) -> Result<()> {
7533 match result {
7534 Some(result) => poller_result(kind, result),
7535 None => Ok(()),
7536 }
7537}
7538
7539async fn join_pollers(
7540 workflow_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7541 activity_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7542 query_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7543) -> Result<()> {
7544 let mut first_error = None;
7545
7546 if let Some(handle) = workflow_poller {
7547 if let Err(error) = poller_result("workflow", handle.await) {
7548 first_error.get_or_insert(error);
7549 }
7550 }
7551
7552 if let Some(handle) = activity_poller {
7553 if let Err(error) = poller_result("activity", handle.await) {
7554 first_error.get_or_insert(error);
7555 }
7556 }
7557
7558 if let Some(handle) = query_poller {
7559 if let Err(error) = poller_result("query", handle.await) {
7560 first_error.get_or_insert(error);
7561 }
7562 }
7563
7564 if let Some(error) = first_error {
7565 Err(error)
7566 } else {
7567 Ok(())
7568 }
7569}
7570
7571fn default_worker_id() -> String {
7572 let millis = SystemTime::now()
7573 .duration_since(UNIX_EPOCH)
7574 .unwrap_or_default()
7575 .as_millis();
7576 format!("rust-worker-{}-{millis}", std::process::id())
7577}
7578
7579fn percent_encode_path_segment(segment: &str) -> String {
7580 const HEX: &[u8; 16] = b"0123456789ABCDEF";
7581 let mut encoded = String::with_capacity(segment.len());
7582
7583 for byte in segment.bytes() {
7584 if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') {
7585 encoded.push(char::from(byte));
7586 } else {
7587 encoded.push('%');
7588 encoded.push(char::from(HEX[(byte >> 4) as usize]));
7589 encoded.push(char::from(HEX[(byte & 0x0f) as usize]));
7590 }
7591 }
7592
7593 encoded
7594}
7595
7596fn unique_request_id(prefix: &str) -> String {
7597 let nanos = SystemTime::now()
7598 .duration_since(UNIX_EPOCH)
7599 .unwrap_or_default()
7600 .as_nanos();
7601 format!("{prefix}-{}-{nanos}", std::process::id())
7602}
7603
7604#[derive(Debug)]
7605struct QueryTaskExecutionFailure {
7606 reason: String,
7607 message: String,
7608 failure_type: String,
7609}
7610
7611impl QueryTaskExecutionFailure {
7612 fn new(
7613 reason: impl Into<String>,
7614 message: impl Into<String>,
7615 failure_type: impl Into<String>,
7616 ) -> Self {
7617 Self {
7618 reason: reason.into(),
7619 message: message.into(),
7620 failure_type: failure_type.into(),
7621 }
7622 }
7623}
7624
7625#[derive(Clone, Debug)]
7631pub struct WorkflowInstance<S> {
7632 state: Arc<Mutex<S>>,
7633}
7634
7635impl<S> WorkflowInstance<S> {
7636 fn new(state: S) -> Self {
7637 Self {
7638 state: Arc::new(Mutex::new(state)),
7639 }
7640 }
7641
7642 pub fn read<R>(&self, reader: impl FnOnce(&S) -> R) -> Result<R> {
7644 let state = self
7645 .state
7646 .lock()
7647 .map_err(|_| Error::WorkflowStatePoisoned)?;
7648 Ok(reader(&state))
7649 }
7650
7651 pub fn update<R>(&self, transition: impl FnOnce(&mut S) -> R) -> Result<R> {
7653 let mut state = self
7654 .state
7655 .lock()
7656 .map_err(|_| Error::WorkflowStatePoisoned)?;
7657 Ok(transition(&mut state))
7658 }
7659}
7660
7661impl<S: Clone> WorkflowInstance<S> {
7662 fn snapshot(&self) -> Result<S> {
7663 self.read(Clone::clone)
7664 }
7665}
7666
7667#[derive(Clone, Debug, PartialEq)]
7668pub struct MessageStreamMessage {
7669 pub stream_name: String,
7670 pub message_id: String,
7671 pub position: u64,
7672 pub arguments: Vec<AvroValue>,
7673}
7674
7675#[derive(Clone, Debug)]
7676pub struct MessageStream {
7677 ctx: WorkflowContext,
7678 name: String,
7679}
7680
7681impl MessageStream {
7682 pub async fn receive(&self, max_items: usize) -> Result<Vec<MessageStreamMessage>> {
7684 if !(1..=MESSAGE_STREAM_MAX_BATCH).contains(&max_items) {
7685 return Err(Error::Codec(format!(
7686 "message stream max_items must be between 1 and {MESSAGE_STREAM_MAX_BATCH}"
7687 )));
7688 }
7689 loop {
7690 if let Some(batch) = self.ctx.take_message_stream_batch(&self.name, max_items)? {
7691 return Ok(batch);
7692 }
7693
7694 self.ctx.record_message_stream_wait(&self.name)?;
7695 let replay_wait_sequence = self.ctx.next_message_stream_wait_sequence()?;
7696 let arguments = self.ctx.wait_runtime_signal(MESSAGE_STREAM_SIGNAL).await?;
7697 self.ctx.buffer_message_stream_delivery(arguments)?;
7698 if let Some(sequence) = replay_wait_sequence {
7699 self.ctx.buffer_message_stream_history_for_wait(sequence)?;
7700 }
7701 }
7702 }
7703
7704 pub async fn receive_one(&self) -> Result<MessageStreamMessage> {
7705 self.receive(1)
7706 .await?
7707 .into_iter()
7708 .next()
7709 .ok_or_else(|| Error::Codec("message stream resumed without a message".to_string()))
7710 }
7711}
7712
7713#[derive(Clone, Debug)]
7714pub struct WorkflowContext {
7715 state: Arc<Mutex<WorkflowState>>,
7716}
7717
7718fn valid_memo_key(key: &str) -> bool {
7719 let numeric_candidate = key.strip_prefix('-').unwrap_or(key);
7720
7721 !key.is_empty()
7722 && key.len() <= 64
7723 && (numeric_candidate.is_empty()
7724 || !numeric_candidate.bytes().all(|byte| byte.is_ascii_digit()))
7725 && key
7726 .bytes()
7727 .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b':' | b'-'))
7728}
7729
7730fn avro_encoded_size(value: &AvroValue) -> Result<usize> {
7731 BASE64
7732 .decode(encode_avro_value(value)?.blob)
7733 .map(|bytes| bytes.len())
7734 .map_err(|error| Error::Codec(format!("memo Avro encoding was not strict base64: {error}")))
7735}
7736
7737fn canonical_memo_entries(value: AvroValue, require_entries: bool) -> Result<AvroValue> {
7738 let AvroValue::Map(entries) = value else {
7739 return Err(Error::InvalidMemoUpdate(
7740 "entries must serialize to an Avro string-keyed map".to_string(),
7741 ));
7742 };
7743 if require_entries && entries.is_empty() {
7744 return Err(Error::InvalidMemoUpdate(
7745 "at least one entry is required".to_string(),
7746 ));
7747 }
7748 if entries.len() > MAX_MEMO_ENTRIES {
7749 return Err(Error::InvalidMemoUpdate(format!(
7750 "at most {MAX_MEMO_ENTRIES} entries are allowed"
7751 )));
7752 }
7753
7754 for (key, value) in &entries {
7755 if !valid_memo_key(&key) {
7756 return Err(Error::InvalidMemoUpdate(
7757 "keys must match ^(?!-?[0-9]+$)[A-Za-z0-9_.:-]{1,64}$".to_string(),
7758 ));
7759 }
7760 if avro_encoded_size(value)? > MAX_MEMO_VALUE_SIZE_BYTES {
7761 return Err(Error::InvalidMemoUpdate(format!(
7762 "value {key:?} exceeds the {MAX_MEMO_VALUE_SIZE_BYTES}-byte limit"
7763 )));
7764 }
7765 }
7766
7767 let value = AvroValue::Map(entries);
7768 if avro_encoded_size(&value)? > MAX_MEMO_TOTAL_SIZE_BYTES {
7769 return Err(Error::InvalidMemoUpdate(format!(
7770 "update exceeds the {MAX_MEMO_TOTAL_SIZE_BYTES}-byte total limit"
7771 )));
7772 }
7773 Ok(value)
7774}
7775
7776fn decode_memo_history_map(envelope: &Value, require_entries: bool) -> Result<AvroValue> {
7777 let object = envelope.as_object().ok_or_else(|| {
7778 Error::InvalidMemoUpdate(
7779 "history field must use the public {codec, blob} payload envelope".to_string(),
7780 )
7781 })?;
7782 if object.len() != 2 || !object.contains_key("codec") || !object.contains_key("blob") {
7783 return Err(Error::InvalidMemoUpdate(
7784 "history field must use exactly the public {codec, blob} payload envelope".to_string(),
7785 ));
7786 }
7787
7788 canonical_memo_entries(
7789 decode_wire_avro_value(envelope, DEFAULT_CODEC)?,
7790 require_entries,
7791 )
7792}
7793
7794impl WorkflowContext {
7795 pub fn message_stream(&self, name: impl Into<String>) -> Result<MessageStream> {
7796 let name = name.into();
7797 if name.is_empty()
7798 || name.len() > 128
7799 || !name.bytes().all(|byte| {
7800 byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b':' | b'-')
7801 })
7802 {
7803 return Err(Error::Codec(
7804 "message stream names must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens"
7805 .to_string(),
7806 ));
7807 }
7808 Ok(MessageStream {
7809 ctx: self.clone(),
7810 name,
7811 })
7812 }
7813
7814 fn record_message_stream_wait(&self, name: &str) -> Result<()> {
7815 let mut state = self
7816 .state
7817 .lock()
7818 .map_err(|_| Error::WorkflowStatePoisoned)?;
7819 let position = state.message_stream_cursors.get(name).copied().unwrap_or(0);
7820 state
7821 .message_stream_waits
7822 .insert(name.to_string(), position);
7823 Ok(())
7824 }
7825
7826 fn buffer_message_stream(&self, message: MessageStreamMessage) -> Result<()> {
7827 let mut state = self
7828 .state
7829 .lock()
7830 .map_err(|_| Error::WorkflowStatePoisoned)?;
7831 let cursor = state
7832 .message_stream_cursors
7833 .get(&message.stream_name)
7834 .copied()
7835 .unwrap_or(0);
7836 if message.position <= cursor {
7837 return Ok(());
7838 }
7839 let pending = state
7840 .message_stream_messages
7841 .entry(message.stream_name.clone())
7842 .or_default();
7843 if pending.iter().any(|candidate| {
7844 candidate.position == message.position || candidate.message_id == message.message_id
7845 }) {
7846 return Ok(());
7847 }
7848 pending.push(message);
7849 pending.sort_by_key(|candidate| candidate.position);
7850 Ok(())
7851 }
7852
7853 fn buffer_message_stream_delivery(&self, arguments: Vec<Value>) -> Result<Option<String>> {
7854 if let Some(delivery) = decode_message_stream_delivery(arguments)? {
7855 match delivery {
7856 MessageStreamDelivery::Message(message) => {
7857 let stream_name = message.stream_name.clone();
7858 self.buffer_message_stream(message)?;
7859 return Ok(Some(stream_name));
7860 }
7861 MessageStreamDelivery::Cursor {
7862 stream_name,
7863 through_position,
7864 } => self.apply_message_stream_cursor(&stream_name, through_position)?,
7865 }
7866 }
7867 Ok(None)
7868 }
7869
7870 fn next_message_stream_wait_sequence(&self) -> Result<Option<u64>> {
7871 let state = self
7872 .state
7873 .lock()
7874 .map_err(|_| Error::WorkflowStatePoisoned)?;
7875 Ok(match state.recorded_commands.get(state.command_cursor) {
7876 Some(RecordedCommand::SignalWait {
7877 sequence,
7878 signal_name,
7879 ..
7880 }) if signal_name == MESSAGE_STREAM_SIGNAL => Some(*sequence),
7881 _ => None,
7882 })
7883 }
7884
7885 fn buffer_message_stream_history_for_wait(&self, wait_sequence: u64) -> Result<()> {
7886 let (history, payload_codec) = {
7887 let state = self
7888 .state
7889 .lock()
7890 .map_err(|_| Error::WorkflowStatePoisoned)?;
7891 (
7892 Arc::clone(&state.history_events),
7893 state.payload_codec.clone(),
7894 )
7895 };
7896
7897 let Some(opened_index) = history.iter().position(|event| {
7898 event.event_type == "SignalWaitOpened"
7899 && durable_event_sequence(event) == Some(wait_sequence)
7900 && event.payload.get("signal_name").and_then(Value::as_str)
7901 == Some(MESSAGE_STREAM_SIGNAL)
7902 }) else {
7903 return Ok(());
7904 };
7905 let boundary_index = history
7906 .iter()
7907 .enumerate()
7908 .skip(opened_index + 1)
7909 .find_map(|(index, event)| {
7910 (durable_event_sequence(event).is_some_and(|sequence| sequence > wait_sequence)
7911 && is_authored_command_open_event(event))
7912 .then_some(index)
7913 })
7914 .unwrap_or(history.len());
7915
7916 for event in history[opened_index + 1..boundary_index]
7917 .iter()
7918 .filter(|event| {
7919 event.event_type == "SignalReceived"
7920 && event.payload.get("signal_name").and_then(Value::as_str)
7921 == Some(MESSAGE_STREAM_SIGNAL)
7922 })
7923 {
7924 let arguments = decode_signal_event_arguments(event, &payload_codec)?
7925 .into_iter()
7926 .map(AvroValue::into_json)
7927 .collect::<Result<Vec<_>>>()?;
7928 self.buffer_message_stream_delivery(arguments)?;
7929 }
7930 Ok(())
7931 }
7932
7933 fn apply_message_stream_cursor(&self, name: &str, through_position: u64) -> Result<()> {
7934 let mut state = self
7935 .state
7936 .lock()
7937 .map_err(|_| Error::WorkflowStatePoisoned)?;
7938 let cursor = state
7939 .message_stream_cursors
7940 .entry(name.to_string())
7941 .or_default();
7942 *cursor = (*cursor).max(through_position);
7943 if let Some(pending) = state.message_stream_messages.get_mut(name) {
7944 pending.retain(|message| message.position > through_position);
7945 }
7946 Ok(())
7947 }
7948
7949 fn take_message_stream_batch(
7950 &self,
7951 name: &str,
7952 max_items: usize,
7953 ) -> Result<Option<Vec<MessageStreamMessage>>> {
7954 let mut state = self
7955 .state
7956 .lock()
7957 .map_err(|_| Error::WorkflowStatePoisoned)?;
7958 let cursor = state.message_stream_cursors.get(name).copied().unwrap_or(0);
7959 let pending = state
7960 .message_stream_messages
7961 .entry(name.to_string())
7962 .or_default();
7963 let count = contiguous_message_stream_count(pending, cursor, max_items);
7964 if count == 0 {
7965 return Ok(None);
7966 }
7967 let batch = pending.drain(..count).collect::<Vec<_>>();
7968 let position = batch.last().map(|message| message.position).unwrap_or(0);
7969 state
7970 .message_stream_cursors
7971 .insert(name.to_string(), position);
7972 state.message_stream_waits.remove(name);
7973 Ok(Some(batch))
7974 }
7975
7976 fn message_stream_metadata(&self) -> Result<(Vec<Value>, Vec<Value>)> {
7977 let state = self
7978 .state
7979 .lock()
7980 .map_err(|_| Error::WorkflowStatePoisoned)?;
7981 let mut cursors = state.message_stream_cursors.iter().collect::<Vec<_>>();
7982 cursors.sort_by_key(|(name, _)| *name);
7983 let mut waits = state.message_stream_waits.iter().collect::<Vec<_>>();
7984 waits.sort_by_key(|(name, _)| *name);
7985 Ok((
7986 cursors
7987 .into_iter()
7988 .map(|(name, position)| json!({"stream_name": name, "through_position": position}))
7989 .collect(),
7990 waits
7991 .into_iter()
7992 .map(|(name, position)| json!({"stream_name": name, "after_position": position}))
7993 .collect(),
7994 ))
7995 }
7996 pub fn workflow_identity(&self) -> Result<WorkflowIdentity> {
7998 let state = self
7999 .state
8000 .lock()
8001 .map_err(|_| Error::WorkflowStatePoisoned)?;
8002 Ok(WorkflowIdentity {
8003 workflow_id: state.workflow_id.clone(),
8004 run_id: state.run_id.clone(),
8005 })
8006 }
8007
8008 pub fn history_budget(&self) -> Result<WorkflowHistoryBudget> {
8010 let state = self
8011 .state
8012 .lock()
8013 .map_err(|_| Error::WorkflowStatePoisoned)?;
8014 Ok(state.history_budget.clone())
8015 }
8016
8017 pub fn continue_as_new<T: Serialize>(&self, args: T) -> Result<Value> {
8023 self.continue_as_new_with_options(ContinueAsNewOptions::new(), args)
8024 }
8025
8026 pub fn continue_as_new_with_options<T: Serialize>(
8028 &self,
8029 options: ContinueAsNewOptions,
8030 args: T,
8031 ) -> Result<Value> {
8032 options.validate()?;
8033 Err(Error::ContinueAsNew(ContinueAsNewRequest {
8034 arguments: normalize_avro_arguments(AvroValue::from_serialize(&args)?),
8035 options,
8036 }))
8037 }
8038
8039 pub fn activity<T: Serialize>(
8040 &self,
8041 activity_type: impl Into<String>,
8042 args: T,
8043 ) -> ActivityCall {
8044 self.activity_with_options(activity_type, ActivityOptions::new(), args)
8045 }
8046
8047 pub fn activity_on_queue<T, Q>(
8048 &self,
8049 activity_type: impl Into<String>,
8050 task_queue: Option<Q>,
8051 args: T,
8052 ) -> ActivityCall
8053 where
8054 T: Serialize,
8055 Q: Into<String>,
8056 {
8057 let mut options = ActivityOptions::new();
8058 options.task_queue = task_queue.map(Into::into);
8059 self.activity_with_options(activity_type, options, args)
8060 }
8061
8062 pub fn activity_with_options<T: Serialize>(
8100 &self,
8101 activity_type: impl Into<String>,
8102 options: ActivityOptions,
8103 args: T,
8104 ) -> ActivityCall {
8105 ActivityCall {
8106 ctx: self.clone(),
8107 activity_type: activity_type.into(),
8108 options,
8109 args: Some(AvroValue::from_serialize(&args)),
8110 scheduled: false,
8111 parallel_group_path: Vec::new(),
8112 }
8113 }
8114
8115 pub async fn activity_avro_value<T: Serialize>(
8116 &self,
8117 activity_type: impl Into<String>,
8118 args: T,
8119 ) -> Result<AvroValue> {
8120 let mut call = self.activity(activity_type, args);
8121 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8122 }
8123
8124 pub async fn activity_avro_value_with_options<T: Serialize>(
8125 &self,
8126 activity_type: impl Into<String>,
8127 options: ActivityOptions,
8128 args: T,
8129 ) -> Result<AvroValue> {
8130 let mut call = self.activity_with_options(activity_type, options, args);
8131 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8132 }
8133
8134 pub async fn activity_typed<I, O>(&self, activity_type: impl Into<String>, args: I) -> Result<O>
8136 where
8137 I: Serialize,
8138 O: DeserializeOwned,
8139 {
8140 self.activity_typed_with_options(activity_type, ActivityOptions::new(), args)
8141 .await
8142 }
8143
8144 pub async fn activity_typed_with_options<I, O>(
8150 &self,
8151 activity_type: impl Into<String>,
8152 options: ActivityOptions,
8153 args: I,
8154 ) -> Result<O>
8155 where
8156 I: Serialize,
8157 O: DeserializeOwned,
8158 {
8159 let activity_type = activity_type.into();
8160 let encoded = AvroValue::from_serialize(&args).map_err(|error| {
8161 handler_type_error::<I>(
8162 HandlerKind::Activity,
8163 &activity_type,
8164 HandlerValueKind::Input,
8165 error.to_string(),
8166 )
8167 });
8168 let mut call = ActivityCall {
8169 ctx: self.clone(),
8170 activity_type: activity_type.clone(),
8171 options,
8172 args: Some(encoded),
8173 scheduled: false,
8174 parallel_group_path: Vec::new(),
8175 };
8176 let result = std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await?;
8177 decode_handler_result(result, HandlerKind::Activity, &activity_type)
8178 }
8179
8180 pub fn parallel(&self, operations: Vec<ParallelOperation>) -> ParallelCall {
8187 ParallelCall::new(self.clone(), operations)
8188 }
8189
8190 pub fn join(&self, operations: Vec<ParallelOperation>) -> ParallelCall {
8192 self.parallel(operations)
8193 }
8194
8195 pub async fn parallel_avro_value(
8197 &self,
8198 operations: Vec<ParallelOperation>,
8199 ) -> Result<Vec<ParallelAvroResult>> {
8200 let mut call = self.parallel(operations);
8201 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8202 }
8203
8204 pub fn select(&self, operations: Vec<ParallelOperation>) -> SelectCall {
8207 let operations = operations
8208 .into_iter()
8209 .enumerate()
8210 .map(|(index, operation)| (SelectionKey::Index(index), operation))
8211 .collect();
8212 SelectCall::new(self.clone(), operations)
8213 }
8214
8215 pub fn select_keyed<K>(&self, operations: Vec<(K, ParallelOperation)>) -> SelectCall
8217 where
8218 K: Into<SelectionKey>,
8219 {
8220 SelectCall::new(
8221 self.clone(),
8222 operations
8223 .into_iter()
8224 .map(|(key, operation)| (key.into(), operation))
8225 .collect(),
8226 )
8227 }
8228
8229 pub fn saga(&self) -> Saga {
8231 Saga::new(self.clone())
8232 }
8233
8234 pub fn is_cancellation_requested(&self) -> Result<bool> {
8236 let state = self
8237 .state
8238 .lock()
8239 .map_err(|_| Error::WorkflowStatePoisoned)?;
8240 Ok(state.cancel_requested)
8241 }
8242
8243 pub fn throw_if_cancellation_requested(&self) -> Result<()> {
8248 if self.is_cancellation_requested()? {
8249 return Err(Error::WorkflowCancellationRequested(
8250 WorkflowCancellationRequested,
8251 ));
8252 }
8253 Ok(())
8254 }
8255
8256 pub fn wait_signal(&self, signal_name: impl Into<String>) -> SignalCall {
8257 SignalCall {
8258 ctx: self.clone(),
8259 signal_name: signal_name.into(),
8260 runtime_reserved_allowed: false,
8261 opened_wait: false,
8262 matched_pending: false,
8263 parallel_group_path: Vec::new(),
8264 }
8265 }
8266
8267 fn wait_runtime_signal(&self, signal_name: impl Into<String>) -> SignalCall {
8268 SignalCall {
8269 ctx: self.clone(),
8270 signal_name: signal_name.into(),
8271 runtime_reserved_allowed: true,
8272 opened_wait: false,
8273 matched_pending: false,
8274 parallel_group_path: Vec::new(),
8275 }
8276 }
8277
8278 pub async fn wait_signal_avro_value(
8279 &self,
8280 signal_name: impl Into<String>,
8281 ) -> Result<Vec<AvroValue>> {
8282 let mut call = self.wait_signal(signal_name);
8283 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8284 }
8285
8286 pub fn signals(&self, signal_name: &str) -> Result<Vec<Vec<Value>>> {
8292 self.signals_avro_value(signal_name)?
8293 .into_iter()
8294 .map(|arguments| {
8295 arguments
8296 .into_iter()
8297 .map(AvroValue::into_json)
8298 .collect::<Result<Vec<_>>>()
8299 })
8300 .collect()
8301 }
8302
8303 pub fn signals_avro_value(&self, signal_name: &str) -> Result<Vec<Vec<AvroValue>>> {
8305 let state = self
8306 .state
8307 .lock()
8308 .map_err(|_| Error::WorkflowStatePoisoned)?;
8309 state
8310 .history_events
8311 .iter()
8312 .filter(|event| {
8313 event.event_type == "SignalReceived"
8314 && event.payload.get("signal_name").and_then(Value::as_str) == Some(signal_name)
8315 })
8316 .map(|event| decode_signal_event_arguments(event, &state.payload_codec))
8317 .collect()
8318 }
8319
8320 pub fn updates(&self, update_name: &str) -> Result<Vec<Vec<Value>>> {
8326 self.updates_avro_value(update_name)?
8327 .into_iter()
8328 .map(|arguments| {
8329 arguments
8330 .into_iter()
8331 .map(AvroValue::into_json)
8332 .collect::<Result<Vec<_>>>()
8333 })
8334 .collect()
8335 }
8336
8337 pub fn updates_avro_value(&self, update_name: &str) -> Result<Vec<Vec<AvroValue>>> {
8339 let state = self
8340 .state
8341 .lock()
8342 .map_err(|_| Error::WorkflowStatePoisoned)?;
8343 let mut seen = Vec::new();
8344 let mut updates = Vec::new();
8345 for event in state.history_events.iter() {
8346 if !matches!(
8347 event.event_type.as_str(),
8348 "UpdateAccepted" | "UpdateApplied"
8349 ) || event.payload.get("update_name").and_then(Value::as_str) != Some(update_name)
8350 || event.payload.get("arguments").is_none()
8351 {
8352 continue;
8353 }
8354 if let Some(update_id) = event.payload.get("update_id").and_then(Value::as_str) {
8355 if seen.iter().any(|recorded| recorded == update_id) {
8356 continue;
8357 }
8358 seen.push(update_id.to_string());
8359 }
8360 updates.push(decode_update_event_arguments(event, &state.payload_codec)?);
8361 }
8362 Ok(updates)
8363 }
8364
8365 pub fn wait_condition<F>(
8373 &self,
8374 options: ConditionWaitOptions,
8375 predicate: F,
8376 ) -> ConditionWaitCall
8377 where
8378 F: Fn() -> Result<bool> + Send + 'static,
8379 {
8380 ConditionWaitCall {
8381 ctx: self.clone(),
8382 options,
8383 predicate: Box::new(predicate),
8384 occurrence_id: None,
8385 opened_wait: false,
8386 parallel_group_path: Vec::new(),
8387 }
8388 }
8389
8390 pub fn sleep(&self, duration: Duration) -> TimerCall {
8411 let delay_seconds = duration
8412 .as_secs()
8413 .checked_add(u64::from(duration.subsec_nanos() > 0));
8414 TimerCall {
8415 ctx: self.clone(),
8416 delay_seconds,
8417 scheduled: false,
8418 matched_pending: false,
8419 parallel_group_path: Vec::new(),
8420 }
8421 }
8422
8423 pub fn start_timer(&self, duration: Duration) -> TimerCall {
8425 self.sleep(duration)
8426 }
8427
8428 pub fn side_effect<T, F>(&self, callback: F) -> Result<T>
8435 where
8436 T: Serialize + DeserializeOwned,
8437 F: FnOnce() -> T,
8438 {
8439 {
8440 let mut state = self
8441 .state
8442 .lock()
8443 .map_err(|_| Error::WorkflowStatePoisoned)?;
8444 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8445 return match recorded {
8446 RecordedCommand::SideEffect { sequence, value } => {
8447 state.command_cursor += 1;
8448 value.deserialize().map_err(|error| {
8449 Error::NonDeterministicReplay(ReplayFailure::new(
8450 "side_effect_type_mismatch",
8451 Some(sequence),
8452 Some(std::any::type_name::<T>().to_string()),
8453 Some(error.to_string()),
8454 "recorded side-effect value is incompatible with the requested Rust type",
8455 ))
8456 })
8457 }
8458 other => Err(command_mismatch(&other, "side effect")),
8459 };
8460 }
8461 }
8462
8463 let value = callback();
8464 let avro_value = AvroValue::from_serialize(&value)?;
8465 let mut state = self
8466 .state
8467 .lock()
8468 .map_err(|_| Error::WorkflowStatePoisoned)?;
8469 let result = encode_typed_envelope(&avro_value, &state.payload_codec)?;
8470 state.commands.push(json!({
8471 "type": "record_side_effect",
8472 "result": result,
8473 }));
8474 Ok(value)
8475 }
8476
8477 pub fn side_effect_avro_value<F>(&self, callback: F) -> Result<AvroValue>
8479 where
8480 F: FnOnce() -> AvroValue,
8481 {
8482 {
8483 let mut state = self
8484 .state
8485 .lock()
8486 .map_err(|_| Error::WorkflowStatePoisoned)?;
8487 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8488 return match recorded {
8489 RecordedCommand::SideEffect { value, .. } => {
8490 state.command_cursor += 1;
8491 Ok(value)
8492 }
8493 other => Err(command_mismatch(&other, "side effect")),
8494 };
8495 }
8496 }
8497
8498 let value = callback();
8499 let mut state = self
8500 .state
8501 .lock()
8502 .map_err(|_| Error::WorkflowStatePoisoned)?;
8503 let result = encode_typed_envelope(&value, &state.payload_codec)?;
8504 state.commands.push(json!({
8505 "type": "record_side_effect",
8506 "result": result,
8507 }));
8508 Ok(value)
8509 }
8510
8511 pub fn append_workflow_stream(
8517 &self,
8518 stream_name: impl Into<String>,
8519 items: &[WorkflowStreamAppendItem],
8520 max_pending_items: Option<u64>,
8521 ) -> Result<()> {
8522 if items.is_empty() {
8523 return Err(Error::Codec(
8524 "workflow_stream_items_empty: append requires at least one item".to_string(),
8525 ));
8526 }
8527 if max_pending_items == Some(0) {
8528 return Err(Error::Codec(
8529 "workflow_stream_pending_limit_invalid: max_pending_items must be positive"
8530 .to_string(),
8531 ));
8532 }
8533 let stream_name = stream_name.into();
8534 if stream_name.is_empty() {
8535 return Err(Error::Codec(
8536 "workflow_stream_name_invalid: stream name must not be empty".to_string(),
8537 ));
8538 }
8539
8540 let mut state = self
8541 .state
8542 .lock()
8543 .map_err(|_| Error::WorkflowStatePoisoned)?;
8544 let command_ordinal = state.workflow_stream_command_counter;
8545 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8546 state.workflow_stream_command_counter += 1;
8547 return match recorded {
8548 RecordedCommand::SideEffect { .. } => {
8549 state.command_cursor += 1;
8550 Ok(())
8551 }
8552 other => Err(command_mismatch(&other, "workflow stream append")),
8553 };
8554 }
8555
8556 let identity = Self::workflow_stream_command_identity(&state)?.to_string();
8557 state.workflow_stream_command_counter += 1;
8558 let wire_items = items
8559 .iter()
8560 .enumerate()
8561 .map(|(item_index, item)| {
8562 item.wire_value(Some(format!(
8563 "dw-stream:{identity}:{command_ordinal}:{item_index}"
8564 )))
8565 })
8566 .collect::<Vec<_>>();
8567 let mut directive = json!({
8568 "operation": "append",
8569 "stream_name": stream_name,
8570 "command_identity": identity,
8571 "command_ordinal": command_ordinal,
8572 "items": wire_items,
8573 });
8574 if let Some(max_pending_items) = max_pending_items {
8575 directive["max_pending_items"] = json!(max_pending_items);
8576 }
8577 let result = encode_typed_envelope(&AvroValue::Null, &state.payload_codec)?;
8578 state.commands.push(json!({
8579 "type": "record_side_effect",
8580 "result": result,
8581 "workflow_stream": directive,
8582 }));
8583 Ok(())
8584 }
8585
8586 pub fn close_workflow_stream(
8588 &self,
8589 stream_name: impl Into<String>,
8590 retention_seconds: Option<u64>,
8591 ) -> Result<()> {
8592 self.finish_workflow_stream(stream_name.into(), None, retention_seconds)
8593 }
8594
8595 pub fn error_workflow_stream(
8597 &self,
8598 stream_name: impl Into<String>,
8599 error_reason: impl Into<String>,
8600 retention_seconds: Option<u64>,
8601 ) -> Result<()> {
8602 let error_reason = error_reason.into();
8603 if error_reason.is_empty() {
8604 return Err(Error::Codec(
8605 "workflow_stream_error_invalid: error reason must not be empty".to_string(),
8606 ));
8607 }
8608 self.finish_workflow_stream(stream_name.into(), Some(error_reason), retention_seconds)
8609 }
8610
8611 fn finish_workflow_stream(
8612 &self,
8613 stream_name: String,
8614 error_reason: Option<String>,
8615 retention_seconds: Option<u64>,
8616 ) -> Result<()> {
8617 if stream_name.is_empty() {
8618 return Err(Error::Codec(
8619 "workflow_stream_name_invalid: stream name must not be empty".to_string(),
8620 ));
8621 }
8622 if retention_seconds == Some(0) {
8623 return Err(Error::Codec(
8624 "workflow_stream_retention_invalid: retention_seconds must be positive".to_string(),
8625 ));
8626 }
8627 let mut state = self
8628 .state
8629 .lock()
8630 .map_err(|_| Error::WorkflowStatePoisoned)?;
8631 let command_ordinal = state.workflow_stream_command_counter;
8632 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8633 state.workflow_stream_command_counter += 1;
8634 return match recorded {
8635 RecordedCommand::SideEffect { .. } => {
8636 state.command_cursor += 1;
8637 Ok(())
8638 }
8639 other => Err(command_mismatch(&other, "workflow stream close")),
8640 };
8641 }
8642 let identity = Self::workflow_stream_command_identity(&state)?.to_string();
8643 state.workflow_stream_command_counter += 1;
8644 let mut directive = json!({
8645 "operation": if error_reason.is_some() { "error" } else { "close" },
8646 "stream_name": stream_name,
8647 "command_identity": identity,
8648 "command_ordinal": command_ordinal,
8649 });
8650 if let Some(error_reason) = error_reason {
8651 directive["error_reason"] = json!(error_reason);
8652 }
8653 if let Some(retention_seconds) = retention_seconds {
8654 directive["retention_seconds"] = json!(retention_seconds);
8655 }
8656 let result = encode_typed_envelope(&AvroValue::Null, &state.payload_codec)?;
8657 state.commands.push(json!({
8658 "type": "record_side_effect",
8659 "result": result,
8660 "workflow_stream": directive,
8661 }));
8662 Ok(())
8663 }
8664
8665 fn workflow_stream_command_identity(state: &WorkflowState) -> Result<&str> {
8666 let identity = state.workflow_command_identity.as_str();
8667 if identity.is_empty() {
8668 return Err(Error::MissingWorkflowCommandIdentity);
8669 }
8670 Ok(identity)
8671 }
8672
8673 pub fn upsert_search_attributes(&self, update: SearchAttributeUpdate) -> Result<()> {
8679 update.validate()?;
8680 let (attributes, attribute_types) = update.into_wire_parts();
8681 let mut state = self
8682 .state
8683 .lock()
8684 .map_err(|_| Error::WorkflowStatePoisoned)?;
8685
8686 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8687 return match recorded {
8688 RecordedCommand::SearchAttributes {
8689 sequence,
8690 attributes: recorded_attributes,
8691 attribute_types: recorded_attribute_types,
8692 } => {
8693 if recorded_attributes != attributes {
8694 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8695 "search_attribute_value_mismatch",
8696 Some(sequence),
8697 Some(recorded_attributes.to_string()),
8698 Some(attributes.to_string()),
8699 "search-attribute values differ from the recorded durable command",
8700 )));
8701 }
8702 if let RecordedSnapshotValue::Known(recorded_types) = recorded_attribute_types {
8703 if recorded_types != attribute_types {
8704 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8705 "search_attribute_type_mismatch",
8706 Some(sequence),
8707 Some(json!(recorded_types).to_string()),
8708 Some(json!(attribute_types).to_string()),
8709 "search-attribute declared types differ from the recorded durable command",
8710 )));
8711 }
8712 }
8713 state.command_cursor += 1;
8714 Ok(())
8715 }
8716 other => Err(command_mismatch(&other, "search-attribute update")),
8717 };
8718 }
8719
8720 let mut command = serde_json::Map::from_iter([
8721 ("type".to_string(), json!("upsert_search_attributes")),
8722 ("attributes".to_string(), attributes),
8723 ]);
8724 if !attribute_types.is_empty() {
8725 command.insert("attribute_types".to_string(), json!(attribute_types));
8726 }
8727 state.commands.push(Value::Object(command));
8728 Ok(())
8729 }
8730
8731 pub fn uuid_v4(&self) -> Result<Uuid> {
8733 self.side_effect(Uuid::new_v4)
8734 }
8735
8736 pub fn get_version(
8739 &self,
8740 change_id: impl Into<String>,
8741 min_supported: i32,
8742 max_supported: i32,
8743 ) -> Result<i32> {
8744 let change_id = change_id.into();
8745 if change_id.trim().is_empty() {
8746 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8747 "version_change_id_invalid",
8748 None,
8749 Some("non-empty change ID".to_string()),
8750 Some(change_id),
8751 "version markers require a stable non-empty change ID",
8752 )));
8753 }
8754 if min_supported > max_supported {
8755 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8756 "version_range_invalid",
8757 None,
8758 Some("min_supported <= max_supported".to_string()),
8759 Some(format!("{min_supported}..={max_supported}")),
8760 "version marker supported range is invalid",
8761 )));
8762 }
8763
8764 let mut state = self
8765 .state
8766 .lock()
8767 .map_err(|_| Error::WorkflowStatePoisoned)?;
8768 if let Some((version, sequence)) = state.version_markers.get(&change_id).copied() {
8769 ensure_version_supported(&change_id, version, min_supported, max_supported, sequence)?;
8770 return Ok(version);
8771 }
8772
8773 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8774 return match recorded {
8775 RecordedCommand::VersionMarker {
8776 sequence,
8777 change_id: recorded_change_id,
8778 version,
8779 ..
8780 } => {
8781 if recorded_change_id != change_id {
8782 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8783 "version_change_id_mismatch",
8784 Some(sequence),
8785 Some(recorded_change_id),
8786 Some(change_id),
8787 "recorded version marker change ID differs from current workflow code",
8788 )));
8789 }
8790 ensure_version_supported(
8791 &change_id,
8792 version,
8793 min_supported,
8794 max_supported,
8795 sequence,
8796 )?;
8797 state.command_cursor += 1;
8798 state.version_markers.insert(change_id, (version, sequence));
8799 Ok(version)
8800 }
8801 other => Err(command_mismatch(
8802 &other,
8803 format!("version marker:{change_id}"),
8804 )),
8805 };
8806 }
8807
8808 let version = max_supported;
8809 state.commands.push(json!({
8810 "type": "record_version_marker",
8811 "change_id": change_id,
8812 "version": version,
8813 "min_supported": min_supported,
8814 "max_supported": max_supported,
8815 }));
8816 state.version_markers.insert(change_id, (version, 0));
8819 Ok(version)
8820 }
8821
8822 pub fn patched(&self, change_id: impl Into<String>) -> Result<bool> {
8824 Ok(self.get_version(change_id, -1, 1)? == 1)
8825 }
8826
8827 pub fn deprecate_patch(&self, change_id: impl Into<String>) -> Result<()> {
8829 self.get_version(change_id, -1, 1).map(|_| ())
8830 }
8831
8832 pub fn upsert_memo<T: Serialize>(&self, entries: T) -> Result<()> {
8837 let entries = canonical_memo_entries(AvroValue::from_serialize(&entries)?, true)?;
8838 let mut state = self
8839 .state
8840 .lock()
8841 .map_err(|_| Error::WorkflowStatePoisoned)?;
8842
8843 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8844 return match recorded {
8845 RecordedCommand::Memo {
8846 sequence,
8847 entries: recorded_entries,
8848 } => {
8849 if recorded_entries != entries {
8850 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8851 "memo_update_mismatch",
8852 Some(sequence),
8853 Some(format!("{recorded_entries:?}")),
8854 Some(format!("{entries:?}")),
8855 "recorded memo entries differ from the current workflow update",
8856 )));
8857 }
8858 state.command_cursor += 1;
8859 Ok(())
8860 }
8861 other => Err(command_mismatch(&other, "memo upsert")),
8862 };
8863 }
8864
8865 let entries_envelope = encode_typed_envelope(&entries, DEFAULT_CODEC)?;
8866 state.commands.push(json!({
8867 "type": "upsert_memo",
8868 "entries": entries_envelope,
8869 }));
8870 Ok(())
8871 }
8872
8873 pub fn start_child_workflow<T: Serialize>(
8900 &self,
8901 workflow_type: impl Into<String>,
8902 options: ChildWorkflowOptions,
8903 args: T,
8904 ) -> ChildWorkflowCall {
8905 ChildWorkflowCall {
8906 ctx: self.clone(),
8907 workflow_type: workflow_type.into(),
8908 options,
8909 args: Some(AvroValue::from_serialize(&args)),
8910 scheduled: false,
8911 matched_pending: false,
8912 parallel_group_path: Vec::new(),
8913 }
8914 }
8915
8916 pub async fn start_child_workflow_avro_value<T: Serialize>(
8917 &self,
8918 workflow_type: impl Into<String>,
8919 options: ChildWorkflowOptions,
8920 args: T,
8921 ) -> Result<ChildWorkflowAvroResult> {
8922 let mut call = self.start_child_workflow(workflow_type, options, args);
8923 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8924 }
8925
8926 fn take_commands(&self) -> Result<Vec<Value>> {
8927 let mut state = self
8928 .state
8929 .lock()
8930 .map_err(|_| Error::WorkflowStatePoisoned)?;
8931 Ok(std::mem::take(&mut state.commands))
8932 }
8933
8934 fn continue_as_new_command(&self, request: ContinueAsNewRequest) -> Result<Option<Value>> {
8935 let mut state = self
8936 .state
8937 .lock()
8938 .map_err(|_| Error::WorkflowStatePoisoned)?;
8939
8940 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8941 return Err(command_mismatch(&recorded, "continue as new"));
8942 }
8943 if state.recorded_continue_as_new_sequence.is_some() {
8944 state.continue_as_new_consumed = true;
8945 return Ok(None);
8946 }
8947
8948 let arguments = encode_typed_envelope(&request.arguments, &state.payload_codec)?;
8949 let mut command = serde_json::Map::from_iter([
8950 ("type".to_string(), json!("continue_as_new")),
8951 ("arguments".to_string(), arguments),
8952 ("queue".to_string(), json!(state.task_queue.clone())),
8953 ]);
8954 if let Some(workflow_type) = request.options.workflow_type {
8955 command.insert("workflow_type".to_string(), json!(workflow_type));
8956 }
8957 if let Some(task_queue) = request.options.task_queue {
8958 command.insert("queue".to_string(), json!(task_queue));
8959 }
8960 Ok(Some(Value::Object(command)))
8961 }
8962
8963 fn matched_recorded_pending(&self) -> Result<bool> {
8964 let state = self
8965 .state
8966 .lock()
8967 .map_err(|_| Error::WorkflowStatePoisoned)?;
8968 Ok(state.matched_recorded_pending)
8969 }
8970
8971 fn ensure_history_consumed(&self) -> Result<()> {
8972 let state = self
8973 .state
8974 .lock()
8975 .map_err(|_| Error::WorkflowStatePoisoned)?;
8976 if let Some(command) = state.recorded_commands.get(state.command_cursor) {
8977 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8978 "recorded_commands_unconsumed",
8979 Some(command.sequence()),
8980 Some(command.shape().to_string()),
8981 Some("workflow completion".to_string()),
8982 "workflow completed before consuming all recorded durable commands",
8983 )));
8984 }
8985 if let Some(sequence) = state
8986 .recorded_continue_as_new_sequence
8987 .filter(|_| !state.continue_as_new_consumed)
8988 {
8989 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8990 "recorded_continue_as_new_unconsumed",
8991 Some(sequence),
8992 Some("continue as new".to_string()),
8993 Some("workflow completion".to_string()),
8994 "workflow completed without consuming its recorded continue-as-new transition",
8995 )));
8996 }
8997 Ok(())
8998 }
8999}
9000
9001fn contiguous_message_stream_count(
9002 pending: &[MessageStreamMessage],
9003 cursor: u64,
9004 max_items: usize,
9005) -> usize {
9006 pending
9007 .iter()
9008 .take(max_items)
9009 .enumerate()
9010 .take_while(|(offset, message)| {
9011 u64::try_from(*offset)
9012 .ok()
9013 .and_then(|offset| cursor.checked_add(offset + 1))
9014 == Some(message.position)
9015 })
9016 .count()
9017}
9018
9019fn is_authored_command_open_event(event: &HistoryEvent) -> bool {
9020 matches!(
9021 event.event_type.as_str(),
9022 "ActivityScheduled"
9023 | "TimerScheduled"
9024 | "ChildWorkflowScheduled"
9025 | "SignalWaitOpened"
9026 | "ConditionWaitOpened"
9027 | "SearchAttributesUpserted"
9028 | "SideEffectRecorded"
9029 | "VersionMarkerRecorded"
9030 | "MemoUpserted"
9031 | "WorkflowContinuedAsNew"
9032 )
9033}
9034
9035#[derive(Debug)]
9036struct WorkflowState {
9037 workflow_id: Option<String>,
9038 run_id: Option<String>,
9039 task_queue: String,
9040 payload_codec: String,
9041 history_events: Arc<Vec<HistoryEvent>>,
9042 history_budget: WorkflowHistoryBudget,
9043 cancel_requested: bool,
9044 resume_signal: Option<ResumeSignal>,
9045 recorded_commands: Vec<RecordedCommand>,
9046 selection_markers: Vec<SelectionMarker>,
9047 selection_marker_cursor: usize,
9048 cancelled_selection_members: Vec<SelectionCancellation>,
9049 recorded_continue_as_new_sequence: Option<u64>,
9050 continue_as_new_consumed: bool,
9051 command_cursor: usize,
9052 condition_wait_occurrence_counter: u64,
9053 matched_recorded_pending: bool,
9054 version_markers: HashMap<String, (i32, u64)>,
9055 workflow_command_identity: String,
9056 workflow_stream_command_counter: u64,
9057 commands: Vec<Value>,
9058 message_stream_messages: HashMap<String, Vec<MessageStreamMessage>>,
9059 message_stream_cursors: HashMap<String, u64>,
9060 message_stream_waits: HashMap<String, u64>,
9061}
9062
9063impl WorkflowState {
9064 #[cfg(test)]
9065 fn new(
9066 history: Vec<HistoryEvent>,
9067 task_queue: String,
9068 payload_codec: String,
9069 resume_signal: Option<ResumeSignal>,
9070 ) -> Result<Self> {
9071 Self::new_with_identity(
9072 history,
9073 None,
9074 None,
9075 task_queue,
9076 payload_codec,
9077 resume_signal,
9078 )
9079 }
9080
9081 fn new_with_identity(
9082 history: Vec<HistoryEvent>,
9083 workflow_id: Option<String>,
9084 run_id: Option<String>,
9085 task_queue: String,
9086 payload_codec: String,
9087 resume_signal: Option<ResumeSignal>,
9088 ) -> Result<Self> {
9089 let recorded_commands = recorded_commands(
9090 &history,
9091 &payload_codec,
9092 WorkflowIdentity {
9093 workflow_id: workflow_id.clone(),
9094 run_id: run_id.clone(),
9095 },
9096 )?;
9097 let selection_markers = recorded_selection_markers(&history)?;
9098 let cancelled_selection_members = recorded_selection_cancellations(&history)?;
9099 let recorded_continue_as_new = history
9100 .iter()
9101 .filter(|event| event.event_type == "WorkflowContinuedAsNew")
9102 .collect::<Vec<_>>();
9103 if recorded_continue_as_new.len() > 1 {
9104 return Err(invalid_recorded_history(
9105 "duplicate_continue_as_new_transition",
9106 recorded_continue_as_new
9107 .last()
9108 .and_then(|event| durable_event_sequence(event))
9109 .unwrap_or(0),
9110 "one WorkflowContinuedAsNew event",
9111 &format!(
9112 "{} WorkflowContinuedAsNew events",
9113 recorded_continue_as_new.len()
9114 ),
9115 "workflow history records one continue-as-new transition more than once",
9116 ));
9117 }
9118 let recorded_continue_as_new_sequence = recorded_continue_as_new
9119 .first()
9120 .map(|event| {
9121 durable_event_sequence(event).ok_or_else(|| {
9122 Error::NonDeterministicReplay(ReplayFailure::new(
9123 "continue_as_new_sequence_missing",
9124 None,
9125 Some("recorded transition sequence".to_string()),
9126 Some("missing sequence".to_string()),
9127 "WorkflowContinuedAsNew history is missing its recorded sequence",
9128 ))
9129 })
9130 })
9131 .transpose()?;
9132 let mut message_stream_cursors = HashMap::new();
9133 for event in &history {
9134 if !matches!(
9135 event.event_type.as_str(),
9136 "SignalReceived" | "SignalApplied"
9137 ) || event.payload.get("signal_name").and_then(Value::as_str)
9138 != Some(MESSAGE_STREAM_SIGNAL)
9139 {
9140 continue;
9141 }
9142 let arguments = decode_signal_event_arguments(event, &payload_codec)?;
9143 if arguments.len() != 1 {
9144 continue;
9145 }
9146 let envelope = arguments[0].clone().into_json()?;
9147 let Some(envelope) = envelope.as_object() else {
9148 continue;
9149 };
9150 if envelope.get("schema").and_then(Value::as_str) != Some(MESSAGE_STREAM_CURSOR_SCHEMA)
9151 {
9152 continue;
9153 }
9154 let Some(stream_name) = envelope.get("stream_name").and_then(Value::as_str) else {
9155 continue;
9156 };
9157 let Some(through_position) = envelope.get("through_position").and_then(Value::as_u64)
9158 else {
9159 continue;
9160 };
9161 let cursor = message_stream_cursors
9162 .entry(stream_name.to_string())
9163 .or_insert(0);
9164 *cursor = (*cursor).max(through_position);
9165 }
9166 let event_count = u64::try_from(history.len()).unwrap_or(u64::MAX);
9167 let cancel_requested = history.iter().any(|event| {
9168 matches!(
9169 event.event_type.as_str(),
9170 "WorkflowCancellationRequested" | "WorkflowCancelRequested"
9171 )
9172 });
9173 Ok(Self {
9174 workflow_command_identity: String::new(),
9175 workflow_stream_command_counter: 0,
9176 workflow_id,
9177 run_id,
9178 task_queue,
9179 payload_codec,
9180 history_events: Arc::new(history),
9181 history_budget: WorkflowHistoryBudget {
9182 event_count,
9183 ..WorkflowHistoryBudget::default()
9184 },
9185 cancel_requested,
9186 resume_signal,
9187 recorded_commands,
9188 selection_markers,
9189 selection_marker_cursor: 0,
9190 cancelled_selection_members,
9191 recorded_continue_as_new_sequence,
9192 continue_as_new_consumed: false,
9193 command_cursor: 0,
9194 condition_wait_occurrence_counter: 0,
9195 matched_recorded_pending: false,
9196 version_markers: HashMap::new(),
9197 commands: Vec::new(),
9198 message_stream_messages: HashMap::new(),
9199 message_stream_cursors,
9200 message_stream_waits: HashMap::new(),
9201 })
9202 }
9203}
9204
9205enum MessageStreamDelivery {
9206 Message(MessageStreamMessage),
9207 Cursor {
9208 stream_name: String,
9209 through_position: u64,
9210 },
9211}
9212
9213fn decode_message_stream_delivery(arguments: Vec<Value>) -> Result<Option<MessageStreamDelivery>> {
9214 if arguments.len() != 1 {
9215 return Ok(None);
9216 }
9217 let envelope = arguments
9218 .into_iter()
9219 .next()
9220 .expect("one argument was checked");
9221 let Some(envelope) = envelope.as_object() else {
9222 return Ok(None);
9223 };
9224 let Some(stream_name) = envelope.get("stream_name").and_then(Value::as_str) else {
9225 return Ok(None);
9226 };
9227 if envelope.get("schema").and_then(Value::as_str) == Some(MESSAGE_STREAM_CURSOR_SCHEMA) {
9228 let Some(through_position) = envelope.get("through_position").and_then(Value::as_u64)
9229 else {
9230 return Ok(None);
9231 };
9232 return Ok(Some(MessageStreamDelivery::Cursor {
9233 stream_name: stream_name.to_string(),
9234 through_position,
9235 }));
9236 }
9237 if envelope.get("schema").and_then(Value::as_str) != Some(MESSAGE_STREAM_SCHEMA) {
9238 return Ok(None);
9239 }
9240 let Some(message_id) = envelope.get("message_id").and_then(Value::as_str) else {
9241 return Ok(None);
9242 };
9243 let Some(position) = envelope
9244 .get("position")
9245 .and_then(Value::as_u64)
9246 .filter(|value| *value > 0)
9247 else {
9248 return Ok(None);
9249 };
9250 let Some(payload_envelope) = envelope.get("payload_envelope") else {
9251 return Ok(None);
9252 };
9253 let Ok(payload_envelope) = serde_json::from_value::<PayloadEnvelope>(payload_envelope.clone())
9254 else {
9255 return Ok(None);
9256 };
9257 let decoded = decode_avro_value(&payload_envelope)?;
9258 let AvroValue::Array(values) = decoded else {
9259 return Ok(None);
9260 };
9261 Ok(Some(MessageStreamDelivery::Message(MessageStreamMessage {
9262 stream_name: stream_name.to_string(),
9263 message_id: message_id.to_string(),
9264 position,
9265 arguments: values,
9266 })))
9267}
9268
9269#[derive(Clone, Debug)]
9270enum RecordedCommand {
9271 Activity {
9272 sequence: u64,
9273 activity_type: Option<String>,
9274 options: Option<RecordedActivityOptions>,
9275 outcome: Option<ActivityOutcome>,
9276 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9277 },
9278 Timer {
9279 sequence: u64,
9280 delay_seconds: u64,
9281 fired: bool,
9282 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9283 },
9284 ChildWorkflow {
9285 sequence: u64,
9286 workflow_type: Option<String>,
9287 outcome: Option<ChildWorkflowOutcome>,
9288 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9289 },
9290 SignalWait {
9291 sequence: u64,
9292 signal_name: String,
9293 value: Option<Vec<AvroValue>>,
9294 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9295 },
9296 ConditionWait {
9297 sequence: u64,
9298 occurrence_id: String,
9299 condition_key: Option<String>,
9300 predicate_identity: String,
9301 timeout_seconds: Option<u64>,
9302 result: Option<ConditionWaitResult>,
9303 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9304 },
9305 SearchAttributes {
9306 sequence: u64,
9307 attributes: Value,
9308 attribute_types: RecordedSnapshotValue<BTreeMap<String, String>>,
9309 },
9310 SideEffect {
9311 sequence: u64,
9312 value: AvroValue,
9313 },
9314 VersionMarker {
9315 sequence: u64,
9316 change_id: String,
9317 version: i32,
9318 },
9319 Memo {
9320 sequence: u64,
9321 entries: AvroValue,
9322 },
9323}
9324
9325#[derive(Clone, Debug, PartialEq, Eq)]
9326struct SelectionMarker {
9327 selection_group_id: String,
9328 selection_group_base_sequence: u64,
9329 selection_group_size: usize,
9330 member_key: SelectionKey,
9331 member_index: usize,
9332 member_base_sequence: u64,
9333 member_size: usize,
9334 operation_kind: String,
9335 operation_identity: String,
9336 outcome: String,
9337 resolution_event_id: String,
9338 resolution_event_type: String,
9339}
9340
9341#[derive(Clone, Debug, PartialEq, Eq)]
9342struct SelectionCancellation {
9343 selection_group_id: String,
9344 member_key: SelectionKey,
9345 member_index: usize,
9346 member_base_sequence: u64,
9347 member_size: usize,
9348 operation_kind: String,
9349 operation_identity: String,
9350}
9351
9352fn recorded_selection_markers(events: &[HistoryEvent]) -> Result<Vec<SelectionMarker>> {
9353 let mut markers: Vec<SelectionMarker> = Vec::new();
9354 for event in events
9355 .iter()
9356 .filter(|event| event.event_type == "SelectionResolved")
9357 {
9358 let payload = &event.payload;
9359 let base_sequence = required_selection_u64(payload, "selection_group_base_sequence")?;
9360 let group_size = required_selection_usize(payload, "selection_group_size")?;
9361 let member_base_sequence = required_selection_u64(payload, "member_base_sequence")?;
9362 let member_size = required_selection_usize(payload, "member_size")?;
9363 let member_index = required_selection_usize_allow_zero(payload, "member_index")?;
9364 let group_id = payload_string(payload, "selection_group_id").ok_or_else(|| {
9365 invalid_recorded_history(
9366 "selection_marker_invalid",
9367 base_sequence,
9368 "non-empty selection_group_id",
9369 &payload.to_string(),
9370 "selection winner history is missing its durable group identity",
9371 )
9372 })?;
9373 let expected_group_id = format!("select-calls:{base_sequence}:{group_size}");
9374 if group_id != expected_group_id {
9375 return Err(invalid_recorded_history(
9376 "selection_marker_invalid",
9377 base_sequence,
9378 &expected_group_id,
9379 &group_id,
9380 "selection winner history contains an incompatible group identity",
9381 ));
9382 }
9383 let group_end = base_sequence
9384 .checked_add(u64::try_from(group_size).unwrap_or(u64::MAX))
9385 .unwrap_or(u64::MAX);
9386 let member_end = member_base_sequence
9387 .checked_add(u64::try_from(member_size).unwrap_or(u64::MAX))
9388 .unwrap_or(u64::MAX);
9389 if member_index >= group_size
9390 || member_base_sequence < base_sequence
9391 || member_end > group_end
9392 {
9393 return Err(invalid_recorded_history(
9394 "selection_marker_invalid",
9395 base_sequence,
9396 "winner member within selection group bounds",
9397 &payload.to_string(),
9398 "selection winner history contains an invalid member range",
9399 ));
9400 }
9401 let operation_kind = payload_string(payload, "operation_kind").ok_or_else(|| {
9402 invalid_recorded_history(
9403 "selection_marker_invalid",
9404 base_sequence,
9405 "selection operation kind",
9406 &payload.to_string(),
9407 "selection winner history is missing its operation kind",
9408 )
9409 })?;
9410 if !matches!(
9411 operation_kind.as_str(),
9412 "activity" | "child" | "timer" | "signal" | "condition" | "group"
9413 ) {
9414 return Err(invalid_recorded_history(
9415 "selection_marker_invalid",
9416 base_sequence,
9417 "activity, child, timer, signal, condition, or group",
9418 &operation_kind,
9419 "selection winner history contains an unsupported operation kind",
9420 ));
9421 }
9422 let operation_identity =
9423 payload_string(payload, "operation_identity").ok_or_else(|| {
9424 invalid_recorded_history(
9425 "selection_marker_invalid",
9426 base_sequence,
9427 "non-empty operation identity",
9428 &payload.to_string(),
9429 "selection winner history is missing its durable operation identity",
9430 )
9431 })?;
9432 let outcome = payload_string(payload, "outcome").ok_or_else(|| {
9433 invalid_recorded_history(
9434 "selection_marker_invalid",
9435 base_sequence,
9436 "completed or failed selection outcome",
9437 &payload.to_string(),
9438 "selection winner history is missing its outcome",
9439 )
9440 })?;
9441 if !matches!(outcome.as_str(), "completed" | "failed") {
9442 return Err(invalid_recorded_history(
9443 "selection_marker_invalid",
9444 base_sequence,
9445 "completed or failed selection outcome",
9446 &outcome,
9447 "selection winner history contains an unsupported outcome",
9448 ));
9449 }
9450 let marker = SelectionMarker {
9451 selection_group_id: group_id,
9452 selection_group_base_sequence: base_sequence,
9453 selection_group_size: group_size,
9454 member_key: selection_key_from_value(payload.get("member_key"), base_sequence)?,
9455 member_index,
9456 member_base_sequence,
9457 member_size,
9458 operation_kind,
9459 operation_identity,
9460 outcome,
9461 resolution_event_id: payload_string(payload, "resolution_event_id").ok_or_else(
9462 || {
9463 invalid_recorded_history(
9464 "selection_marker_invalid",
9465 base_sequence,
9466 "durable resolution_event_id",
9467 &payload.to_string(),
9468 "selection winner history is missing its terminal event identity",
9469 )
9470 },
9471 )?,
9472 resolution_event_type: payload_string(payload, "resolution_event_type").ok_or_else(
9473 || {
9474 invalid_recorded_history(
9475 "selection_marker_invalid",
9476 base_sequence,
9477 "durable resolution_event_type",
9478 &payload.to_string(),
9479 "selection winner history is missing its terminal event type",
9480 )
9481 },
9482 )?,
9483 };
9484 if let Some(existing) = markers
9485 .iter()
9486 .find(|existing| existing.selection_group_id == marker.selection_group_id)
9487 {
9488 if existing != &marker {
9489 return Err(invalid_recorded_history(
9490 "selection_marker_conflict",
9491 base_sequence,
9492 &format!("one winner for {}", marker.selection_group_id),
9493 &payload.to_string(),
9494 "selection history records conflicting winners for one durable group",
9495 ));
9496 }
9497 continue;
9498 }
9499 markers.push(marker);
9500 }
9501 Ok(markers)
9502}
9503
9504fn recorded_selection_cancellations(events: &[HistoryEvent]) -> Result<Vec<SelectionCancellation>> {
9505 let mut cancelled: Vec<SelectionCancellation> = Vec::new();
9506 for event in events
9507 .iter()
9508 .filter(|event| event.event_type == "SelectionOperationCancelled")
9509 {
9510 let group_id = payload_string(&event.payload, "selection_group_id").ok_or_else(|| {
9511 invalid_recorded_history(
9512 "selection_cancellation_invalid",
9513 0,
9514 "non-empty selection_group_id",
9515 &event.payload.to_string(),
9516 "selection cancellation history is missing its group identity",
9517 )
9518 })?;
9519 let member_base_sequence = required_selection_u64(&event.payload, "member_base_sequence")?;
9520 let marker = SelectionCancellation {
9521 selection_group_id: group_id,
9522 member_key: selection_key_from_value(
9523 event.payload.get("member_key"),
9524 member_base_sequence,
9525 )?,
9526 member_index: required_selection_usize_allow_zero(&event.payload, "member_index")?,
9527 member_base_sequence,
9528 member_size: required_selection_usize(&event.payload, "member_size")?,
9529 operation_kind: payload_string(&event.payload, "operation_kind").ok_or_else(|| {
9530 invalid_recorded_history(
9531 "selection_cancellation_invalid",
9532 member_base_sequence,
9533 "selection operation kind",
9534 &event.payload.to_string(),
9535 "selection cancellation is missing its operation kind",
9536 )
9537 })?,
9538 operation_identity: payload_string(&event.payload, "operation_identity").ok_or_else(
9539 || {
9540 invalid_recorded_history(
9541 "selection_cancellation_invalid",
9542 member_base_sequence,
9543 "selection operation identity",
9544 &event.payload.to_string(),
9545 "selection cancellation is missing its operation identity",
9546 )
9547 },
9548 )?,
9549 };
9550 if let Some(existing) = cancelled.iter().find(|recorded| {
9551 recorded.selection_group_id == marker.selection_group_id
9552 && recorded.member_base_sequence == marker.member_base_sequence
9553 }) {
9554 if existing != &marker {
9555 return Err(invalid_recorded_history(
9556 "selection_cancellation_conflict",
9557 member_base_sequence,
9558 "one stable SelectionOperationCancelled marker",
9559 &event.payload.to_string(),
9560 "selection cancellation history contains conflicting member metadata",
9561 ));
9562 }
9563 } else {
9564 cancelled.push(marker);
9565 }
9566 }
9567 Ok(cancelled)
9568}
9569
9570fn required_selection_u64(payload: &Value, field: &str) -> Result<u64> {
9571 payload
9572 .get(field)
9573 .and_then(value_as_u64)
9574 .filter(|value| *value > 0)
9575 .ok_or_else(|| {
9576 invalid_recorded_history(
9577 "selection_marker_invalid",
9578 0,
9579 &format!("positive integer {field}"),
9580 &payload.to_string(),
9581 "selection history contains invalid durable identity metadata",
9582 )
9583 })
9584}
9585
9586fn required_selection_usize(payload: &Value, field: &str) -> Result<usize> {
9587 required_selection_usize_allow_zero(payload, field).and_then(|value| {
9588 if value > 0 {
9589 Ok(value)
9590 } else {
9591 Err(invalid_recorded_history(
9592 "selection_marker_invalid",
9593 0,
9594 &format!("positive integer {field}"),
9595 &payload.to_string(),
9596 "selection history contains invalid durable identity metadata",
9597 ))
9598 }
9599 })
9600}
9601
9602fn required_selection_usize_allow_zero(payload: &Value, field: &str) -> Result<usize> {
9603 payload
9604 .get(field)
9605 .and_then(value_as_u64)
9606 .and_then(|value| usize::try_from(value).ok())
9607 .ok_or_else(|| {
9608 invalid_recorded_history(
9609 "selection_marker_invalid",
9610 0,
9611 &format!("non-negative integer {field}"),
9612 &payload.to_string(),
9613 "selection history contains invalid durable identity metadata",
9614 )
9615 })
9616}
9617
9618#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9619struct RecordedActivityOptions {
9620 task_queue: RecordedSnapshotValue<Option<String>>,
9621 execution_mode: RecordedSnapshotValue<Option<String>>,
9622 retry_policy: ActivityRetrySnapshot,
9623}
9624
9625#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9626enum RecordedSnapshotValue<T> {
9627 Unknown,
9629 Known(T),
9630}
9631
9632impl<T: PartialEq> RecordedSnapshotValue<T> {
9633 fn matches_current(&self, current: &Self) -> bool {
9634 match self {
9635 Self::Unknown => true,
9636 Self::Known(recorded) => matches!(current, Self::Known(value) if value == recorded),
9637 }
9638 }
9639}
9640
9641#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9642struct ActivityRetrySnapshot {
9643 snapshot_version: RecordedSnapshotValue<Option<u64>>,
9644 max_attempts: RecordedSnapshotValue<Option<u64>>,
9645 backoff_seconds: RecordedSnapshotValue<Vec<u64>>,
9646 start_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
9647 schedule_to_start_timeout: RecordedSnapshotValue<Option<u64>>,
9648 schedule_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
9649 heartbeat_timeout: RecordedSnapshotValue<Option<u64>>,
9650 non_retryable_error_types: RecordedSnapshotValue<Vec<String>>,
9651}
9652
9653impl ActivityRetrySnapshot {
9654 fn matches_current(&self, current: &Self) -> bool {
9655 self.snapshot_version
9656 .matches_current(¤t.snapshot_version)
9657 && self.max_attempts.matches_current(¤t.max_attempts)
9658 && self
9659 .backoff_seconds
9660 .matches_current(¤t.backoff_seconds)
9661 && self
9662 .start_to_close_timeout
9663 .matches_current(¤t.start_to_close_timeout)
9664 && self
9665 .schedule_to_start_timeout
9666 .matches_current(¤t.schedule_to_start_timeout)
9667 && self
9668 .schedule_to_close_timeout
9669 .matches_current(¤t.schedule_to_close_timeout)
9670 && self
9671 .heartbeat_timeout
9672 .matches_current(¤t.heartbeat_timeout)
9673 && self
9674 .non_retryable_error_types
9675 .matches_current(¤t.non_retryable_error_types)
9676 }
9677}
9678
9679fn recorded_optional_u64(
9680 object: Option<&serde_json::Map<String, Value>>,
9681 field: &str,
9682) -> RecordedSnapshotValue<Option<u64>> {
9683 match object.and_then(|object| object.get(field)) {
9684 None => RecordedSnapshotValue::Unknown,
9685 Some(Value::Null) => RecordedSnapshotValue::Known(None),
9686 Some(value) => RecordedSnapshotValue::Known(value_as_u64(value)),
9687 }
9688}
9689
9690fn recorded_optional_string(
9691 object: &serde_json::Map<String, Value>,
9692 field: &str,
9693) -> RecordedSnapshotValue<Option<String>> {
9694 match object.get(field) {
9695 None => RecordedSnapshotValue::Unknown,
9696 Some(Value::Null) => RecordedSnapshotValue::Known(None),
9697 Some(value) => RecordedSnapshotValue::Known(value.as_str().map(str::to_string)),
9698 }
9699}
9700
9701fn recorded_activity_retry_snapshot(policy: Option<&Value>) -> ActivityRetrySnapshot {
9702 let policy = policy.and_then(Value::as_object);
9703 let backoff_seconds = policy
9704 .and_then(|policy| policy.get("backoff_seconds"))
9705 .and_then(Value::as_array)
9706 .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
9707 .map_or(RecordedSnapshotValue::Unknown, RecordedSnapshotValue::Known);
9708 let mut non_retryable_error_types = Vec::new();
9709 for error_type in policy
9710 .and_then(|policy| policy.get("non_retryable_error_types"))
9711 .and_then(Value::as_array)
9712 .into_iter()
9713 .flatten()
9714 .filter_map(Value::as_str)
9715 .map(str::trim)
9716 .filter(|error_type| !error_type.is_empty())
9717 {
9718 if !non_retryable_error_types
9719 .iter()
9720 .any(|recorded| recorded == error_type)
9721 {
9722 non_retryable_error_types.push(error_type.to_string());
9723 }
9724 }
9725
9726 ActivityRetrySnapshot {
9727 snapshot_version: recorded_optional_u64(policy, "snapshot_version"),
9728 max_attempts: recorded_optional_u64(policy, "max_attempts"),
9729 backoff_seconds,
9730 start_to_close_timeout: recorded_optional_u64(policy, "start_to_close_timeout"),
9731 schedule_to_start_timeout: recorded_optional_u64(policy, "schedule_to_start_timeout"),
9732 schedule_to_close_timeout: recorded_optional_u64(policy, "schedule_to_close_timeout"),
9733 heartbeat_timeout: recorded_optional_u64(policy, "heartbeat_timeout"),
9734 non_retryable_error_types: if policy
9735 .is_some_and(|policy| policy.contains_key("non_retryable_error_types"))
9736 {
9737 RecordedSnapshotValue::Known(non_retryable_error_types)
9738 } else {
9739 RecordedSnapshotValue::Unknown
9740 },
9741 }
9742}
9743
9744fn current_activity_retry_snapshot(options: &ValidatedActivityOptions) -> ActivityRetrySnapshot {
9745 let policy = options.retry_policy.as_ref();
9746 let max_attempts = match policy.and_then(|policy| policy.get("max_attempts")) {
9747 Some(Value::Null) => None,
9748 Some(value) => value_as_u64(value),
9749 None => Some(1),
9750 };
9751 let backoff_seconds = policy
9752 .and_then(|policy| policy.get("backoff_seconds"))
9753 .and_then(Value::as_array)
9754 .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
9755 .unwrap_or_default();
9756 let non_retryable_error_types = policy
9757 .and_then(|policy| policy.get("non_retryable_error_types"))
9758 .and_then(Value::as_array)
9759 .into_iter()
9760 .flatten()
9761 .filter_map(Value::as_str)
9762 .map(str::to_string)
9763 .collect();
9764
9765 ActivityRetrySnapshot {
9766 snapshot_version: RecordedSnapshotValue::Known(Some(1)),
9767 max_attempts: RecordedSnapshotValue::Known(max_attempts),
9768 backoff_seconds: RecordedSnapshotValue::Known(backoff_seconds),
9769 start_to_close_timeout: RecordedSnapshotValue::Known(options.start_to_close_timeout),
9770 schedule_to_start_timeout: RecordedSnapshotValue::Known(options.schedule_to_start_timeout),
9771 schedule_to_close_timeout: RecordedSnapshotValue::Known(options.schedule_to_close_timeout),
9772 heartbeat_timeout: RecordedSnapshotValue::Known(options.heartbeat_timeout),
9773 non_retryable_error_types: RecordedSnapshotValue::Known(non_retryable_error_types),
9774 }
9775}
9776
9777fn activity_options_description(options: &RecordedActivityOptions) -> String {
9778 serde_json::to_string(options).unwrap_or_else(|_| format!("{options:?}"))
9779}
9780
9781impl RecordedCommand {
9782 fn sequence(&self) -> u64 {
9783 match self {
9784 Self::Activity { sequence, .. }
9785 | Self::Timer { sequence, .. }
9786 | Self::ChildWorkflow { sequence, .. }
9787 | Self::SignalWait { sequence, .. }
9788 | Self::ConditionWait { sequence, .. }
9789 | Self::SearchAttributes { sequence, .. }
9790 | Self::SideEffect { sequence, .. }
9791 | Self::VersionMarker { sequence, .. }
9792 | Self::Memo { sequence, .. } => *sequence,
9793 }
9794 }
9795
9796 fn shape(&self) -> &'static str {
9797 match self {
9798 Self::Activity { .. } => "activity",
9799 Self::Timer { .. } => "timer",
9800 Self::ChildWorkflow { .. } => "child workflow",
9801 Self::SignalWait { .. } => "signal wait",
9802 Self::ConditionWait { .. } => "condition wait",
9803 Self::SearchAttributes { .. } => "search-attribute update",
9804 Self::SideEffect { .. } => "side effect",
9805 Self::VersionMarker { .. } => "version marker",
9806 Self::Memo { .. } => "memo upsert",
9807 }
9808 }
9809}
9810
9811fn ensure_version_supported(
9812 change_id: &str,
9813 version: i32,
9814 min_supported: i32,
9815 max_supported: i32,
9816 sequence: u64,
9817) -> Result<()> {
9818 if (min_supported..=max_supported).contains(&version) {
9819 return Ok(());
9820 }
9821 Err(Error::NonDeterministicReplay(ReplayFailure::new(
9822 "version_marker_incompatible_range",
9823 (sequence != 0).then_some(sequence),
9824 Some(format!("{min_supported}..={max_supported}")),
9825 Some(format!("{change_id}:{version}")),
9826 "recorded workflow version is outside the range supported by current code",
9827 )))
9828}
9829
9830#[derive(Clone, Debug)]
9831struct ResumeSignal {
9832 signal_name: String,
9833 arguments: Vec<AvroValue>,
9834}
9835
9836const MAX_PARALLEL_OPERATIONS: usize = 1000;
9837
9838fn parallel_group_prefix(kind: &str) -> &'static str {
9839 match kind {
9840 "activity" => "parallel-activities",
9841 "child" => "parallel-children",
9842 "timer" => "parallel-timers",
9843 _ => "parallel-calls",
9844 }
9845}
9846
9847fn parallel_group_entry(
9848 base_sequence: u64,
9849 size: usize,
9850 index: usize,
9851 kind: &str,
9852) -> ParallelGroupMetadata {
9853 ParallelGroupMetadata {
9854 parallel_group_id: format!("{}:{base_sequence}:{size}", parallel_group_prefix(kind)),
9855 parallel_group_kind: kind.to_string(),
9856 parallel_group_base_sequence: base_sequence,
9857 parallel_group_size: size,
9858 parallel_group_index: index,
9859 parallel_group_mode: None,
9860 selection_member_key: None,
9861 selection_member_index: None,
9862 selection_member_base_sequence: None,
9863 selection_member_size: None,
9864 selection_member_kind: None,
9865 }
9866}
9867
9868struct SelectionMemberMetadata {
9869 key: SelectionKey,
9870 index: usize,
9871 base_sequence: u64,
9872 size: usize,
9873 kind: String,
9874}
9875
9876fn selection_group_entry(
9877 base_sequence: u64,
9878 size: usize,
9879 index: usize,
9880 kind: &str,
9881 member: &SelectionMemberMetadata,
9882) -> ParallelGroupMetadata {
9883 ParallelGroupMetadata {
9884 parallel_group_id: format!("select-calls:{base_sequence}:{size}"),
9885 parallel_group_kind: kind.to_string(),
9886 parallel_group_base_sequence: base_sequence,
9887 parallel_group_size: size,
9888 parallel_group_index: index,
9889 parallel_group_mode: Some("select".to_string()),
9890 selection_member_key: Some(member.key.clone()),
9891 selection_member_index: Some(member.index),
9892 selection_member_base_sequence: Some(member.base_sequence),
9893 selection_member_size: Some(member.size),
9894 selection_member_kind: Some(member.kind.clone()),
9895 }
9896}
9897
9898fn apply_parallel_group_path(
9899 command: &mut serde_json::Map<String, Value>,
9900 path: &[ParallelGroupMetadata],
9901) {
9902 let Some(inner) = path.last() else {
9903 return;
9904 };
9905 command.insert(
9906 "parallel_group_id".to_string(),
9907 json!(inner.parallel_group_id),
9908 );
9909 command.insert(
9910 "parallel_group_kind".to_string(),
9911 json!(inner.parallel_group_kind),
9912 );
9913 command.insert(
9914 "parallel_group_base_sequence".to_string(),
9915 json!(inner.parallel_group_base_sequence),
9916 );
9917 command.insert(
9918 "parallel_group_size".to_string(),
9919 json!(inner.parallel_group_size),
9920 );
9921 command.insert(
9922 "parallel_group_index".to_string(),
9923 json!(inner.parallel_group_index),
9924 );
9925 if let Some(mode) = &inner.parallel_group_mode {
9926 command.insert("parallel_group_mode".to_string(), json!(mode));
9927 }
9928 if let Some(key) = &inner.selection_member_key {
9929 command.insert("selection_member_key".to_string(), json!(key));
9930 }
9931 if let Some(index) = inner.selection_member_index {
9932 command.insert("selection_member_index".to_string(), json!(index));
9933 }
9934 if let Some(base_sequence) = inner.selection_member_base_sequence {
9935 command.insert(
9936 "selection_member_base_sequence".to_string(),
9937 json!(base_sequence),
9938 );
9939 }
9940 if let Some(size) = inner.selection_member_size {
9941 command.insert("selection_member_size".to_string(), json!(size));
9942 }
9943 if let Some(kind) = &inner.selection_member_kind {
9944 command.insert("selection_member_kind".to_string(), json!(kind));
9945 }
9946 command.insert("parallel_group_path".to_string(), json!(path));
9947}
9948
9949fn ensure_parallel_path_matches(
9950 sequence: u64,
9951 recorded: Option<&[ParallelGroupMetadata]>,
9952 expected: &[ParallelGroupMetadata],
9953) -> Result<()> {
9954 match (recorded, expected.is_empty()) {
9955 (None, true) => Ok(()),
9956 (Some(recorded), false) if recorded == expected => Ok(()),
9957 (None, false) => Err(invalid_recorded_history(
9958 "parallel_group_metadata_missing",
9959 sequence,
9960 &serde_json::to_string(expected).unwrap_or_default(),
9961 "<missing>",
9962 "recorded parallel member is missing its durable group path",
9963 )),
9964 (Some(recorded), true) => Err(invalid_recorded_history(
9965 "parallel_group_shape_mismatch",
9966 sequence,
9967 "sequential command",
9968 &serde_json::to_string(recorded).unwrap_or_default(),
9969 "recorded command belonged to a parallel group but current code schedules it sequentially",
9970 )),
9971 (Some(recorded), false) => Err(invalid_recorded_history(
9972 "parallel_group_shape_mismatch",
9973 sequence,
9974 &serde_json::to_string(recorded).unwrap_or_default(),
9975 &serde_json::to_string(expected).unwrap_or_default(),
9976 "recorded parallel-group identity or path changed during replay",
9977 )),
9978 }
9979}
9980
9981#[derive(Clone, Debug)]
9982enum ParallelShape {
9983 Leaf,
9984 Group(Vec<ParallelShape>),
9985}
9986
9987struct ParallelDescriptor {
9988 operation: ParallelOperation,
9989 offset: usize,
9990 member_path: Vec<usize>,
9991 group_path: Vec<ParallelGroupMetadata>,
9992}
9993
9994fn parallel_leaf_count(operations: &[ParallelOperation]) -> usize {
9995 operations
9996 .iter()
9997 .map(|operation| match operation {
9998 ParallelOperation::Group(children) => parallel_leaf_count(children),
9999 _ => 1,
10000 })
10001 .sum()
10002}
10003
10004fn parallel_operation_kind(operation: &ParallelOperation) -> Option<&'static str> {
10005 match operation {
10006 ParallelOperation::Activity { .. } => Some("activity"),
10007 ParallelOperation::ChildWorkflow { .. } => Some("child"),
10008 ParallelOperation::Timer(_) => Some("timer"),
10009 ParallelOperation::Signal(_) => Some("signal"),
10010 ParallelOperation::Condition { .. } => Some("condition"),
10011 ParallelOperation::Group(children) => parallel_group_kind(children),
10012 }
10013}
10014
10015fn parallel_group_kind(operations: &[ParallelOperation]) -> Option<&'static str> {
10016 let mut kind = None;
10017 for operation in operations {
10018 let Some(operation_kind) = parallel_operation_kind(operation) else {
10019 continue;
10020 };
10021 match kind {
10022 None => kind = Some(operation_kind),
10023 Some(current) if current == operation_kind => {}
10024 Some(_) => return Some("mixed"),
10025 }
10026 }
10027 kind
10028}
10029
10030fn validate_parallel_operations(
10031 operations: &[ParallelOperation],
10032 member_path: &mut Vec<usize>,
10033 root: bool,
10034) -> Result<()> {
10035 let leaves = parallel_leaf_count(operations);
10036 if leaves > MAX_PARALLEL_OPERATIONS {
10037 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10038 reason: "fan_out_limit_exceeded",
10039 member_path: member_path.clone(),
10040 message: format!(
10041 "group contains {leaves} durable leaves; the limit is {MAX_PARALLEL_OPERATIONS}"
10042 ),
10043 }));
10044 }
10045 if !root && operations.is_empty() {
10046 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10047 reason: "nested_group_empty",
10048 member_path: member_path.clone(),
10049 message: "a nested group must contain at least one durable leaf".to_string(),
10050 }));
10051 }
10052
10053 for (index, operation) in operations.iter().enumerate() {
10054 member_path.push(index);
10055 match operation {
10056 ParallelOperation::Activity {
10057 options, arguments, ..
10058 } => {
10059 options
10060 .validate()
10061 .map_err(|error| Error::InvalidActivityOptions(error))?;
10062 if let Err(error) = arguments {
10063 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10064 reason: "arguments_invalid",
10065 member_path: member_path.clone(),
10066 message: error.to_string(),
10067 }));
10068 }
10069 }
10070 ParallelOperation::ChildWorkflow {
10071 options, arguments, ..
10072 } => {
10073 validate_parallel_child_options(options)?;
10074 if let Err(error) = arguments {
10075 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10076 reason: "arguments_invalid",
10077 member_path: member_path.clone(),
10078 message: error.to_string(),
10079 }));
10080 }
10081 }
10082 ParallelOperation::Timer(duration)
10083 if duration.as_secs() == u64::MAX && duration.subsec_nanos() > 0 =>
10084 {
10085 return Err(Error::TimerDurationOverflow);
10086 }
10087 ParallelOperation::Timer(_) => {}
10088 ParallelOperation::Signal(signal_name) => {
10089 validate_user_signal_name(signal_name)?;
10090 if signal_name.trim().is_empty() {
10091 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10092 reason: "signal_name_empty",
10093 member_path: member_path.clone(),
10094 message: "signal wait name must not be empty".to_string(),
10095 }));
10096 }
10097 }
10098 ParallelOperation::Condition { options, .. } => {
10099 options.validate()?;
10100 }
10101 ParallelOperation::Group(children) => {
10102 validate_parallel_operations(children, member_path, false)?;
10103 }
10104 }
10105 member_path.pop();
10106 }
10107 Ok(())
10108}
10109
10110fn validate_parallel_child_options(options: &ChildWorkflowOptions) -> Result<()> {
10111 if options.task_queue.trim().is_empty() {
10112 return Err(Error::InvalidChildWorkflowOptions(
10113 "task_queue must not be empty".to_string(),
10114 ));
10115 }
10116 for (name, value) in [
10117 (
10118 "execution_timeout_seconds",
10119 options.execution_timeout_seconds,
10120 ),
10121 ("run_timeout_seconds", options.run_timeout_seconds),
10122 ] {
10123 if value == Some(0) {
10124 return Err(Error::InvalidChildWorkflowOptions(format!(
10125 "{name} must be at least 1"
10126 )));
10127 }
10128 }
10129 if options
10130 .retry_policy
10131 .as_ref()
10132 .is_some_and(|policy| policy.max_attempts == Some(0))
10133 {
10134 return Err(Error::InvalidChildWorkflowOptions(
10135 "retry_policy.max_attempts must be at least 1".to_string(),
10136 ));
10137 }
10138 Ok(())
10139}
10140
10141fn parallel_shape(operations: &[ParallelOperation]) -> ParallelShape {
10142 ParallelShape::Group(
10143 operations
10144 .iter()
10145 .map(|operation| match operation {
10146 ParallelOperation::Group(children) => parallel_shape(children),
10147 _ => ParallelShape::Leaf,
10148 })
10149 .collect(),
10150 )
10151}
10152
10153fn parallel_descriptors(
10154 operations: Vec<ParallelOperation>,
10155 base_sequence: u64,
10156) -> Result<Vec<ParallelDescriptor>> {
10157 let size = parallel_leaf_count(&operations);
10158 let kind = parallel_group_kind(&operations).unwrap_or("activity");
10159 let mut descriptors = Vec::with_capacity(size);
10160 let mut cursor = 0;
10161
10162 for (index, operation) in operations.into_iter().enumerate() {
10163 match operation {
10164 ParallelOperation::Group(children) => {
10165 let child_base = base_sequence
10166 .checked_add(u64::try_from(cursor).unwrap_or(u64::MAX))
10167 .ok_or(Error::TimerDurationOverflow)?;
10168 for mut descriptor in parallel_descriptors(children, child_base)? {
10169 let outer_index = cursor + descriptor.offset;
10170 descriptor.group_path.insert(
10171 0,
10172 parallel_group_entry(base_sequence, size, outer_index, kind),
10173 );
10174 descriptor.member_path.insert(0, index);
10175 descriptor.offset = outer_index;
10176 descriptors.push(descriptor);
10177 }
10178 cursor = descriptors.len();
10179 }
10180 operation => {
10181 descriptors.push(ParallelDescriptor {
10182 operation,
10183 offset: cursor,
10184 member_path: vec![index],
10185 group_path: vec![parallel_group_entry(base_sequence, size, cursor, kind)],
10186 });
10187 cursor += 1;
10188 }
10189 }
10190 }
10191 Ok(descriptors)
10192}
10193
10194enum ParallelLeafCall {
10195 Activity(ActivityCall),
10196 ChildWorkflow(ChildWorkflowCall),
10197 Timer(TimerCall),
10198 Signal(SignalCall),
10199 Condition(ConditionWaitCall),
10200}
10201
10202fn parallel_leaf_call(
10203 ctx: &WorkflowContext,
10204 operation: ParallelOperation,
10205 parallel_group_path: Vec<ParallelGroupMetadata>,
10206) -> ParallelLeafCall {
10207 match operation {
10208 ParallelOperation::Activity {
10209 activity_type,
10210 options,
10211 arguments,
10212 } => ParallelLeafCall::Activity(ActivityCall {
10213 ctx: ctx.clone(),
10214 activity_type,
10215 options,
10216 args: Some(arguments),
10217 scheduled: false,
10218 parallel_group_path,
10219 }),
10220 ParallelOperation::ChildWorkflow {
10221 workflow_type,
10222 options,
10223 arguments,
10224 } => ParallelLeafCall::ChildWorkflow(ChildWorkflowCall {
10225 ctx: ctx.clone(),
10226 workflow_type,
10227 options,
10228 args: Some(arguments),
10229 scheduled: false,
10230 matched_pending: false,
10231 parallel_group_path,
10232 }),
10233 ParallelOperation::Timer(duration) => {
10234 let delay_seconds = duration
10235 .as_secs()
10236 .checked_add(u64::from(duration.subsec_nanos() > 0));
10237 ParallelLeafCall::Timer(TimerCall {
10238 ctx: ctx.clone(),
10239 delay_seconds,
10240 scheduled: false,
10241 matched_pending: false,
10242 parallel_group_path,
10243 })
10244 }
10245 ParallelOperation::Signal(signal_name) => ParallelLeafCall::Signal(SignalCall {
10246 ctx: ctx.clone(),
10247 signal_name,
10248 runtime_reserved_allowed: false,
10249 opened_wait: false,
10250 matched_pending: false,
10251 parallel_group_path,
10252 }),
10253 ParallelOperation::Condition { options, predicate } => {
10254 ParallelLeafCall::Condition(ConditionWaitCall {
10255 ctx: ctx.clone(),
10256 options,
10257 predicate,
10258 occurrence_id: None,
10259 opened_wait: false,
10260 parallel_group_path,
10261 })
10262 }
10263 ParallelOperation::Group(_) => {
10264 unreachable!("parallel descriptors contain only durable leaves")
10265 }
10266 }
10267}
10268
10269impl ParallelLeafCall {
10270 fn poll_avro_value(&mut self, cx: &mut TaskContext<'_>) -> Poll<Result<ParallelAvroResult>> {
10271 match self {
10272 Self::Activity(call) => Pin::new(call)
10273 .poll_avro_value(cx)
10274 .map_ok(ParallelAvroResult::Activity),
10275 Self::ChildWorkflow(call) => Pin::new(call)
10276 .poll_avro_value(cx)
10277 .map_ok(ParallelAvroResult::ChildWorkflow),
10278 Self::Timer(call) => Pin::new(call)
10279 .poll(cx)
10280 .map_ok(|()| ParallelAvroResult::Timer),
10281 Self::Signal(call) => Pin::new(call)
10282 .poll_avro_value(cx)
10283 .map_ok(ParallelAvroResult::Signal),
10284 Self::Condition(call) => Pin::new(call)
10285 .poll(cx)
10286 .map_ok(ParallelAvroResult::Condition),
10287 }
10288 }
10289}
10290
10291struct ParallelLeaf {
10292 call: ParallelLeafCall,
10293 member_path: Vec<usize>,
10294 group_path: Vec<ParallelGroupMetadata>,
10295 result: Option<ParallelAvroResult>,
10296}
10297
10298pub struct ParallelCall {
10300 ctx: WorkflowContext,
10301 operations: Option<Vec<ParallelOperation>>,
10302 shape: Option<ParallelShape>,
10303 leaves: Vec<ParallelLeaf>,
10304}
10305
10306impl ParallelCall {
10307 fn new(ctx: WorkflowContext, operations: Vec<ParallelOperation>) -> Self {
10308 Self {
10309 ctx,
10310 operations: Some(operations),
10311 shape: None,
10312 leaves: Vec::new(),
10313 }
10314 }
10315
10316 fn initialize(&mut self) -> Result<()> {
10317 let operations = self.operations.take().unwrap_or_default();
10318 validate_parallel_operations(&operations, &mut Vec::new(), true)?;
10319 self.shape = Some(parallel_shape(&operations));
10320 if operations.is_empty() {
10321 return Ok(());
10322 }
10323
10324 let base_sequence = {
10325 let state = self
10326 .ctx
10327 .state
10328 .lock()
10329 .map_err(|_| Error::WorkflowStatePoisoned)?;
10330 if let Some(recorded) = state.recorded_commands.get(state.command_cursor) {
10331 recorded.sequence()
10332 } else {
10333 let last = state
10334 .recorded_commands
10335 .last()
10336 .map(RecordedCommand::sequence)
10337 .unwrap_or(0);
10338 last.checked_add(u64::try_from(state.commands.len()).unwrap_or(u64::MAX))
10339 .and_then(|sequence| sequence.checked_add(1))
10340 .ok_or_else(|| {
10341 Error::InvalidParallelGroup(ParallelGroupError {
10342 reason: "sequence_overflow",
10343 member_path: Vec::new(),
10344 message: "parallel group sequence identity overflowed u64".to_string(),
10345 })
10346 })?
10347 }
10348 };
10349
10350 self.leaves = parallel_descriptors(operations, base_sequence)?
10351 .into_iter()
10352 .map(|descriptor| {
10353 let call = parallel_leaf_call(
10354 &self.ctx,
10355 descriptor.operation,
10356 descriptor.group_path.clone(),
10357 );
10358 ParallelLeaf {
10359 call,
10360 member_path: descriptor.member_path,
10361 group_path: descriptor.group_path,
10362 result: None,
10363 }
10364 })
10365 .collect();
10366 Ok(())
10367 }
10368
10369 fn poll_avro_value(
10370 mut self: Pin<&mut Self>,
10371 cx: &mut TaskContext<'_>,
10372 ) -> Poll<Result<Vec<ParallelAvroResult>>> {
10373 if self.operations.is_some() {
10374 if let Err(error) = self.initialize() {
10375 return Poll::Ready(Err(error));
10376 }
10377 }
10378 if self.leaves.is_empty() {
10379 return Poll::Ready(Ok(Vec::new()));
10380 }
10381
10382 let mut failures = Vec::new();
10383 let mut pending = false;
10384 for (index, leaf) in self.leaves.iter_mut().enumerate() {
10385 if leaf.result.is_some() {
10386 continue;
10387 }
10388 match leaf.call.poll_avro_value(cx) {
10389 Poll::Ready(Ok(result)) => leaf.result = Some(result),
10390 Poll::Ready(Err(error)) => failures.push((index, error)),
10391 Poll::Pending => pending = true,
10392 }
10393 }
10394
10395 if !failures.is_empty() {
10396 if let Some(position) = failures
10397 .iter()
10398 .position(|(_, error)| workflow_task_integrity_error(error))
10399 {
10400 return Poll::Ready(Err(failures.remove(position).1));
10401 }
10402 failures.sort_by_key(|(index, _)| *index);
10403 let (failed_index, cause) = failures.remove(0);
10404 let failed = &self.leaves[failed_index];
10405 let completed = self
10406 .leaves
10407 .iter()
10408 .filter_map(|leaf| {
10409 leaf.result
10410 .clone()
10411 .and_then(|result| result.into_json_result().ok())
10412 .map(|result| ParallelCompletion {
10413 member_path: leaf.member_path.clone(),
10414 result,
10415 })
10416 })
10417 .collect();
10418 let group_id = failed
10419 .group_path
10420 .first()
10421 .map(|entry| entry.parallel_group_id.clone())
10422 .unwrap_or_default();
10423 return Poll::Ready(Err(Error::ParallelFailed(ParallelFailure {
10424 group_id,
10425 member_path: failed.member_path.clone(),
10426 group_path: failed.group_path.clone(),
10427 completed,
10428 cause: Box::new(cause),
10429 })));
10430 }
10431 if pending {
10432 return Poll::Pending;
10433 }
10434
10435 let mut flat_results = self
10436 .leaves
10437 .iter_mut()
10438 .map(|leaf| leaf.result.take().expect("completed parallel leaf"))
10439 .collect::<Vec<_>>()
10440 .into_iter();
10441 let results = parallel_results_for_shape(
10442 self.shape.as_ref().expect("initialized parallel shape"),
10443 &mut flat_results,
10444 );
10445 Poll::Ready(Ok(match results {
10446 ParallelAvroResult::Group(results) => results,
10447 ParallelAvroResult::Activity(_)
10448 | ParallelAvroResult::ChildWorkflow(_)
10449 | ParallelAvroResult::Timer
10450 | ParallelAvroResult::Signal(_)
10451 | ParallelAvroResult::Condition(_) => {
10452 unreachable!("root parallel shape is a group")
10453 }
10454 }))
10455 }
10456}
10457
10458fn parallel_results_for_shape(
10459 shape: &ParallelShape,
10460 flat_results: &mut impl Iterator<Item = ParallelAvroResult>,
10461) -> ParallelAvroResult {
10462 match shape {
10463 ParallelShape::Leaf => flat_results.next().expect("one result per parallel leaf"),
10464 ParallelShape::Group(children) => ParallelAvroResult::Group(
10465 children
10466 .iter()
10467 .map(|child| parallel_results_for_shape(child, flat_results))
10468 .collect(),
10469 ),
10470 }
10471}
10472
10473impl Future for ParallelCall {
10474 type Output = Result<Vec<ParallelResult>>;
10475
10476 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
10477 self.poll_avro_value(cx)
10478 .map_ok(|results| {
10479 results
10480 .into_iter()
10481 .map(ParallelAvroResult::into_json_result)
10482 .collect::<Result<Vec<_>>>()
10483 })
10484 .map_ok(|result| result)
10485 .flatten_result()
10486 }
10487}
10488
10489#[derive(Clone, Debug)]
10490struct SelectionMemberPlan {
10491 key: SelectionKey,
10492 index: usize,
10493 base_sequence: u64,
10494 size: usize,
10495 kind: String,
10496 shape: ParallelShape,
10497 leaf_start: usize,
10498}
10499
10500fn selection_operation_kind(operation: &ParallelOperation) -> &'static str {
10501 match operation {
10502 ParallelOperation::Activity { .. } => "activity",
10503 ParallelOperation::ChildWorkflow { .. } => "child",
10504 ParallelOperation::Timer(_) => "timer",
10505 ParallelOperation::Signal(_) => "signal",
10506 ParallelOperation::Condition { .. } => "condition",
10507 ParallelOperation::Group(_) => "group",
10508 }
10509}
10510
10511fn selection_operation_shape(operation: &ParallelOperation) -> ParallelShape {
10512 match operation {
10513 ParallelOperation::Group(children) => parallel_shape(children),
10514 _ => ParallelShape::Leaf,
10515 }
10516}
10517
10518fn selection_descriptors(
10519 operations: Vec<(SelectionKey, ParallelOperation)>,
10520 base_sequence: u64,
10521) -> Result<(Vec<ParallelDescriptor>, Vec<SelectionMemberPlan>)> {
10522 if operations.is_empty() {
10523 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10524 reason: "selection_empty",
10525 member_path: Vec::new(),
10526 message: "durable selection requires at least one operation".to_string(),
10527 }));
10528 }
10529 let operation_refs = operations
10530 .iter()
10531 .map(|(_, operation)| operation)
10532 .collect::<Vec<_>>();
10533 let total_size = operation_refs
10534 .iter()
10535 .map(|operation| match operation {
10536 ParallelOperation::Group(children) => parallel_leaf_count(children),
10537 _ => 1,
10538 })
10539 .sum::<usize>();
10540 if total_size > MAX_PARALLEL_OPERATIONS {
10541 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10542 reason: "fan_out_limit_exceeded",
10543 member_path: Vec::new(),
10544 message: format!(
10545 "selection contains {total_size} durable leaves; the limit is {MAX_PARALLEL_OPERATIONS}"
10546 ),
10547 }));
10548 }
10549 let group_kind = {
10550 let mut kind = None;
10551 for operation in &operation_refs {
10552 let operation_kind = parallel_operation_kind(operation).unwrap_or("mixed");
10553 match kind {
10554 None => kind = Some(operation_kind),
10555 Some(current) if current == operation_kind => {}
10556 Some(_) => {
10557 kind = Some("mixed");
10558 break;
10559 }
10560 }
10561 }
10562 kind.unwrap_or("mixed")
10563 };
10564
10565 let mut descriptors = Vec::with_capacity(total_size);
10566 let mut members = Vec::with_capacity(operations.len());
10567 let mut cursor = 0usize;
10568 let mut seen_keys: Vec<SelectionKey> = Vec::new();
10569 for (member_index, (key, operation)) in operations.into_iter().enumerate() {
10570 if matches!(&key, SelectionKey::Name(value) if value.is_empty()) {
10571 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10572 reason: "selection_key_invalid",
10573 member_path: vec![member_index],
10574 message: "selection member keys must be non-empty strings or non-negative integers"
10575 .to_string(),
10576 }));
10577 }
10578 if seen_keys.contains(&key) {
10579 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10580 reason: "selection_key_duplicate",
10581 member_path: vec![member_index],
10582 message: format!("selection member key {key:?} is duplicated"),
10583 }));
10584 }
10585 seen_keys.push(key.clone());
10586 let member_size = match &operation {
10587 ParallelOperation::Group(children) => parallel_leaf_count(children),
10588 _ => 1,
10589 };
10590 if member_size == 0 {
10591 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10592 reason: "selection_member_empty",
10593 member_path: vec![member_index],
10594 message: "a selection member must contain at least one durable leaf".to_string(),
10595 }));
10596 }
10597 let member_base = base_sequence
10598 .checked_add(u64::try_from(cursor).unwrap_or(u64::MAX))
10599 .ok_or(Error::TimerDurationOverflow)?;
10600 let member_kind = selection_operation_kind(&operation).to_string();
10601 let member_shape = selection_operation_shape(&operation);
10602 let leaf_start = descriptors.len();
10603 match operation {
10604 ParallelOperation::Group(children) => {
10605 validate_parallel_operations(&children, &mut vec![member_index], false)?;
10606 for mut descriptor in parallel_descriptors(children, member_base)? {
10607 let flat_index = cursor + descriptor.offset;
10608 descriptor.group_path.insert(
10609 0,
10610 selection_group_entry(
10611 base_sequence,
10612 total_size,
10613 flat_index,
10614 group_kind,
10615 &SelectionMemberMetadata {
10616 key: key.clone(),
10617 index: member_index,
10618 base_sequence: member_base,
10619 size: member_size,
10620 kind: member_kind.clone(),
10621 },
10622 ),
10623 );
10624 descriptor.member_path.insert(0, member_index);
10625 descriptor.offset = flat_index;
10626 descriptors.push(descriptor);
10627 }
10628 }
10629 operation => {
10630 validate_parallel_operations(
10631 std::slice::from_ref(&operation),
10632 &mut Vec::new(),
10633 true,
10634 )?;
10635 descriptors.push(ParallelDescriptor {
10636 operation,
10637 offset: cursor,
10638 member_path: vec![member_index],
10639 group_path: vec![selection_group_entry(
10640 base_sequence,
10641 total_size,
10642 cursor,
10643 group_kind,
10644 &SelectionMemberMetadata {
10645 key: key.clone(),
10646 index: member_index,
10647 base_sequence: member_base,
10648 size: member_size,
10649 kind: member_kind.clone(),
10650 },
10651 )],
10652 });
10653 }
10654 }
10655 members.push(SelectionMemberPlan {
10656 key,
10657 index: member_index,
10658 base_sequence: member_base,
10659 size: member_size,
10660 kind: member_kind,
10661 shape: member_shape,
10662 leaf_start,
10663 });
10664 cursor += member_size;
10665 }
10666 Ok((descriptors, members))
10667}
10668
10669struct SelectionLeaf {
10670 call: ParallelLeafCall,
10671 outcome: Option<Result<ParallelAvroResult>>,
10672}
10673
10674#[derive(Clone)]
10676pub struct DurableOperationHandle {
10677 ctx: WorkflowContext,
10678 pub key: SelectionKey,
10679 pub index: usize,
10680 pub kind: String,
10681 pub identity: String,
10682 pub base_sequence: u64,
10683 pub size: usize,
10684 pub selection_group_id: String,
10685 shape: ParallelShape,
10686}
10687
10688impl std::fmt::Debug for DurableOperationHandle {
10689 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10690 formatter
10691 .debug_struct("DurableOperationHandle")
10692 .field("key", &self.key)
10693 .field("index", &self.index)
10694 .field("kind", &self.kind)
10695 .field("identity", &self.identity)
10696 .field("base_sequence", &self.base_sequence)
10697 .field("size", &self.size)
10698 .field("selection_group_id", &self.selection_group_id)
10699 .finish()
10700 }
10701}
10702
10703impl DurableOperationHandle {
10704 pub fn await_result(&self) -> DurableOperationAwaitCall {
10706 DurableOperationAwaitCall {
10707 handle: self.clone(),
10708 }
10709 }
10710
10711 pub fn cancel(&self) -> CancelDurableOperationCall {
10715 CancelDurableOperationCall {
10716 handle: self.clone(),
10717 emitted: false,
10718 }
10719 }
10720}
10721
10722#[derive(Debug)]
10724pub struct SelectionResult {
10725 pub key: SelectionKey,
10726 pub index: usize,
10727 pub kind: String,
10728 pub identity: String,
10729 pub value: Option<ParallelResult>,
10730 pub failure: Option<Error>,
10731 pub winner: DurableOperationHandle,
10732 pub handles: Vec<DurableOperationHandle>,
10733}
10734
10735impl SelectionResult {
10736 pub fn succeeded(&self) -> bool {
10737 self.failure.is_none()
10738 }
10739
10740 pub fn handle(&self, key: &SelectionKey) -> Option<&DurableOperationHandle> {
10741 self.handles.iter().find(|handle| &handle.key == key)
10742 }
10743
10744 pub fn remaining(&self) -> Vec<&DurableOperationHandle> {
10745 self.handles
10746 .iter()
10747 .filter(|handle| handle.index != self.index)
10748 .collect()
10749 }
10750
10751 pub fn into_result(self) -> Result<ParallelResult> {
10752 match (self.value, self.failure) {
10753 (Some(value), None) => Ok(value),
10754 (_, Some(error)) => Err(error),
10755 _ => Err(Error::WorkerLoop(
10756 "selection result contained neither a value nor a failure".to_string(),
10757 )),
10758 }
10759 }
10760}
10761
10762pub struct SelectCall {
10764 ctx: WorkflowContext,
10765 operations: Option<Vec<(SelectionKey, ParallelOperation)>>,
10766 members: Vec<SelectionMemberPlan>,
10767 leaves: Vec<SelectionLeaf>,
10768 group_id: Option<String>,
10769}
10770
10771impl SelectCall {
10772 fn new(ctx: WorkflowContext, operations: Vec<(SelectionKey, ParallelOperation)>) -> Self {
10773 Self {
10774 ctx,
10775 operations: Some(operations),
10776 members: Vec::new(),
10777 leaves: Vec::new(),
10778 group_id: None,
10779 }
10780 }
10781
10782 fn initialize(&mut self) -> Result<()> {
10783 let operations = self.operations.take().unwrap_or_default();
10784 let base_sequence = {
10785 let state = self
10786 .ctx
10787 .state
10788 .lock()
10789 .map_err(|_| Error::WorkflowStatePoisoned)?;
10790 if let Some(marker) = state.selection_markers.get(state.selection_marker_cursor) {
10791 marker.selection_group_base_sequence
10792 } else if let Some(recorded) = state.recorded_commands.get(state.command_cursor) {
10793 recorded.sequence()
10794 } else {
10795 let last = state
10796 .recorded_commands
10797 .last()
10798 .map(RecordedCommand::sequence)
10799 .unwrap_or(0);
10800 last.checked_add(u64::try_from(state.commands.len()).unwrap_or(u64::MAX))
10801 .and_then(|sequence| sequence.checked_add(1))
10802 .ok_or(Error::TimerDurationOverflow)?
10803 }
10804 };
10805 let (descriptors, members) = selection_descriptors(operations, base_sequence)?;
10806 let group_id = format!("select-calls:{base_sequence}:{}", descriptors.len());
10807 self.leaves = descriptors
10808 .into_iter()
10809 .map(|descriptor| SelectionLeaf {
10810 call: parallel_leaf_call(&self.ctx, descriptor.operation, descriptor.group_path),
10811 outcome: None,
10812 })
10813 .collect();
10814 self.members = members;
10815 self.group_id = Some(group_id);
10816 Ok(())
10817 }
10818}
10819
10820impl Future for SelectCall {
10821 type Output = Result<SelectionResult>;
10822
10823 fn poll(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
10824 if self.operations.is_some() {
10825 if let Err(error) = self.initialize() {
10826 return Poll::Ready(Err(error));
10827 }
10828 }
10829
10830 for leaf in &mut self.leaves {
10831 if leaf.outcome.is_some() {
10832 continue;
10833 }
10834 if let Poll::Ready(outcome) = leaf.call.poll_avro_value(cx) {
10835 if outcome
10836 .as_ref()
10837 .err()
10838 .is_some_and(workflow_task_integrity_error)
10839 {
10840 return Poll::Ready(outcome.map(|_| unreachable!()));
10841 }
10842 leaf.outcome = Some(outcome);
10843 }
10844 }
10845
10846 let all_members_terminal = self.leaves.iter().all(|leaf| leaf.outcome.is_some());
10847 let selection_member_range = self
10848 .members
10849 .first()
10850 .map(|member| member.base_sequence)
10851 .zip(self.leaves.len().try_into().ok())
10852 .map(|(base_sequence, size): (u64, u64)| {
10853 base_sequence..base_sequence.saturating_add(size)
10854 });
10855 let marker = {
10856 let mut state = match self.ctx.state.lock() {
10857 Ok(state) => state,
10858 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
10859 };
10860 let marker = state
10861 .selection_markers
10862 .get(state.selection_marker_cursor)
10863 .cloned();
10864 if marker.is_none()
10865 && all_members_terminal
10866 && selection_member_range.as_ref().is_some_and(|member_range| {
10867 state
10868 .recorded_commands
10869 .iter()
10870 .any(|command| member_range.contains(&command.sequence()))
10871 })
10872 {
10873 state.matched_recorded_pending = true;
10879 }
10880 marker
10881 };
10882 let Some(marker) = marker else {
10883 return Poll::Pending;
10884 };
10885 if self.group_id.as_deref() != Some(marker.selection_group_id.as_str())
10886 || marker.selection_group_size != self.leaves.len()
10887 || self.members.first().map(|member| member.base_sequence)
10888 != Some(marker.selection_group_base_sequence)
10889 {
10890 return Poll::Ready(Err(invalid_recorded_history(
10891 "selection_group_shape_mismatch",
10892 marker.selection_group_base_sequence,
10893 self.group_id
10894 .as_deref()
10895 .unwrap_or("initialized selection group"),
10896 &marker.selection_group_id,
10897 "recorded selection group differs from current workflow code",
10898 )));
10899 }
10900 let Some(member_position) = self.members.iter().position(|member| {
10901 member.key == marker.member_key
10902 && member.index == marker.member_index
10903 && member.base_sequence == marker.member_base_sequence
10904 && member.size == marker.member_size
10905 && member.kind == marker.operation_kind
10906 }) else {
10907 return Poll::Ready(Err(invalid_recorded_history(
10908 "selection_member_shape_mismatch",
10909 marker.member_base_sequence,
10910 "winner member matching current workflow code",
10911 &format!("{:?}", marker.member_key),
10912 "recorded selection winner differs from the authored member identity",
10913 )));
10914 };
10915 let member = self.members[member_position].clone();
10916 let (handles, resolution_sequence) = {
10917 let mut state = match self.ctx.state.lock() {
10918 Ok(state) => state,
10919 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
10920 };
10921 let identities = self
10922 .members
10923 .iter()
10924 .map(|candidate| {
10925 selection_operation_identity(
10926 &state,
10927 &candidate.kind,
10928 candidate.base_sequence,
10929 candidate.size,
10930 )
10931 })
10932 .collect::<Vec<_>>();
10933 if let Some((position, missing)) = identities
10934 .iter()
10935 .enumerate()
10936 .find(|(_, identity)| identity.is_empty())
10937 .map(|(position, identity)| (position, identity.clone()))
10938 {
10939 let candidate = &self.members[position];
10940 return Poll::Ready(Err(invalid_recorded_history(
10941 "selection_operation_identity_missing",
10942 candidate.base_sequence,
10943 &format!(
10944 "durable {} resource identity from scheduled/open history",
10945 candidate.kind
10946 ),
10947 &missing,
10948 "selection member history is missing its canonical durable identity",
10949 )));
10950 }
10951 let expected_winner_identity = &identities[member_position];
10952 let resolution_sequence = match validated_selection_resolution_sequence(
10953 &state,
10954 &marker,
10955 &member,
10956 expected_winner_identity,
10957 ) {
10958 Ok(sequence) => sequence,
10959 Err(error) => return Poll::Ready(Err(error)),
10960 };
10961 let handles = self
10962 .members
10963 .iter()
10964 .zip(identities)
10965 .map(|(member, identity)| DurableOperationHandle {
10966 ctx: self.ctx.clone(),
10967 key: member.key.clone(),
10968 index: member.index,
10969 kind: member.kind.clone(),
10970 identity,
10971 base_sequence: member.base_sequence,
10972 size: member.size,
10973 selection_group_id: marker.selection_group_id.clone(),
10974 shape: member.shape.clone(),
10975 })
10976 .collect::<Vec<_>>();
10977 if let Err(error) = validate_selection_cancellations_for_handles(&state, &handles) {
10978 return Poll::Ready(Err(error));
10979 }
10980 state.selection_marker_cursor += 1;
10981 (handles, resolution_sequence)
10982 };
10983
10984 let mut winner_failure = None;
10985 let mut flat_results = Vec::with_capacity(member.size);
10986 if marker.outcome == "failed" {
10987 let resolution_offset = match resolution_sequence
10988 .checked_sub(member.base_sequence)
10989 .and_then(|offset| usize::try_from(offset).ok())
10990 {
10991 Some(offset) if offset < member.size => offset,
10992 _ => {
10993 return Poll::Ready(Err(invalid_recorded_history(
10994 "selection_resolution_event_mismatch",
10995 member.base_sequence,
10996 "failure event within selected member bounds",
10997 &resolution_sequence.to_string(),
10998 "selection failure event is outside the authored member",
10999 )))
11000 }
11001 };
11002 let leaf = &mut self.leaves[member.leaf_start + resolution_offset];
11003 match leaf.outcome.take() {
11004 Some(Err(error)) => winner_failure = Some(error),
11005 _ => {
11006 return Poll::Ready(Err(invalid_recorded_history(
11007 "selection_winner_outcome_mismatch",
11008 member.base_sequence,
11009 "exact failed terminal history referenced by SelectionResolved",
11010 "missing or successful resolution event",
11011 "selection winner marker disagrees with terminal operation history",
11012 )))
11013 }
11014 }
11015 } else {
11016 for leaf in &mut self.leaves[member.leaf_start..member.leaf_start + member.size] {
11017 match leaf.outcome.take() {
11018 Some(Ok(result)) => flat_results.push(result),
11019 Some(Err(_)) => {
11020 return Poll::Ready(Err(invalid_recorded_history(
11021 "selection_winner_outcome_mismatch",
11022 member.base_sequence,
11023 "fully completed nested selection member",
11024 "failed durable leaf",
11025 "completed selection winner contains a failed leaf",
11026 )))
11027 }
11028 None => {
11029 return Poll::Ready(Err(invalid_recorded_history(
11030 "selection_winner_unresolved",
11031 member.base_sequence,
11032 "terminal history for every completed winner leaf",
11033 "pending member history",
11034 "completed SelectionResolved member has an unfinished durable barrier",
11035 )))
11036 }
11037 }
11038 }
11039 }
11040 let value = if winner_failure.is_none() {
11041 let mut flat_results = flat_results.into_iter();
11042 let value = parallel_results_for_shape(&member.shape, &mut flat_results);
11043 match value.into_json_result() {
11044 Ok(value) => Some(value),
11045 Err(error) => return Poll::Ready(Err(error)),
11046 }
11047 } else {
11048 None
11049 };
11050 let winner = handles[member_position].clone();
11051 Poll::Ready(Ok(SelectionResult {
11052 key: winner.key.clone(),
11053 index: winner.index,
11054 kind: winner.kind.clone(),
11055 identity: winner.identity.clone(),
11056 value,
11057 failure: winner_failure,
11058 winner,
11059 handles,
11060 }))
11061 }
11062}
11063
11064fn selection_operation_identity(
11065 state: &WorkflowState,
11066 kind: &str,
11067 base_sequence: u64,
11068 size: usize,
11069) -> String {
11070 if kind == "group" {
11071 return format!("group:{base_sequence}:{size}");
11072 }
11073 let fields: &[&str] = match kind {
11074 "activity" => &["activity_execution_id"],
11075 "child" => &["child_workflow_run_id"],
11076 "timer" => &["timer_id"],
11077 "signal" => &["signal_wait_id"],
11078 "condition" => &["condition_wait_id"],
11079 _ => &[],
11080 };
11081 for sequence in base_sequence..base_sequence.saturating_add(size as u64) {
11082 for event in state
11083 .history_events
11084 .iter()
11085 .filter(|event| durable_event_sequence(event) == Some(sequence))
11086 {
11087 for field in fields {
11088 if let Some(identity) = event.payload.get(*field).and_then(Value::as_str) {
11089 if !identity.is_empty() {
11090 return identity.to_string();
11091 }
11092 }
11093 }
11094 }
11095 }
11096 String::new()
11097}
11098
11099fn validated_selection_resolution_sequence(
11100 state: &WorkflowState,
11101 marker: &SelectionMarker,
11102 member: &SelectionMemberPlan,
11103 expected_identity: &str,
11104) -> Result<u64> {
11105 if expected_identity.is_empty() {
11106 return Err(invalid_recorded_history(
11107 "selection_operation_identity_missing",
11108 member.base_sequence,
11109 &format!(
11110 "durable {} resource identity from scheduled/open history",
11111 member.kind
11112 ),
11113 "missing operation identity",
11114 "selection member history is missing its canonical durable identity",
11115 ));
11116 }
11117 if marker.operation_identity != expected_identity {
11118 return Err(invalid_recorded_history(
11119 "selection_operation_identity_mismatch",
11120 member.base_sequence,
11121 expected_identity,
11122 &marker.operation_identity,
11123 "selection winner identity does not match durable scheduled/open history",
11124 ));
11125 }
11126
11127 let failure_types = [
11128 "ActivityFailed",
11129 "ActivityCancelled",
11130 "ActivityTimedOut",
11131 "ChildRunFailed",
11132 "ChildRunCancelled",
11133 "ChildRunTerminated",
11134 ];
11135 let success_types = [
11136 "ActivityCompleted",
11137 "ChildRunCompleted",
11138 "TimerFired",
11139 "SignalApplied",
11140 "ConditionWaitSatisfied",
11141 "ConditionWaitTimedOut",
11142 ];
11143 let terminal_types: &[&str] = if marker.outcome == "failed" {
11144 &failure_types
11145 } else {
11146 &success_types
11147 };
11148 let mut candidates = Vec::new();
11149 for event in state.history_events.iter() {
11150 let Some(sequence) = durable_event_sequence(event) else {
11151 continue;
11152 };
11153 if sequence < member.base_sequence
11154 || sequence >= member.base_sequence.saturating_add(member.size as u64)
11155 || !terminal_types.contains(&event.event_type.as_str())
11156 {
11157 continue;
11158 }
11159 let event_id = event
11160 .raw
11161 .get("id")
11162 .or_else(|| event.raw.get("event_id"))
11163 .and_then(Value::as_str)
11164 .filter(|value| !value.is_empty())
11165 .ok_or_else(|| {
11166 invalid_recorded_history(
11167 "selection_resolution_event_id_missing",
11168 member.base_sequence,
11169 "terminal selection history with a durable event id",
11170 &event.payload.to_string(),
11171 "selection terminal history cannot be bound to its winner marker",
11172 )
11173 })?;
11174 candidates.push((event_id.to_string(), event.event_type.clone(), sequence));
11175 }
11176 let resolution = if marker.outcome == "failed" {
11177 candidates.first()
11178 } else {
11179 candidates.last()
11180 };
11181 let Some((event_id, event_type, sequence)) = resolution else {
11182 return Err(invalid_recorded_history(
11183 "selection_resolution_event_missing",
11184 member.base_sequence,
11185 "terminal history for the selected member",
11186 &format!("{:?}", marker.member_key),
11187 "selection winner marker has no matching durable terminal event",
11188 ));
11189 };
11190 if event_id != &marker.resolution_event_id || event_type != &marker.resolution_event_type {
11191 return Err(invalid_recorded_history(
11192 "selection_resolution_event_mismatch",
11193 member.base_sequence,
11194 &format!("{event_type}:{event_id}"),
11195 &format!(
11196 "{}:{}",
11197 marker.resolution_event_type, marker.resolution_event_id
11198 ),
11199 "selection winner marker does not reference the event that made its member terminal",
11200 ));
11201 }
11202 Ok(*sequence)
11203}
11204
11205fn recorded_selection_member_outcome(
11206 state: &WorkflowState,
11207 handle: &DurableOperationHandle,
11208) -> Result<Option<ParallelResult>> {
11209 for event in state.history_events.iter() {
11210 let Some(sequence) = durable_event_sequence(event) else {
11211 continue;
11212 };
11213 if sequence < handle.base_sequence
11214 || sequence >= handle.base_sequence.saturating_add(handle.size as u64)
11215 || !matches!(
11216 event.event_type.as_str(),
11217 "ActivityFailed"
11218 | "ActivityCancelled"
11219 | "ActivityTimedOut"
11220 | "ChildRunFailed"
11221 | "ChildRunCancelled"
11222 | "ChildRunTerminated"
11223 )
11224 {
11225 continue;
11226 }
11227 let Some(command) = state
11228 .recorded_commands
11229 .iter()
11230 .find(|command| command.sequence() == sequence)
11231 else {
11232 continue;
11233 };
11234 match command {
11235 RecordedCommand::Activity {
11236 outcome: Some(Err(failure)),
11237 ..
11238 } => return Err(Error::ActivityFailed(failure.clone())),
11239 RecordedCommand::ChildWorkflow {
11240 outcome: Some(Err(failure)),
11241 ..
11242 } => return Err(Error::ChildWorkflowFailed(failure.clone())),
11243 _ => {}
11244 }
11245 }
11246
11247 let mut results = Vec::with_capacity(handle.size);
11248 for sequence in handle.base_sequence..handle.base_sequence.saturating_add(handle.size as u64) {
11249 let Some(command) = state
11250 .recorded_commands
11251 .iter()
11252 .find(|command| command.sequence() == sequence)
11253 else {
11254 return Ok(None);
11255 };
11256 let result = match command {
11257 RecordedCommand::Activity { outcome, .. } => match outcome {
11258 Some(Ok(value)) => ParallelAvroResult::Activity(value.clone()),
11259 Some(Err(failure)) => return Err(Error::ActivityFailed(failure.clone())),
11260 None => return Ok(None),
11261 },
11262 RecordedCommand::Timer { fired, .. } => {
11263 if !fired {
11264 return Ok(None);
11265 }
11266 ParallelAvroResult::Timer
11267 }
11268 RecordedCommand::ChildWorkflow { outcome, .. } => match outcome {
11269 Some(Ok(value)) => ParallelAvroResult::ChildWorkflow(value.clone()),
11270 Some(Err(failure)) => return Err(Error::ChildWorkflowFailed(failure.clone())),
11271 None => return Ok(None),
11272 },
11273 RecordedCommand::SignalWait { value, .. } => match value {
11274 Some(value) => ParallelAvroResult::Signal(value.clone()),
11275 None => return Ok(None),
11276 },
11277 RecordedCommand::ConditionWait { result, .. } => match result {
11278 Some(result) => ParallelAvroResult::Condition(*result),
11279 None => return Ok(None),
11280 },
11281 other => {
11282 return Err(command_mismatch(
11283 other,
11284 format!("selected {} member", handle.kind),
11285 ))
11286 }
11287 };
11288 results.push(result);
11289 }
11290 let mut results = results.into_iter();
11291 parallel_results_for_shape(&handle.shape, &mut results)
11292 .into_json_result()
11293 .map(Some)
11294}
11295
11296fn recorded_selection_member_is_terminal(
11297 state: &WorkflowState,
11298 handle: &DurableOperationHandle,
11299) -> bool {
11300 let mut completed = 0usize;
11301 let mut all_completed = true;
11302 for sequence in handle.base_sequence..handle.base_sequence.saturating_add(handle.size as u64) {
11303 let Some(command) = state
11304 .recorded_commands
11305 .iter()
11306 .find(|command| command.sequence() == sequence)
11307 else {
11308 all_completed = false;
11309 continue;
11310 };
11311 let terminal = match command {
11312 RecordedCommand::Activity {
11313 outcome: Some(Err(_)),
11314 ..
11315 }
11316 | RecordedCommand::ChildWorkflow {
11317 outcome: Some(Err(_)),
11318 ..
11319 } => return true,
11320 RecordedCommand::Activity { outcome, .. } => outcome.is_some(),
11321 RecordedCommand::ChildWorkflow { outcome, .. } => outcome.is_some(),
11322 RecordedCommand::Timer { fired, .. } => *fired,
11323 RecordedCommand::SignalWait { value, .. } => value.is_some(),
11324 RecordedCommand::ConditionWait { result, .. } => result.is_some(),
11325 RecordedCommand::SearchAttributes { .. }
11326 | RecordedCommand::SideEffect { .. }
11327 | RecordedCommand::VersionMarker { .. }
11328 | RecordedCommand::Memo { .. } => false,
11329 };
11330 if !terminal {
11331 all_completed = false;
11332 continue;
11333 }
11334 completed += 1;
11335 }
11336 all_completed && completed == handle.size
11337}
11338
11339fn selection_cancellation_for_handle(
11340 state: &WorkflowState,
11341 handle: &DurableOperationHandle,
11342) -> Result<bool> {
11343 let Some(marker) = state.cancelled_selection_members.iter().find(|recorded| {
11344 recorded.selection_group_id == handle.selection_group_id
11345 && recorded.member_base_sequence == handle.base_sequence
11346 }) else {
11347 return Ok(false);
11348 };
11349 validate_selection_cancellation_marker(marker, handle)?;
11350 Ok(true)
11351}
11352
11353fn validate_selection_cancellations_for_handles(
11354 state: &WorkflowState,
11355 handles: &[DurableOperationHandle],
11356) -> Result<()> {
11357 let Some(group_id) = handles
11358 .first()
11359 .map(|handle| handle.selection_group_id.as_str())
11360 else {
11361 return Ok(());
11362 };
11363 for marker in state
11364 .cancelled_selection_members
11365 .iter()
11366 .filter(|marker| marker.selection_group_id == group_id)
11367 {
11368 let Some(handle) = handles
11369 .iter()
11370 .find(|handle| handle.base_sequence == marker.member_base_sequence)
11371 else {
11372 return Err(invalid_recorded_history(
11373 "selection_cancellation_member_mismatch",
11374 marker.member_base_sequence,
11375 "SelectionOperationCancelled matching an authored selection handle",
11376 &format!("{marker:?}"),
11377 "selection cancellation member base does not name an authored member",
11378 ));
11379 };
11380 validate_selection_cancellation_marker(marker, handle)?;
11381 }
11382 Ok(())
11383}
11384
11385fn validate_selection_cancellation_marker(
11386 marker: &SelectionCancellation,
11387 handle: &DurableOperationHandle,
11388) -> Result<()> {
11389 if marker.selection_group_id != handle.selection_group_id
11390 || marker.member_key != handle.key
11391 || marker.member_index != handle.index
11392 || marker.member_base_sequence != handle.base_sequence
11393 || marker.member_size != handle.size
11394 || marker.operation_kind != handle.kind
11395 || marker.operation_identity != handle.identity
11396 {
11397 return Err(invalid_recorded_history(
11398 "selection_cancellation_member_mismatch",
11399 handle.base_sequence,
11400 "SelectionOperationCancelled matching the authored selection handle",
11401 &format!("{marker:?}"),
11402 "selection cancellation history targets different authored member metadata",
11403 ));
11404 }
11405 Ok(())
11406}
11407
11408pub struct DurableOperationAwaitCall {
11410 handle: DurableOperationHandle,
11411}
11412
11413impl Future for DurableOperationAwaitCall {
11414 type Output = Result<ParallelResult>;
11415
11416 fn poll(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11417 let state = match self.handle.ctx.state.lock() {
11418 Ok(state) => state,
11419 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11420 };
11421 match selection_cancellation_for_handle(&state, &self.handle) {
11422 Err(error) => return Poll::Ready(Err(error)),
11423 Ok(false) => {}
11424 Ok(true) => {
11425 return Poll::Ready(Err(Error::DurableOperationCancelled(
11426 DurableOperationCancelled {
11427 selection_group_id: self.handle.selection_group_id.clone(),
11428 member_key: self.handle.key.clone(),
11429 member_index: self.handle.index,
11430 operation_kind: self.handle.kind.clone(),
11431 operation_identity: self.handle.identity.clone(),
11432 },
11433 )));
11434 }
11435 }
11436 match recorded_selection_member_outcome(&state, &self.handle) {
11437 Ok(Some(result)) => Poll::Ready(Ok(result)),
11438 Ok(None) => Poll::Pending,
11439 Err(error) => Poll::Ready(Err(error)),
11440 }
11441 }
11442}
11443
11444pub struct CancelDurableOperationCall {
11446 handle: DurableOperationHandle,
11447 emitted: bool,
11448}
11449
11450impl Future for CancelDurableOperationCall {
11451 type Output = Result<()>;
11452
11453 fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11454 let ctx = self.handle.ctx.clone();
11455 let mut state = match ctx.state.lock() {
11456 Ok(state) => state,
11457 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11458 };
11459 match selection_cancellation_for_handle(&state, &self.handle) {
11460 Err(error) => return Poll::Ready(Err(error)),
11461 Ok(true) => return Poll::Ready(Ok(())),
11462 Ok(false) => {}
11463 }
11464 if recorded_selection_member_is_terminal(&state, &self.handle) {
11465 return Poll::Ready(Ok(()));
11466 }
11467 if !self.emitted {
11468 state.commands.push(json!({
11469 "type": "cancel_selection_operation",
11470 "selection_group_id": self.handle.selection_group_id,
11471 "member_key": self.handle.key,
11472 "member_index": self.handle.index,
11473 "member_base_sequence": self.handle.base_sequence,
11474 "member_size": self.handle.size,
11475 "operation_kind": self.handle.kind,
11476 "operation_identity": self.handle.identity,
11477 }));
11478 self.emitted = true;
11479 }
11480 Poll::Pending
11484 }
11485}
11486
11487trait PollNestedResultExt<T> {
11488 fn flatten_result(self) -> Poll<Result<T>>;
11489}
11490
11491impl<T> PollNestedResultExt<T> for Poll<Result<Result<T>>> {
11492 fn flatten_result(self) -> Poll<Result<T>> {
11493 match self {
11494 Poll::Ready(Ok(result)) => Poll::Ready(result),
11495 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
11496 Poll::Pending => Poll::Pending,
11497 }
11498 }
11499}
11500
11501struct SagaCompensation {
11502 activity_type: String,
11503 options: ActivityOptions,
11504 arguments: AvroValue,
11505 registration_order: usize,
11506}
11507
11508pub struct Saga {
11516 ctx: WorkflowContext,
11517 compensations: Vec<SagaCompensation>,
11518}
11519
11520impl Saga {
11521 fn new(ctx: WorkflowContext) -> Self {
11522 Self {
11523 ctx,
11524 compensations: Vec::new(),
11525 }
11526 }
11527
11528 pub fn add_compensation<T: Serialize>(
11529 &mut self,
11530 activity_type: impl Into<String>,
11531 args: T,
11532 ) -> Result<&mut Self> {
11533 self.add_compensation_with_options(activity_type, ActivityOptions::new(), args)
11534 }
11535
11536 pub fn add_compensation_with_options<T: Serialize>(
11537 &mut self,
11538 activity_type: impl Into<String>,
11539 options: ActivityOptions,
11540 args: T,
11541 ) -> Result<&mut Self> {
11542 let activity_type = activity_type.into();
11543 if activity_type.trim().is_empty() || activity_type.trim() != activity_type {
11544 return Err(Error::Codec(
11545 "saga compensation activity type must be non-empty without surrounding whitespace"
11546 .to_string(),
11547 ));
11548 }
11549 options.validate().map_err(Error::InvalidActivityOptions)?;
11550 let arguments = AvroValue::from_serialize(&args)?;
11551 let registration_order = self.compensations.len() + 1;
11552 self.compensations.push(SagaCompensation {
11553 activity_type,
11554 options,
11555 arguments,
11556 registration_order,
11557 });
11558 Ok(self)
11559 }
11560
11561 pub async fn compensate(mut self, initiating_failure: Error) -> Error {
11563 while let Some(compensation) = self.compensations.pop() {
11564 if let Err(compensation_failure) = self
11565 .ctx
11566 .activity_with_options(
11567 compensation.activity_type.clone(),
11568 compensation.options,
11569 compensation.arguments,
11570 )
11571 .await
11572 {
11573 if workflow_task_integrity_error(&compensation_failure) {
11574 return compensation_failure;
11575 }
11576 return Error::SagaCompensationFailed(SagaCompensationFailure {
11577 initiating_failure: Box::new(initiating_failure),
11578 compensation_failure: Box::new(compensation_failure),
11579 compensation_activity_type: compensation.activity_type,
11580 compensation_registration_order: compensation.registration_order,
11581 });
11582 }
11583 }
11584 initiating_failure
11585 }
11586
11587 pub async fn finish<T>(self, outcome: Result<T>) -> Result<T> {
11589 match outcome {
11590 Ok(value) => Ok(value),
11591 Err(error) => Err(self.compensate(error).await),
11592 }
11593 }
11594}
11595
11596pub struct ActivityCall {
11597 ctx: WorkflowContext,
11598 activity_type: String,
11599 options: ActivityOptions,
11600 args: Option<Result<AvroValue>>,
11601 scheduled: bool,
11602 parallel_group_path: Vec<ParallelGroupMetadata>,
11603}
11604
11605impl ActivityCall {
11606 fn poll_avro_value(
11607 mut self: Pin<&mut Self>,
11608 _cx: &mut TaskContext<'_>,
11609 ) -> Poll<Result<AvroValue>> {
11610 let ctx = self.ctx.clone();
11611 let mut state = match ctx.state.lock() {
11612 Ok(state) => state,
11613 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11614 };
11615
11616 if self.scheduled {
11617 return Poll::Pending;
11618 }
11619
11620 let options = match self.options.validate() {
11621 Ok(options) => options,
11622 Err(error) => {
11623 return Poll::Ready(Err(Error::InvalidActivityOptions(error)));
11624 }
11625 };
11626 let task_queue = options
11627 .task_queue
11628 .clone()
11629 .unwrap_or_else(|| state.task_queue.clone());
11630 let current_recorded_options = RecordedActivityOptions {
11631 task_queue: RecordedSnapshotValue::Known(Some(task_queue.clone())),
11632 execution_mode: RecordedSnapshotValue::Known(None),
11635 retry_policy: current_activity_retry_snapshot(&options),
11636 };
11637
11638 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
11639 let sequence = recorded.sequence();
11640 match recorded {
11641 RecordedCommand::Activity {
11642 activity_type,
11643 options: recorded_options,
11644 outcome,
11645 parallel_group_path,
11646 ..
11647 } => {
11648 if let Err(error) = ensure_parallel_path_matches(
11649 sequence,
11650 parallel_group_path.as_deref(),
11651 &self.parallel_group_path,
11652 ) {
11653 return Poll::Ready(Err(error));
11654 }
11655 if let Some(recorded_type) = activity_type {
11656 if recorded_type != self.activity_type {
11657 return Poll::Ready(Err(Error::NonDeterministicReplay(
11658 ReplayFailure::new(
11659 "recorded_command_detail_mismatch",
11660 Some(sequence),
11661 Some(format!("activity:{recorded_type}")),
11662 Some(format!("activity:{}", self.activity_type)),
11663 "recorded activity type differs from the current workflow command",
11664 ),
11665 )));
11666 }
11667 }
11668 if let Some(recorded_options) = recorded_options {
11669 if !recorded_options
11670 .task_queue
11671 .matches_current(¤t_recorded_options.task_queue)
11672 {
11673 return Poll::Ready(Err(Error::NonDeterministicReplay(
11674 ReplayFailure::new(
11675 "activity_task_queue_mismatch",
11676 Some(sequence),
11677 Some(activity_options_description(&recorded_options)),
11678 Some(activity_options_description(¤t_recorded_options)),
11679 "recorded activity task queue differs from the current workflow command",
11680 ),
11681 )));
11682 }
11683 if !recorded_options
11684 .execution_mode
11685 .matches_current(¤t_recorded_options.execution_mode)
11686 {
11687 return Poll::Ready(Err(Error::NonDeterministicReplay(
11688 ReplayFailure::new(
11689 "activity_execution_mode_mismatch",
11690 Some(sequence),
11691 Some(activity_options_description(&recorded_options)),
11692 Some(activity_options_description(¤t_recorded_options)),
11693 "recorded activity execution mode differs from the current workflow command",
11694 ),
11695 )));
11696 }
11697 if !recorded_options
11698 .retry_policy
11699 .matches_current(¤t_recorded_options.retry_policy)
11700 {
11701 return Poll::Ready(Err(Error::NonDeterministicReplay(
11702 ReplayFailure::new(
11703 "activity_retry_policy_mismatch",
11704 Some(sequence),
11705 Some(activity_options_description(&recorded_options)),
11706 Some(activity_options_description(¤t_recorded_options)),
11707 "recorded activity retry policy differs from the current workflow command",
11708 ),
11709 )));
11710 }
11711 }
11712 state.command_cursor += 1;
11713 if let Some(outcome) = outcome {
11714 return Poll::Ready(outcome.map_err(Error::ActivityFailed));
11715 }
11716 state.matched_recorded_pending = true;
11717 self.scheduled = true;
11718 return Poll::Pending;
11719 }
11720 other => {
11721 return Poll::Ready(Err(command_mismatch(
11722 &other,
11723 format!("activity:{}", self.activity_type),
11724 )));
11725 }
11726 }
11727 }
11728
11729 if !self.scheduled {
11730 let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
11731 Ok(args) => args,
11732 Err(error) => return Poll::Ready(Err(error)),
11733 };
11734 let arguments = normalize_avro_arguments(args);
11735 let envelope = match encode_typed_envelope(&arguments, &state.payload_codec) {
11736 Ok(envelope) => envelope,
11737 Err(error) => return Poll::Ready(Err(error)),
11738 };
11739
11740 let mut command = serde_json::Map::from_iter([
11741 ("type".to_string(), json!("schedule_activity")),
11742 (
11743 "activity_type".to_string(),
11744 json!(self.activity_type.clone()),
11745 ),
11746 ("queue".to_string(), json!(task_queue)),
11747 ("arguments".to_string(), envelope),
11748 ]);
11749 for (field, value) in [
11750 ("start_to_close_timeout", options.start_to_close_timeout),
11751 (
11752 "schedule_to_start_timeout",
11753 options.schedule_to_start_timeout,
11754 ),
11755 (
11756 "schedule_to_close_timeout",
11757 options.schedule_to_close_timeout,
11758 ),
11759 ("heartbeat_timeout", options.heartbeat_timeout),
11760 ] {
11761 if let Some(value) = value {
11762 command.insert(field.to_string(), json!(value));
11763 }
11764 }
11765 if let Some(retry_policy) = options.retry_policy {
11766 command.insert("retry_policy".to_string(), retry_policy);
11767 }
11768 apply_parallel_group_path(&mut command, &self.parallel_group_path);
11769 state.commands.push(Value::Object(command));
11770 self.scheduled = true;
11771 }
11772
11773 Poll::Pending
11774 }
11775}
11776
11777impl Future for ActivityCall {
11778 type Output = Result<Value>;
11779
11780 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11781 match self.poll_avro_value(cx) {
11782 Poll::Ready(Ok(value)) => Poll::Ready(value.into_json()),
11783 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
11784 Poll::Pending => Poll::Pending,
11785 }
11786 }
11787}
11788
11789pub struct TimerCall {
11791 ctx: WorkflowContext,
11792 delay_seconds: Option<u64>,
11793 scheduled: bool,
11794 matched_pending: bool,
11795 parallel_group_path: Vec<ParallelGroupMetadata>,
11796}
11797
11798impl Future for TimerCall {
11799 type Output = Result<()>;
11800
11801 fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11802 if self.matched_pending {
11803 return Poll::Pending;
11804 }
11805
11806 let ctx = self.ctx.clone();
11807 let Some(requested_delay) = self.delay_seconds else {
11808 return Poll::Ready(Err(Error::TimerDurationOverflow));
11809 };
11810 let mut state = match ctx.state.lock() {
11811 Ok(state) => state,
11812 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11813 };
11814
11815 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
11816 match recorded {
11817 RecordedCommand::Timer {
11818 sequence,
11819 delay_seconds,
11820 fired,
11821 parallel_group_path,
11822 ..
11823 } => {
11824 if let Err(error) = ensure_parallel_path_matches(
11825 sequence,
11826 parallel_group_path.as_deref(),
11827 &self.parallel_group_path,
11828 ) {
11829 return Poll::Ready(Err(error));
11830 }
11831 if delay_seconds != requested_delay {
11832 return Poll::Ready(Err(Error::NonDeterministicReplay(
11833 ReplayFailure::new(
11834 "timer_delay_mismatch",
11835 Some(sequence),
11836 Some(format!("timer:{delay_seconds}s")),
11837 Some(format!("timer:{requested_delay}s")),
11838 "recorded timer delay differs from the current workflow command",
11839 ),
11840 )));
11841 }
11842 state.command_cursor += 1;
11843 if fired {
11844 return Poll::Ready(Ok(()));
11845 }
11846 state.matched_recorded_pending = true;
11847 self.scheduled = true;
11848 self.matched_pending = true;
11849 return Poll::Pending;
11850 }
11851 other => return Poll::Ready(Err(command_mismatch(&other, "timer"))),
11852 }
11853 }
11854
11855 if !self.scheduled {
11856 let mut command = serde_json::Map::from_iter([
11857 ("type".to_string(), json!("start_timer")),
11858 ("delay_seconds".to_string(), json!(requested_delay)),
11859 ]);
11860 apply_parallel_group_path(&mut command, &self.parallel_group_path);
11861 state.commands.push(Value::Object(command));
11862 self.scheduled = true;
11863 }
11864
11865 Poll::Pending
11866 }
11867}
11868
11869pub struct ConditionWaitCall {
11871 ctx: WorkflowContext,
11872 options: ConditionWaitOptions,
11873 predicate: Box<dyn Fn() -> Result<bool> + Send + 'static>,
11874 occurrence_id: Option<String>,
11875 opened_wait: bool,
11876 parallel_group_path: Vec<ParallelGroupMetadata>,
11877}
11878
11879impl Future for ConditionWaitCall {
11880 type Output = Result<ConditionWaitResult>;
11881
11882 fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11883 if self.opened_wait {
11884 return Poll::Pending;
11885 }
11886
11887 let options = match self.options.validate() {
11888 Ok(options) => options,
11889 Err(error) => return Poll::Ready(Err(Error::InvalidConditionWaitOptions(error))),
11890 };
11891 let ctx = self.ctx.clone();
11892 let occurrence_id = match self.occurrence_id.as_ref() {
11893 Some(occurrence_id) => occurrence_id.clone(),
11894 None => {
11895 let mut state = match ctx.state.lock() {
11896 Ok(state) => state,
11897 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11898 };
11899 let ordinal = state.condition_wait_occurrence_counter;
11900 state.condition_wait_occurrence_counter = match ordinal.checked_add(1) {
11901 Some(next) => next,
11902 None => {
11903 return Poll::Ready(Err(Error::WorkerLoop(
11904 "condition wait occurrence counter overflowed".to_string(),
11905 )))
11906 }
11907 };
11908 let occurrence_id = format!("{CONDITION_WAIT_OCCURRENCE_PREFIX}{ordinal}");
11909 drop(state);
11910 self.occurrence_id = Some(occurrence_id.clone());
11911 occurrence_id
11912 }
11913 };
11914
11915 let recorded_result = {
11916 let mut state = match ctx.state.lock() {
11917 Ok(state) => state,
11918 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11919 };
11920 let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() else {
11921 drop(state);
11922 return self.poll_new_condition(options);
11923 };
11924 if !matches!(recorded, RecordedCommand::ConditionWait { .. }) {
11925 return Poll::Ready(Err(command_mismatch(&recorded, "condition wait")));
11926 }
11927
11928 let mut cursor = state.command_cursor;
11929 let mut result = None;
11930 loop {
11931 let Some(RecordedCommand::ConditionWait {
11932 sequence,
11933 occurrence_id: recorded_occurrence_id,
11934 condition_key,
11935 predicate_identity,
11936 timeout_seconds,
11937 result: recorded_result,
11938 parallel_group_path,
11939 ..
11940 }) = state.recorded_commands.get(cursor)
11941 else {
11942 break;
11943 };
11944
11945 if cursor > state.command_cursor && recorded_occurrence_id != &occurrence_id {
11946 break;
11947 }
11948 if let Err(error) = ensure_parallel_path_matches(
11949 *sequence,
11950 parallel_group_path.as_deref(),
11951 &self.parallel_group_path,
11952 ) {
11953 return Poll::Ready(Err(error));
11954 }
11955 if let Err(error) = validate_recorded_condition_wait(
11956 *sequence,
11957 recorded_occurrence_id,
11958 condition_key.as_deref(),
11959 predicate_identity,
11960 *timeout_seconds,
11961 &occurrence_id,
11962 &options,
11963 ) {
11964 return Poll::Ready(Err(error));
11965 }
11966 if result == Some(ConditionWaitResult::TimedOut) {
11967 return Poll::Ready(Err(Error::NonDeterministicReplay(ReplayFailure::new(
11968 "condition_wait_reopened_after_timeout",
11969 Some(*sequence),
11970 Some("timed-out condition is terminal".to_string()),
11971 Some("another physical wait-open".to_string()),
11972 "condition history reopened one logical wait after its durable timeout",
11973 ))));
11974 }
11975 result = *recorded_result;
11976 cursor += 1;
11977 }
11978 state.command_cursor = cursor;
11979 result
11980 };
11981
11982 if let Some(result) = recorded_result {
11983 return Poll::Ready(Ok(result));
11984 }
11985
11986 self.poll_open_condition(options)
11987 }
11988}
11989
11990impl ConditionWaitCall {
11991 fn poll_new_condition(
11992 self: Pin<&mut Self>,
11993 options: ValidatedConditionWaitOptions,
11994 ) -> Poll<Result<ConditionWaitResult>> {
11995 self.poll_open_condition(options)
11996 }
11997
11998 fn poll_open_condition(
11999 mut self: Pin<&mut Self>,
12000 options: ValidatedConditionWaitOptions,
12001 ) -> Poll<Result<ConditionWaitResult>> {
12002 let selection_member = self
12003 .parallel_group_path
12004 .first()
12005 .is_some_and(|entry| entry.parallel_group_mode.as_deref() == Some("select"));
12006 match (self.predicate)() {
12007 Ok(true) if !selection_member => {
12008 return Poll::Ready(Ok(ConditionWaitResult::Satisfied))
12009 }
12010 Ok(_) => {}
12011 Err(error) => return Poll::Ready(Err(error)),
12012 }
12013 if options.timeout_seconds == Some(0) && !selection_member {
12014 return Poll::Ready(Ok(ConditionWaitResult::TimedOut));
12015 }
12016
12017 let ctx = self.ctx.clone();
12018 let mut state = match ctx.state.lock() {
12019 Ok(state) => state,
12020 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12021 };
12022 let mut command = serde_json::Map::from_iter([
12023 ("type".to_string(), json!("open_condition_wait")),
12024 (
12025 "condition_wait_occurrence_id".to_string(),
12026 json!(self.occurrence_id.as_deref().unwrap_or_default()),
12027 ),
12028 ("condition_key".to_string(), json!(options.condition_key)),
12029 (
12030 "condition_definition_fingerprint".to_string(),
12031 json!(options.predicate_identity),
12032 ),
12033 ]);
12034 if let Some(timeout_seconds) = options.timeout_seconds {
12035 command.insert("timeout_seconds".to_string(), json!(timeout_seconds));
12036 }
12037 apply_parallel_group_path(&mut command, &self.parallel_group_path);
12038 state.commands.push(Value::Object(command));
12039 drop(state);
12040 self.opened_wait = true;
12041 Poll::Pending
12042 }
12043}
12044
12045fn validate_recorded_condition_wait(
12046 sequence: u64,
12047 recorded_occurrence_id: &str,
12048 recorded_key: Option<&str>,
12049 recorded_predicate_identity: &str,
12050 recorded_timeout_seconds: Option<u64>,
12051 current_occurrence_id: &str,
12052 current: &ValidatedConditionWaitOptions,
12053) -> Result<()> {
12054 if recorded_occurrence_id != current_occurrence_id {
12055 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12056 "condition_wait_occurrence_mismatch",
12057 Some(sequence),
12058 Some(recorded_occurrence_id.to_string()),
12059 Some(current_occurrence_id.to_string()),
12060 "recorded condition occurrence differs from the current authored wait position",
12061 )));
12062 }
12063 if recorded_key != Some(current.condition_key.as_str()) {
12064 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12065 "condition_wait_key_mismatch",
12066 Some(sequence),
12067 recorded_key.map(str::to_string),
12068 Some(current.condition_key.clone()),
12069 "recorded condition identity differs from the current workflow wait",
12070 )));
12071 }
12072 if recorded_predicate_identity != current.predicate_identity {
12073 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12074 "condition_wait_predicate_mismatch",
12075 Some(sequence),
12076 Some(recorded_predicate_identity.to_string()),
12077 Some(current.predicate_identity.clone()),
12078 "recorded condition predicate behavior differs from current workflow code",
12079 )));
12080 }
12081 if recorded_timeout_seconds != current.timeout_seconds {
12082 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12083 "condition_wait_timeout_mismatch",
12084 Some(sequence),
12085 recorded_timeout_seconds.map(|seconds| format!("{seconds}s")),
12086 current.timeout_seconds.map(|seconds| format!("{seconds}s")),
12087 "recorded condition timeout differs from the current workflow wait",
12088 )));
12089 }
12090 Ok(())
12091}
12092
12093pub struct ChildWorkflowCall {
12095 ctx: WorkflowContext,
12096 workflow_type: String,
12097 options: ChildWorkflowOptions,
12098 args: Option<Result<AvroValue>>,
12099 scheduled: bool,
12100 matched_pending: bool,
12101 parallel_group_path: Vec<ParallelGroupMetadata>,
12102}
12103
12104impl ChildWorkflowCall {
12105 fn poll_avro_value(
12106 mut self: Pin<&mut Self>,
12107 _cx: &mut TaskContext<'_>,
12108 ) -> Poll<Result<ChildWorkflowAvroResult>> {
12109 if self.matched_pending {
12110 return Poll::Pending;
12111 }
12112
12113 let ctx = self.ctx.clone();
12114 let mut state = match ctx.state.lock() {
12115 Ok(state) => state,
12116 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12117 };
12118
12119 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
12120 let sequence = recorded.sequence();
12121 match recorded {
12122 RecordedCommand::ChildWorkflow {
12123 workflow_type,
12124 outcome,
12125 parallel_group_path,
12126 ..
12127 } => {
12128 if let Err(error) = ensure_parallel_path_matches(
12129 sequence,
12130 parallel_group_path.as_deref(),
12131 &self.parallel_group_path,
12132 ) {
12133 return Poll::Ready(Err(error));
12134 }
12135 if let Some(recorded_type) = workflow_type {
12136 if recorded_type != self.workflow_type {
12137 return Poll::Ready(Err(Error::NonDeterministicReplay(
12138 ReplayFailure::new(
12139 "recorded_command_detail_mismatch",
12140 Some(sequence),
12141 Some(format!("child workflow:{recorded_type}")),
12142 Some(format!("child workflow:{}", self.workflow_type)),
12143 "recorded child workflow type differs from the current workflow command",
12144 ),
12145 )));
12146 }
12147 }
12148 state.command_cursor += 1;
12149 if let Some(outcome) = outcome {
12150 return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed));
12151 }
12152 state.matched_recorded_pending = true;
12153 self.scheduled = true;
12154 self.matched_pending = true;
12155 return Poll::Pending;
12156 }
12157 other => {
12158 return Poll::Ready(Err(command_mismatch(
12159 &other,
12160 format!("child workflow:{}", self.workflow_type),
12161 )));
12162 }
12163 }
12164 }
12165
12166 if !self.scheduled {
12167 if self.options.task_queue.trim().is_empty() {
12168 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
12169 "task_queue must not be empty".to_string(),
12170 )));
12171 }
12172 for (name, value) in [
12173 (
12174 "execution_timeout_seconds",
12175 self.options.execution_timeout_seconds,
12176 ),
12177 ("run_timeout_seconds", self.options.run_timeout_seconds),
12178 ] {
12179 if value == Some(0) {
12180 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(format!(
12181 "{name} must be at least 1"
12182 ))));
12183 }
12184 }
12185
12186 let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
12187 Ok(args) => args,
12188 Err(error) => return Poll::Ready(Err(error)),
12189 };
12190 let arguments = match encode_typed_envelope(
12191 &normalize_avro_arguments(args),
12192 &state.payload_codec,
12193 ) {
12194 Ok(arguments) => arguments,
12195 Err(error) => return Poll::Ready(Err(error)),
12196 };
12197 let mut command = json!({
12198 "type": "start_child_workflow",
12199 "workflow_type": self.workflow_type,
12200 "queue": self.options.task_queue,
12201 "parent_close_policy": self.options.parent_close_policy.as_str(),
12202 "arguments": arguments,
12203 });
12204 let object = command
12205 .as_object_mut()
12206 .expect("child workflow command is always an object");
12207 if let Some(policy) = &self.options.retry_policy {
12208 let mut retry_policy = serde_json::Map::new();
12209 if let Some(max_attempts) = policy.max_attempts {
12210 if max_attempts == 0 {
12211 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
12212 "retry_policy.max_attempts must be at least 1".to_string(),
12213 )));
12214 }
12215 retry_policy.insert("max_attempts".to_string(), json!(max_attempts));
12216 }
12217 if !policy.backoff_seconds.is_empty() {
12218 retry_policy
12219 .insert("backoff_seconds".to_string(), json!(policy.backoff_seconds));
12220 }
12221 if !policy.non_retryable_error_types.is_empty() {
12222 retry_policy.insert(
12223 "non_retryable_error_types".to_string(),
12224 json!(policy.non_retryable_error_types),
12225 );
12226 }
12227 if retry_policy.is_empty() {
12228 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
12229 "retry_policy must configure at least one field".to_string(),
12230 )));
12231 }
12232 object.insert("retry_policy".to_string(), Value::Object(retry_policy));
12233 }
12234 if let Some(seconds) = self.options.execution_timeout_seconds {
12235 object.insert("execution_timeout_seconds".to_string(), json!(seconds));
12236 }
12237 if let Some(seconds) = self.options.run_timeout_seconds {
12238 object.insert("run_timeout_seconds".to_string(), json!(seconds));
12239 }
12240 apply_parallel_group_path(object, &self.parallel_group_path);
12241 state.commands.push(command);
12242 self.scheduled = true;
12243 }
12244
12245 Poll::Pending
12246 }
12247}
12248
12249impl Future for ChildWorkflowCall {
12250 type Output = Result<ChildWorkflowResult>;
12251
12252 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12253 match self.poll_avro_value(cx) {
12254 Poll::Ready(Ok(result)) => match result.result.into_json() {
12255 Ok(projected) => Poll::Ready(Ok(ChildWorkflowResult {
12256 parent: result.parent,
12257 child: result.child,
12258 child_workflow_type: result.child_workflow_type,
12259 result: projected,
12260 })),
12261 Err(error) => Poll::Ready(Err(error)),
12262 },
12263 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
12264 Poll::Pending => Poll::Pending,
12265 }
12266 }
12267}
12268
12269fn command_mismatch(recorded: &RecordedCommand, actual: impl Into<String>) -> Error {
12270 Error::NonDeterministicReplay(ReplayFailure::new(
12271 "recorded_command_mismatch",
12272 Some(recorded.sequence()),
12273 Some(recorded.shape().to_string()),
12274 Some(actual.into()),
12275 "current workflow command does not match the recorded durable command sequence",
12276 ))
12277}
12278
12279pub struct SignalCall {
12280 ctx: WorkflowContext,
12281 signal_name: String,
12282 runtime_reserved_allowed: bool,
12283 opened_wait: bool,
12284 matched_pending: bool,
12285 parallel_group_path: Vec<ParallelGroupMetadata>,
12286}
12287
12288impl SignalCall {
12289 fn poll_avro_value(
12290 mut self: Pin<&mut Self>,
12291 _cx: &mut TaskContext<'_>,
12292 ) -> Poll<Result<Vec<AvroValue>>> {
12293 if self.matched_pending {
12294 return Poll::Pending;
12295 }
12296 if !self.runtime_reserved_allowed {
12297 if let Err(error) = validate_user_signal_name(&self.signal_name) {
12298 return Poll::Ready(Err(error));
12299 }
12300 }
12301
12302 let ctx = self.ctx.clone();
12303 let mut state = match ctx.state.lock() {
12304 Ok(state) => state,
12305 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12306 };
12307
12308 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
12309 match recorded {
12310 RecordedCommand::SignalWait {
12311 sequence,
12312 signal_name,
12313 value,
12314 parallel_group_path,
12315 } => {
12316 if let Err(error) = ensure_parallel_path_matches(
12317 sequence,
12318 parallel_group_path.as_deref(),
12319 &self.parallel_group_path,
12320 ) {
12321 return Poll::Ready(Err(error));
12322 }
12323 if signal_name != self.signal_name {
12324 return Poll::Ready(Err(Error::NonDeterministicReplay(
12325 ReplayFailure::new(
12326 "recorded_command_detail_mismatch",
12327 Some(sequence),
12328 Some(format!("signal wait:{signal_name}")),
12329 Some(format!("signal wait:{}", self.signal_name)),
12330 "recorded signal name differs from the current workflow command",
12331 ),
12332 )));
12333 }
12334
12335 state.command_cursor += 1;
12336 if let Some(value) = value {
12337 return Poll::Ready(Ok(value));
12338 }
12339 if state
12340 .resume_signal
12341 .as_ref()
12342 .is_some_and(|signal| signal.signal_name == self.signal_name)
12343 {
12344 let signal = state
12345 .resume_signal
12346 .take()
12347 .expect("matching resume signal is present");
12348 return Poll::Ready(Ok(signal.arguments));
12349 }
12350
12351 state.matched_recorded_pending = true;
12352 self.opened_wait = true;
12353 self.matched_pending = true;
12354 return Poll::Pending;
12355 }
12356 other => {
12357 return Poll::Ready(Err(command_mismatch(
12358 &other,
12359 format!("signal wait:{}", self.signal_name),
12360 )));
12361 }
12362 }
12363 }
12364
12365 if state
12366 .resume_signal
12367 .as_ref()
12368 .is_some_and(|signal| signal.signal_name == self.signal_name)
12369 {
12370 let signal = state
12371 .resume_signal
12372 .take()
12373 .expect("matching resume signal is present");
12374 return Poll::Ready(Ok(signal.arguments));
12375 }
12376
12377 if !self.opened_wait {
12378 let mut command = serde_json::Map::from_iter([
12379 ("type".to_string(), json!("open_signal_wait")),
12380 ("signal_name".to_string(), json!(self.signal_name)),
12381 ]);
12382 apply_parallel_group_path(&mut command, &self.parallel_group_path);
12383 state.commands.push(Value::Object(command));
12384 self.opened_wait = true;
12385 }
12386
12387 Poll::Pending
12388 }
12389}
12390
12391impl Future for SignalCall {
12392 type Output = Result<Vec<Value>>;
12393
12394 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12395 match self.poll_avro_value(cx) {
12396 Poll::Ready(Ok(values)) => Poll::Ready(
12397 values
12398 .into_iter()
12399 .map(AvroValue::into_json)
12400 .collect::<Result<Vec<_>>>(),
12401 ),
12402 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
12403 Poll::Pending => Poll::Pending,
12404 }
12405 }
12406}
12407
12408#[derive(Clone, Debug)]
12409pub struct ActivityContext {
12410 client: Client,
12411 pub task_id: String,
12412 pub activity_attempt_id: String,
12413 pub lease_owner: String,
12414 pub activity_type: String,
12415 pub attempt_number: u64,
12416 pub task_queue: String,
12417 pub worker_id: String,
12418}
12419
12420impl ActivityContext {
12421 pub async fn heartbeat<T: Serialize>(&self, details: T) -> Result<ActivityHeartbeatResponse> {
12422 self.client
12423 .heartbeat_activity_task(
12424 &self.task_id,
12425 &self.activity_attempt_id,
12426 &self.lease_owner,
12427 details,
12428 )
12429 .await
12430 }
12431}
12432
12433fn decode_task_avro_arguments(value: Option<&Value>, codec: &str) -> Result<AvroValue> {
12434 validate_payload_codec(codec)?;
12435 match value {
12436 Some(value) => Ok(normalize_avro_arguments(decode_wire_avro_value(
12437 value, codec,
12438 )?)),
12439 None => Ok(AvroValue::Array(Vec::new())),
12440 }
12441}
12442
12443fn decode_resume_signal(task: &WorkflowTask) -> Result<Option<ResumeSignal>> {
12444 let Some(signal_name) = task
12445 .signal_name
12446 .as_deref()
12447 .filter(|value| !value.is_empty())
12448 else {
12449 return Ok(None);
12450 };
12451 let decoded = decode_task_avro_arguments(task.signal_arguments.as_ref(), &task.payload_codec)?;
12452 let AvroValue::Array(arguments) = decoded else {
12453 unreachable!("normalize_avro_arguments always returns an array");
12454 };
12455
12456 Ok(Some(ResumeSignal {
12457 signal_name: signal_name.to_string(),
12458 arguments,
12459 }))
12460}
12461
12462fn validate_workflow_task_payloads(task: &WorkflowTask) -> Result<()> {
12463 validate_payload_codec(&task.payload_codec)?;
12464 validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)?;
12465 validate_optional_inbound_payload(task.signal_arguments.as_ref(), &task.payload_codec)?;
12466 for event in &task.history_events {
12467 validate_history_event_payloads(event, &task.payload_codec)?;
12468 }
12469 Ok(())
12470}
12471
12472fn validate_activity_task_payloads(task: &ActivityTask) -> Result<()> {
12473 validate_payload_codec(&task.payload_codec)?;
12474 validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)
12475}
12476
12477fn validate_query_task_payloads(task: &QueryTask) -> Result<()> {
12478 validate_payload_codec(&task.payload_codec)?;
12479 validate_optional_inbound_payload(task.workflow_arguments.as_ref(), &task.payload_codec)?;
12480 validate_optional_inbound_payload(task.query_arguments.as_ref(), &task.payload_codec)?;
12481 for event in &task.history_events {
12482 validate_history_event_payloads(event, &task.payload_codec)?;
12483 }
12484
12485 let Some(export) = task.history_export.as_ref() else {
12486 return Ok(());
12487 };
12488 let export_codec = match export.get("payloads") {
12489 Some(payloads) => declared_payload_codec(payloads, "codec")?,
12490 None => None,
12491 }
12492 .unwrap_or(&task.payload_codec);
12493 validate_payload_codec(export_codec)?;
12494
12495 if let Some(events) = export.get("history_events").and_then(Value::as_array) {
12496 for event in events {
12497 let event_type = event
12498 .get("event_type")
12499 .or_else(|| event.get("type"))
12500 .and_then(Value::as_str)
12501 .unwrap_or_default();
12502 if let Some(payload) = event.get("payload") {
12503 validate_history_payloads(event_type, payload, export_codec)?;
12504 }
12505 }
12506 }
12507 for signal in export
12508 .get("signals")
12509 .and_then(Value::as_array)
12510 .into_iter()
12511 .flatten()
12512 {
12513 let codec = declared_payload_codec(signal, "payload_codec")?.unwrap_or(export_codec);
12514 validate_payload_codec(codec)?;
12515 validate_optional_inbound_payload(signal.get("arguments"), codec)?;
12516 }
12517 for activity in export
12518 .get("activities")
12519 .and_then(Value::as_array)
12520 .into_iter()
12521 .flatten()
12522 {
12523 let codec = declared_payload_codec(activity, "payload_codec")?.unwrap_or(export_codec);
12524 validate_payload_codec(codec)?;
12525 validate_optional_inbound_payload(activity.get("arguments"), codec)?;
12526 validate_optional_inbound_payload(activity.get("result"), codec)?;
12527 }
12528 Ok(())
12529}
12530
12531fn validate_history_event_payloads(event: &HistoryEvent, fallback_codec: &str) -> Result<()> {
12532 validate_history_payloads(&event.event_type, &event.payload, fallback_codec)
12533}
12534
12535fn validate_history_payloads(
12536 event_type: &str,
12537 payload: &Value,
12538 fallback_codec: &str,
12539) -> Result<()> {
12540 let codec = declared_payload_codec(payload, "payload_codec")?.unwrap_or(fallback_codec);
12541 validate_payload_codec(codec)?;
12542 for field in history_payload_fields(event_type) {
12543 validate_optional_inbound_payload(payload.get(*field), codec)?;
12544 }
12545 Ok(())
12546}
12547
12548const SIGNAL_HISTORY_PAYLOAD_FIELDS: &[&str] = &["value", "input", "arguments"];
12549
12550fn history_payload_fields(event_type: &str) -> &'static [&'static str] {
12551 match event_type {
12552 "ActivityCompleted" => &["result"],
12553 "SignalReceived" | "SignalApplied" => SIGNAL_HISTORY_PAYLOAD_FIELDS,
12554 "UpdateAccepted" | "UpdateRejected" | "UpdateApplied" => &["arguments"],
12555 "UpdateCompleted" | "SideEffectRecorded" => &["result"],
12556 "ChildRunCompleted" => &["result", "output"],
12557 "WorkflowCompleted" => &["output"],
12558 "ServiceCallStarted"
12559 | "ServiceCallCompleted"
12560 | "ServiceCallFailed"
12561 | "ServiceCallCancelled" => &["request_payload", "response_payload"],
12562 _ => &[],
12563 }
12564}
12565
12566fn signal_history_payload(payload: &Value) -> Option<&Value> {
12567 SIGNAL_HISTORY_PAYLOAD_FIELDS
12568 .iter()
12569 .find_map(|field| payload.get(*field))
12570}
12571
12572fn declared_payload_codec<'a>(value: &'a Value, field: &str) -> Result<Option<&'a str>> {
12573 match value.get(field) {
12574 None => Ok(None),
12575 Some(Value::String(codec)) => Ok(Some(codec)),
12576 Some(_) => Err(invalid_payload_envelope()),
12577 }
12578}
12579
12580fn validate_optional_inbound_payload(value: Option<&Value>, codec: &str) -> Result<()> {
12581 validate_payload_codec(codec)?;
12582 if let Some(value) = value.filter(|value| !value.is_null()) {
12583 decode_wire_avro_value(value, codec)?;
12584 }
12585 Ok(())
12586}
12587
12588fn recorded_parallel_group_entry(payload: &Value, sequence: u64) -> Result<ParallelGroupMetadata> {
12589 let group_id = payload_string(payload, "parallel_group_id").ok_or_else(|| {
12590 invalid_recorded_history(
12591 "parallel_group_metadata_invalid",
12592 sequence,
12593 "non-empty parallel_group_id",
12594 &payload.to_string(),
12595 "parallel-group history is missing its stable identity",
12596 )
12597 })?;
12598 let kind = payload_string(payload, "parallel_group_kind").ok_or_else(|| {
12599 invalid_recorded_history(
12600 "parallel_group_metadata_invalid",
12601 sequence,
12602 "activity, child, timer, signal, condition, or mixed group kind",
12603 &payload.to_string(),
12604 "parallel-group history is missing its group kind",
12605 )
12606 })?;
12607 if !matches!(
12608 kind.as_str(),
12609 "activity" | "child" | "timer" | "signal" | "condition" | "mixed"
12610 ) {
12611 return Err(invalid_recorded_history(
12612 "parallel_group_metadata_invalid",
12613 sequence,
12614 "activity, child, timer, signal, condition, or mixed group kind",
12615 &kind,
12616 "parallel-group history contains an unsupported group kind",
12617 ));
12618 }
12619 let base_sequence = payload
12620 .get("parallel_group_base_sequence")
12621 .and_then(value_as_u64)
12622 .filter(|value| *value > 0)
12623 .ok_or_else(|| {
12624 invalid_recorded_history(
12625 "parallel_group_metadata_invalid",
12626 sequence,
12627 "positive parallel_group_base_sequence",
12628 &payload.to_string(),
12629 "parallel-group history contains an invalid base sequence",
12630 )
12631 })?;
12632 let size = payload
12633 .get("parallel_group_size")
12634 .and_then(value_as_u64)
12635 .and_then(|value| usize::try_from(value).ok())
12636 .filter(|value| (1..=MAX_PARALLEL_OPERATIONS).contains(value))
12637 .ok_or_else(|| {
12638 invalid_recorded_history(
12639 "parallel_group_metadata_invalid",
12640 sequence,
12641 "bounded positive parallel_group_size",
12642 &payload.to_string(),
12643 "parallel-group history contains an invalid group size",
12644 )
12645 })?;
12646 let index = payload
12647 .get("parallel_group_index")
12648 .and_then(value_as_u64)
12649 .and_then(|value| usize::try_from(value).ok())
12650 .filter(|value| *value < size)
12651 .ok_or_else(|| {
12652 invalid_recorded_history(
12653 "parallel_group_metadata_invalid",
12654 sequence,
12655 "parallel_group_index within group bounds",
12656 &payload.to_string(),
12657 "parallel-group history contains an invalid member index",
12658 )
12659 })?;
12660 if base_sequence.checked_add(u64::try_from(index).unwrap_or(u64::MAX)) != Some(sequence) {
12661 return Err(invalid_recorded_history(
12662 "parallel_group_metadata_invalid",
12663 sequence,
12664 "base sequence plus member index equals workflow sequence",
12665 &payload.to_string(),
12666 "parallel-group path does not preserve durable workflow position",
12667 ));
12668 }
12669 let mode = payload
12670 .get("parallel_group_mode")
12671 .and_then(Value::as_str)
12672 .unwrap_or("all");
12673 if !matches!(mode, "all" | "select") {
12674 return Err(invalid_recorded_history(
12675 "parallel_group_metadata_invalid",
12676 sequence,
12677 "parallel group mode all or select",
12678 mode,
12679 "parallel-group history contains an unsupported group mode",
12680 ));
12681 }
12682 let expected_id = if mode == "select" {
12683 format!("select-calls:{base_sequence}:{size}")
12684 } else {
12685 format!("{}:{base_sequence}:{size}", parallel_group_prefix(&kind))
12686 };
12687 if group_id != expected_id {
12688 return Err(invalid_recorded_history(
12689 "parallel_group_metadata_invalid",
12690 sequence,
12691 &expected_id,
12692 &group_id,
12693 "parallel-group history contains an incompatible stable group ID",
12694 ));
12695 }
12696 let selection_member_key = if mode == "select" {
12697 Some(selection_key_from_value(
12698 payload.get("selection_member_key"),
12699 sequence,
12700 )?)
12701 } else {
12702 None
12703 };
12704 let selection_member_index = if mode == "select" {
12705 Some(required_parallel_usize(
12706 payload,
12707 "selection_member_index",
12708 sequence,
12709 )?)
12710 } else {
12711 None
12712 };
12713 let selection_member_base_sequence = if mode == "select" {
12714 Some(
12715 payload
12716 .get("selection_member_base_sequence")
12717 .and_then(value_as_u64)
12718 .filter(|value| *value >= base_sequence)
12719 .ok_or_else(|| {
12720 invalid_recorded_history(
12721 "parallel_group_metadata_invalid",
12722 sequence,
12723 "selection member base within its group",
12724 &payload.to_string(),
12725 "selection history contains an invalid member base sequence",
12726 )
12727 })?,
12728 )
12729 } else {
12730 None
12731 };
12732 let selection_member_size = if mode == "select" {
12733 let member_size = required_parallel_usize(payload, "selection_member_size", sequence)?;
12734 if member_size == 0 {
12735 return Err(invalid_recorded_history(
12736 "parallel_group_metadata_invalid",
12737 sequence,
12738 "positive selection member size",
12739 &payload.to_string(),
12740 "selection history contains an invalid member size",
12741 ));
12742 }
12743 Some(member_size)
12744 } else {
12745 None
12746 };
12747 let selection_member_kind = if mode == "select" {
12748 let kind = payload_string(payload, "selection_member_kind").ok_or_else(|| {
12749 invalid_recorded_history(
12750 "parallel_group_metadata_invalid",
12751 sequence,
12752 "selection member operation kind",
12753 &payload.to_string(),
12754 "selection history is missing its authored member kind",
12755 )
12756 })?;
12757 if !matches!(
12758 kind.as_str(),
12759 "activity" | "child" | "timer" | "signal" | "condition" | "group"
12760 ) {
12761 return Err(invalid_recorded_history(
12762 "parallel_group_metadata_invalid",
12763 sequence,
12764 "activity, child, timer, signal, condition, or group selection member kind",
12765 &kind,
12766 "selection history contains an unsupported member kind",
12767 ));
12768 }
12769 Some(kind)
12770 } else {
12771 None
12772 };
12773 if let (Some(member_base), Some(member_size)) =
12774 (selection_member_base_sequence, selection_member_size)
12775 {
12776 let member_end = member_base
12777 .checked_add(u64::try_from(member_size).unwrap_or(u64::MAX))
12778 .ok_or_else(|| {
12779 invalid_recorded_history(
12780 "parallel_group_metadata_invalid",
12781 sequence,
12782 "bounded selection member range",
12783 &payload.to_string(),
12784 "selection member range overflowed",
12785 )
12786 })?;
12787 let group_end = base_sequence
12788 .checked_add(u64::try_from(size).unwrap_or(u64::MAX))
12789 .unwrap_or(u64::MAX);
12790 if sequence < member_base || sequence >= member_end || member_end > group_end {
12791 return Err(invalid_recorded_history(
12792 "parallel_group_metadata_invalid",
12793 sequence,
12794 "workflow sequence within one bounded selection member",
12795 &payload.to_string(),
12796 "selection member range does not contain its durable leaf",
12797 ));
12798 }
12799 }
12800 Ok(ParallelGroupMetadata {
12801 parallel_group_id: group_id,
12802 parallel_group_kind: kind,
12803 parallel_group_base_sequence: base_sequence,
12804 parallel_group_size: size,
12805 parallel_group_index: index,
12806 parallel_group_mode: (mode == "select").then(|| "select".to_string()),
12807 selection_member_key,
12808 selection_member_index,
12809 selection_member_base_sequence,
12810 selection_member_size,
12811 selection_member_kind,
12812 })
12813}
12814
12815fn required_parallel_usize(payload: &Value, field: &str, sequence: u64) -> Result<usize> {
12816 payload
12817 .get(field)
12818 .and_then(value_as_u64)
12819 .and_then(|value| usize::try_from(value).ok())
12820 .ok_or_else(|| {
12821 invalid_recorded_history(
12822 "parallel_group_metadata_invalid",
12823 sequence,
12824 &format!("non-negative integer {field}"),
12825 &payload.to_string(),
12826 "selection history contains invalid member metadata",
12827 )
12828 })
12829}
12830
12831fn selection_key_from_value(value: Option<&Value>, sequence: u64) -> Result<SelectionKey> {
12832 match value {
12833 Some(Value::String(value)) if !value.is_empty() => Ok(SelectionKey::Name(value.clone())),
12834 Some(value) => value_as_u64(value)
12835 .and_then(|value| usize::try_from(value).ok())
12836 .map(SelectionKey::Index)
12837 .ok_or_else(|| {
12838 invalid_recorded_history(
12839 "selection_member_key_invalid",
12840 sequence,
12841 "non-empty string or non-negative integer member key",
12842 &value.to_string(),
12843 "selection history contains an invalid member key",
12844 )
12845 }),
12846 None => Err(invalid_recorded_history(
12847 "selection_member_key_missing",
12848 sequence,
12849 "selection_member_key",
12850 "<missing>",
12851 "selection history is missing its stable member key",
12852 )),
12853 }
12854}
12855
12856fn recorded_parallel_group_path(
12857 events: &[&HistoryEvent],
12858 sequence: u64,
12859) -> Result<Option<Vec<ParallelGroupMetadata>>> {
12860 let mut recorded: Option<Vec<ParallelGroupMetadata>> = None;
12861 for event in events {
12862 let payload = &event.payload;
12863 let has_metadata = payload.get("parallel_group_path").is_some()
12864 || payload.get("parallel_group_id").is_some()
12865 || payload.get("parallel_group_kind").is_some()
12866 || payload.get("parallel_group_base_sequence").is_some()
12867 || payload.get("parallel_group_size").is_some()
12868 || payload.get("parallel_group_index").is_some()
12869 || payload.get("parallel_group_mode").is_some()
12870 || payload.get("selection_member_key").is_some();
12871 if !has_metadata {
12872 continue;
12873 }
12874
12875 let top_level = recorded_parallel_group_entry(payload, sequence)?;
12876 let path = match payload.get("parallel_group_path") {
12877 None => vec![top_level.clone()],
12878 Some(Value::Array(entries)) if !entries.is_empty() => entries
12879 .iter()
12880 .map(|entry| recorded_parallel_group_entry(entry, sequence))
12881 .collect::<Result<Vec<_>>>()?,
12882 Some(value) => {
12883 return Err(invalid_recorded_history(
12884 "parallel_group_metadata_invalid",
12885 sequence,
12886 "non-empty parallel_group_path list",
12887 &value.to_string(),
12888 "parallel-group history contains an invalid group path",
12889 ));
12890 }
12891 };
12892 if path.last() != Some(&top_level) {
12893 return Err(invalid_recorded_history(
12894 "parallel_group_metadata_invalid",
12895 sequence,
12896 &serde_json::to_string(&path.last()).unwrap_or_default(),
12897 &serde_json::to_string(&top_level).unwrap_or_default(),
12898 "parallel-group top-level fields do not match the innermost path entry",
12899 ));
12900 }
12901 if recorded.as_ref().is_some_and(|existing| existing != &path) {
12902 return Err(invalid_recorded_history(
12903 "parallel_group_history_conflict",
12904 sequence,
12905 &serde_json::to_string(&recorded.as_ref()).unwrap_or_default(),
12906 &serde_json::to_string(&path).unwrap_or_default(),
12907 "parallel-group metadata changed between scheduling and resolution history",
12908 ));
12909 }
12910 recorded = Some(path);
12911 }
12912 Ok(recorded)
12913}
12914
12915fn recorded_commands(
12916 events: &[HistoryEvent],
12917 fallback_codec: &str,
12918 parent: WorkflowIdentity,
12919) -> Result<Vec<RecordedCommand>> {
12920 let mut events_by_sequence: BTreeMap<u64, Vec<&HistoryEvent>> = BTreeMap::new();
12921 let mut last_new_sequence = None;
12922
12923 for event in events {
12924 let is_activity = matches!(
12925 event.event_type.as_str(),
12926 "ActivityScheduled"
12927 | "ActivityStarted"
12928 | "ActivityHeartbeatRecorded"
12929 | "ActivityRetryScheduled"
12930 | "ActivityCompleted"
12931 | "ActivityFailed"
12932 | "ActivityCancelled"
12933 | "ActivityTimedOut"
12934 );
12935 let is_workflow_timer = matches!(
12936 event.event_type.as_str(),
12937 "TimerScheduled" | "TimerCancelled" | "TimerFired"
12938 ) && !is_internal_timer_event(event);
12939 let is_child_workflow = matches!(
12940 event.event_type.as_str(),
12941 "ChildWorkflowScheduled"
12942 | "ChildRunCompleted"
12943 | "ChildRunFailed"
12944 | "ChildRunCancelled"
12945 | "ChildRunTerminated"
12946 );
12947 let is_signal_wait = is_recorded_signal_wait_event(event);
12948 let is_condition_wait = is_recorded_condition_wait_event(event);
12949 let is_search_attributes = event.event_type == "SearchAttributesUpserted";
12950 let is_side_effect = event.event_type == "SideEffectRecorded";
12951 let is_version_marker = event.event_type == "VersionMarkerRecorded";
12952 let is_memo = event.event_type == "MemoUpserted";
12953 if !is_activity
12954 && !is_workflow_timer
12955 && !is_child_workflow
12956 && !is_signal_wait
12957 && !is_condition_wait
12958 && !is_search_attributes
12959 && !is_side_effect
12960 && !is_version_marker
12961 && !is_memo
12962 {
12963 continue;
12964 }
12965
12966 let sequence = durable_event_sequence(event).ok_or_else(|| {
12967 Error::NonDeterministicReplay(ReplayFailure::new(
12968 "durable_command_sequence_missing",
12969 None,
12970 Some("positive workflow sequence".to_string()),
12971 Some(event.event_type.clone()),
12972 "durable command history event has no workflow sequence",
12973 ))
12974 })?;
12975 if sequence == 0 {
12976 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12977 "durable_command_sequence_invalid",
12978 Some(sequence),
12979 Some("positive workflow sequence".to_string()),
12980 Some(sequence.to_string()),
12981 "durable command history uses an invalid workflow sequence",
12982 )));
12983 }
12984 if !events_by_sequence.contains_key(&sequence) {
12985 if let Some(previous) = last_new_sequence {
12986 if sequence < previous {
12987 return Err(invalid_recorded_history(
12988 "durable_command_sequence_mismatch",
12989 sequence,
12990 &format!("workflow sequence greater than {previous}"),
12991 &sequence.to_string(),
12992 "durable commands are not strictly ordered by their recorded workflow sequence",
12993 ));
12994 }
12995 }
12996 last_new_sequence = Some(sequence);
12997 }
12998 events_by_sequence.entry(sequence).or_default().push(event);
12999 }
13000
13001 let commands: Vec<RecordedCommand> = events_by_sequence
13002 .into_iter()
13003 .map(|(sequence, sequence_events)| {
13004 let activity_events: Vec<_> = sequence_events
13005 .iter()
13006 .copied()
13007 .filter(|event| event.event_type.starts_with("Activity"))
13008 .collect();
13009 let timer_events: Vec<_> = sequence_events
13010 .iter()
13011 .copied()
13012 .filter(|event| event.event_type.starts_with("Timer"))
13013 .collect();
13014 let child_events: Vec<_> = sequence_events
13015 .iter()
13016 .copied()
13017 .filter(|event| {
13018 event.event_type == "ChildWorkflowScheduled"
13019 || event.event_type.starts_with("ChildRun")
13020 })
13021 .collect();
13022 let signal_wait_events: Vec<_> = sequence_events
13023 .iter()
13024 .copied()
13025 .filter(|event| is_recorded_signal_wait_event(event))
13026 .collect();
13027 let condition_wait_events: Vec<_> = sequence_events
13028 .iter()
13029 .copied()
13030 .filter(|event| is_recorded_condition_wait_event(event))
13031 .collect();
13032 let search_attribute_events: Vec<_> = sequence_events
13033 .iter()
13034 .copied()
13035 .filter(|event| event.event_type == "SearchAttributesUpserted")
13036 .collect();
13037 let side_effect_events: Vec<_> = sequence_events
13038 .iter()
13039 .copied()
13040 .filter(|event| event.event_type == "SideEffectRecorded")
13041 .collect();
13042 let version_marker_events: Vec<_> = sequence_events
13043 .iter()
13044 .copied()
13045 .filter(|event| event.event_type == "VersionMarkerRecorded")
13046 .collect();
13047 let memo_events: Vec<_> = sequence_events
13048 .iter()
13049 .copied()
13050 .filter(|event| event.event_type == "MemoUpserted")
13051 .collect();
13052
13053 let command_kind_count = usize::from(!activity_events.is_empty())
13054 + usize::from(!timer_events.is_empty())
13055 + usize::from(!child_events.is_empty())
13056 + usize::from(!signal_wait_events.is_empty())
13057 + usize::from(!condition_wait_events.is_empty())
13058 + usize::from(!search_attribute_events.is_empty())
13059 + usize::from(!side_effect_events.is_empty())
13060 + usize::from(!version_marker_events.is_empty())
13061 + usize::from(!memo_events.is_empty());
13062 if command_kind_count > 1 {
13063 let actual = [
13064 (!activity_events.is_empty()).then_some("activity"),
13065 (!timer_events.is_empty()).then_some("timer"),
13066 (!child_events.is_empty()).then_some("child workflow"),
13067 (!signal_wait_events.is_empty()).then_some("signal wait"),
13068 (!condition_wait_events.is_empty()).then_some("condition wait"),
13069 (!search_attribute_events.is_empty()).then_some("search-attribute update"),
13070 (!side_effect_events.is_empty()).then_some("side effect"),
13071 (!version_marker_events.is_empty()).then_some("version marker"),
13072 (!memo_events.is_empty()).then_some("memo upsert"),
13073 ]
13074 .into_iter()
13075 .flatten()
13076 .collect::<Vec<_>>()
13077 .join(" and ");
13078 return Err(invalid_recorded_history(
13079 "durable_command_sequence_collision",
13080 sequence,
13081 "one durable command kind",
13082 &actual,
13083 "one workflow sequence records more than one durable command kind",
13084 ));
13085 }
13086
13087 if !activity_events.is_empty() {
13088 let parallel_group_path =
13089 recorded_parallel_group_path(&activity_events, sequence)?;
13090 let scheduled_count = activity_events
13091 .iter()
13092 .filter(|event| event.event_type == "ActivityScheduled")
13093 .count();
13094 if scheduled_count > 1 {
13095 return Err(invalid_recorded_history(
13096 "duplicate_activity_schedule",
13097 sequence,
13098 "at most one ActivityScheduled event",
13099 "multiple ActivityScheduled events",
13100 "activity history schedules more than one command at one workflow sequence",
13101 ));
13102 }
13103 let activity_type = activity_events.iter().find_map(|event| {
13104 event
13105 .payload
13106 .get("activity_type")
13107 .or_else(|| event.payload.get("activity_name"))
13108 .and_then(Value::as_str)
13109 .map(str::to_string)
13110 });
13111 if activity_events.iter().filter_map(|event| {
13112 event
13113 .payload
13114 .get("activity_type")
13115 .or_else(|| event.payload.get("activity_name"))
13116 .and_then(Value::as_str)
13117 }).any(|candidate| Some(candidate) != activity_type.as_deref()) {
13118 return Err(invalid_recorded_history(
13119 "activity_identity_mismatch",
13120 sequence,
13121 activity_type.as_deref().unwrap_or("one activity identity"),
13122 "conflicting activity identities",
13123 "activity lifecycle events at one workflow sequence disagree on identity",
13124 ));
13125 }
13126 let terminal: Vec<_> = activity_events
13127 .iter()
13128 .copied()
13129 .filter(|event| {
13130 matches!(
13131 event.event_type.as_str(),
13132 "ActivityCompleted"
13133 | "ActivityFailed"
13134 | "ActivityCancelled"
13135 | "ActivityTimedOut"
13136 )
13137 })
13138 .collect();
13139 let duplicate_delivery = terminal.first().is_some_and(|first| {
13140 terminal.iter().all(|event| {
13141 event.event_type == first.event_type && event.payload == first.payload
13142 })
13143 });
13144 if terminal.len() > 1 && !duplicate_delivery {
13145 return Err(invalid_recorded_history(
13146 "duplicate_activity_terminal_event",
13147 sequence,
13148 "at most one terminal activity event",
13149 "multiple terminal activity events",
13150 "activity history settles one command more than once",
13151 ));
13152 }
13153 let outcome = terminal
13154 .first()
13155 .map(|event| activity_outcome(event, fallback_codec, activity_type.clone()))
13156 .transpose()?;
13157 let options = activity_events
13158 .iter()
13159 .find(|event| event.event_type == "ActivityScheduled")
13160 .and_then(|event| event.payload.get("activity"))
13161 .and_then(Value::as_object)
13162 .map(|activity| RecordedActivityOptions {
13163 task_queue: recorded_optional_string(activity, "queue"),
13164 execution_mode: recorded_optional_string(activity, "execution_mode"),
13165 retry_policy: recorded_activity_retry_snapshot(
13166 activity.get("retry_policy"),
13167 ),
13168 });
13169 return Ok(RecordedCommand::Activity {
13170 sequence,
13171 activity_type,
13172 options,
13173 outcome,
13174 parallel_group_path,
13175 });
13176 }
13177
13178 if !child_events.is_empty() {
13179 let parallel_group_path = recorded_parallel_group_path(&child_events, sequence)?;
13180 let scheduled: Vec<_> = child_events
13181 .iter()
13182 .copied()
13183 .filter(|event| event.event_type == "ChildWorkflowScheduled")
13184 .collect();
13185 if scheduled.len() != 1 {
13186 return Err(invalid_recorded_history(
13187 "child_workflow_schedule_missing_or_duplicate",
13188 sequence,
13189 "one ChildWorkflowScheduled event",
13190 &format!("{} ChildWorkflowScheduled events", scheduled.len()),
13191 "child workflow replay requires exactly one recorded schedule event",
13192 ));
13193 }
13194 let workflow_type = child_events.iter().find_map(|event| {
13195 event
13196 .payload
13197 .get("child_workflow_type")
13198 .or_else(|| event.payload.get("workflow_type"))
13199 .and_then(Value::as_str)
13200 .filter(|value| !value.is_empty())
13201 .map(str::to_string)
13202 });
13203 if child_events
13204 .iter()
13205 .filter_map(|event| {
13206 event
13207 .payload
13208 .get("child_workflow_type")
13209 .or_else(|| event.payload.get("workflow_type"))
13210 .and_then(Value::as_str)
13211 })
13212 .any(|candidate| Some(candidate) != workflow_type.as_deref())
13213 {
13214 return Err(invalid_recorded_history(
13215 "child_workflow_identity_mismatch",
13216 sequence,
13217 workflow_type
13218 .as_deref()
13219 .unwrap_or("one child workflow type"),
13220 "conflicting child workflow types",
13221 "child workflow lifecycle events at one sequence disagree on type",
13222 ));
13223 }
13224 let mut outcomes = child_workflow_outcomes(
13225 &child_events.iter().map(|event| (*event).clone()).collect::<Vec<_>>(),
13226 fallback_codec,
13227 parent.clone(),
13228 )?;
13229 let terminal_events = child_events
13230 .iter()
13231 .copied()
13232 .filter(|event| event.event_type.starts_with("ChildRun"))
13233 .collect::<Vec<_>>();
13234 let duplicate_delivery = terminal_events.first().is_some_and(|first| {
13235 terminal_events.iter().all(|event| {
13236 event.event_type == first.event_type && event.payload == first.payload
13237 })
13238 });
13239 if outcomes.len() > 1 && !duplicate_delivery {
13240 return Err(invalid_recorded_history(
13241 "duplicate_child_workflow_terminal_event",
13242 sequence,
13243 "at most one terminal child event",
13244 "multiple terminal child events",
13245 "child workflow history settles one command more than once",
13246 ));
13247 }
13248 return Ok(RecordedCommand::ChildWorkflow {
13249 sequence,
13250 workflow_type,
13251 outcome: outcomes.pop(),
13252 parallel_group_path,
13253 });
13254 }
13255
13256 if !signal_wait_events.is_empty() {
13257 let opened: Vec<_> = signal_wait_events
13258 .iter()
13259 .copied()
13260 .filter(|event| event.event_type == "SignalWaitOpened")
13261 .collect();
13262 if opened.len() != 1 {
13263 return Err(invalid_recorded_history(
13264 "signal_wait_open_missing_or_duplicate",
13265 sequence,
13266 "one SignalWaitOpened event",
13267 &format!("{} SignalWaitOpened events", opened.len()),
13268 "signal replay requires exactly one canonical wait-open event",
13269 ));
13270 }
13271
13272 let applied: Vec<_> = signal_wait_events
13273 .iter()
13274 .copied()
13275 .filter(|event| event.event_type == "SignalApplied")
13276 .collect();
13277 if applied.len() > 1 {
13278 return Err(invalid_recorded_history(
13279 "duplicate_signal_wait_apply",
13280 sequence,
13281 "at most one SignalApplied event",
13282 "multiple SignalApplied events",
13283 "signal history applies one durable wait more than once",
13284 ));
13285 }
13286
13287 let signal_names = signal_wait_events
13288 .iter()
13289 .map(|event| required_signal_wait_name(event, sequence))
13290 .collect::<Result<Vec<_>>>()?;
13291 let signal_name = signal_names
13292 .first()
13293 .expect("signal wait events are not empty")
13294 .clone();
13295 if signal_names.iter().any(|candidate| candidate != &signal_name) {
13296 return Err(invalid_recorded_history(
13297 "signal_wait_identity_mismatch",
13298 sequence,
13299 &signal_name,
13300 "conflicting signal names",
13301 "signal wait lifecycle events at one workflow sequence disagree on identity",
13302 ));
13303 }
13304 let value = applied
13305 .first()
13306 .map(|event| decode_signal_event_arguments(event, fallback_codec))
13307 .transpose()?;
13308 return Ok(RecordedCommand::SignalWait {
13309 sequence,
13310 signal_name,
13311 value,
13312 parallel_group_path: recorded_parallel_group_path(
13313 &signal_wait_events,
13314 sequence,
13315 )?,
13316 });
13317 }
13318
13319 if !condition_wait_events.is_empty() {
13320 return recorded_condition_wait(
13321 sequence,
13322 &condition_wait_events,
13323 events,
13324 );
13325 }
13326
13327 if !search_attribute_events.is_empty() {
13328 if search_attribute_events.len() != 1 {
13329 return Err(invalid_recorded_history(
13330 "duplicate_search_attribute_update",
13331 sequence,
13332 "one SearchAttributesUpserted event",
13333 &format!(
13334 "{} SearchAttributesUpserted events",
13335 search_attribute_events.len()
13336 ),
13337 "search-attribute history records one workflow command more than once",
13338 ));
13339 }
13340 let payload = &search_attribute_events[0].payload;
13341 let attributes = payload
13342 .get("attributes")
13343 .filter(|value| value.as_object().is_some_and(|values| !values.is_empty()))
13344 .cloned()
13345 .ok_or_else(|| {
13346 invalid_recorded_history(
13347 "search_attribute_update_missing",
13348 sequence,
13349 "non-empty attributes object",
13350 "missing or invalid attributes",
13351 "search-attribute history is missing its recorded mutation",
13352 )
13353 })?;
13354 let attribute_types =
13355 recorded_search_attribute_types(payload, &attributes, sequence)?;
13356 return Ok(RecordedCommand::SearchAttributes {
13357 sequence,
13358 attributes,
13359 attribute_types,
13360 });
13361 }
13362
13363 if !side_effect_events.is_empty() {
13364 if side_effect_events.len() != 1 {
13365 return Err(invalid_recorded_history(
13366 "duplicate_side_effect_record",
13367 sequence,
13368 "one SideEffectRecorded event",
13369 &format!("{} SideEffectRecorded events", side_effect_events.len()),
13370 "side-effect history records one workflow command more than once",
13371 ));
13372 }
13373 let event = side_effect_events[0];
13374 let result = event.payload.get("result").ok_or_else(|| {
13375 invalid_recorded_history(
13376 "side_effect_result_missing",
13377 sequence,
13378 "recorded result payload",
13379 "missing result",
13380 "side-effect history is missing its recorded value",
13381 )
13382 })?;
13383 let has_published_envelope = result.as_str().is_some()
13384 || result.as_object().is_some_and(|envelope| {
13385 envelope.get("codec").and_then(Value::as_str).is_some()
13386 && envelope.get("blob").and_then(Value::as_str).is_some()
13387 });
13388 if !has_published_envelope {
13389 return Err(invalid_recorded_history(
13390 "side_effect_payload_malformed",
13391 sequence,
13392 "payload blob or {codec, blob} envelope",
13393 &result.to_string(),
13394 "side-effect history result does not use a published payload envelope",
13395 ));
13396 }
13397 let codec = event
13398 .payload
13399 .get("payload_codec")
13400 .and_then(Value::as_str)
13401 .unwrap_or(fallback_codec);
13402 let value = decode_wire_avro_value(result, codec).map_err(|error| {
13403 if error.to_string().contains("unsupported_payload_codec") {
13404 return error;
13405 }
13406
13407 invalid_recorded_history(
13408 "side_effect_payload_incompatible",
13409 sequence,
13410 &format!("valid {codec} payload envelope"),
13411 &error.to_string(),
13412 "side-effect history payload cannot be decoded with its recorded codec",
13413 )
13414 })?;
13415 return Ok(RecordedCommand::SideEffect { sequence, value });
13416 }
13417
13418 if !version_marker_events.is_empty() {
13419 if version_marker_events.len() != 1 {
13420 return Err(invalid_recorded_history(
13421 "duplicate_version_marker_record",
13422 sequence,
13423 "one VersionMarkerRecorded event",
13424 &format!("{} VersionMarkerRecorded events", version_marker_events.len()),
13425 "version-marker history records one workflow command more than once",
13426 ));
13427 }
13428 let payload = &version_marker_events[0].payload;
13429 let change_id = payload
13430 .get("change_id")
13431 .and_then(Value::as_str)
13432 .filter(|value| !value.is_empty())
13433 .map(str::to_string)
13434 .ok_or_else(|| {
13435 invalid_recorded_history(
13436 "version_marker_field_missing",
13437 sequence,
13438 "non-empty change_id",
13439 "missing or invalid change_id",
13440 "version-marker history is missing its stable change ID",
13441 )
13442 })?;
13443 let version = required_version_i32(payload, "version", sequence)?;
13444 let min_supported = required_version_i32(payload, "min_supported", sequence)?;
13445 let max_supported = required_version_i32(payload, "max_supported", sequence)?;
13446 if min_supported > max_supported || version < min_supported || version > max_supported {
13447 return Err(invalid_recorded_history(
13448 "version_marker_history_range_invalid",
13449 sequence,
13450 "min_supported <= version <= max_supported",
13451 &format!("{min_supported} <= {version} <= {max_supported}"),
13452 "recorded version marker contains an internally incompatible range",
13453 ));
13454 }
13455 return Ok(RecordedCommand::VersionMarker {
13456 sequence,
13457 change_id,
13458 version,
13459 });
13460 }
13461
13462 if !memo_events.is_empty() {
13463 if memo_events.len() != 1 {
13464 return Err(invalid_recorded_history(
13465 "duplicate_memo_upsert_record",
13466 sequence,
13467 "one MemoUpserted event",
13468 &format!("{} MemoUpserted events", memo_events.len()),
13469 "memo history records one workflow update more than once",
13470 ));
13471 }
13472 let payload = &memo_events[0].payload;
13473 let entries = payload.get("entries").cloned().ok_or_else(|| {
13474 invalid_recorded_history(
13475 "memo_entries_missing",
13476 sequence,
13477 "memo entries object",
13478 "missing entries",
13479 "MemoUpserted history is missing replay identity entries",
13480 )
13481 })?;
13482 let entries = decode_memo_history_map(&entries, true).map_err(|error| {
13483 invalid_recorded_history(
13484 "memo_entries_invalid",
13485 sequence,
13486 "valid canonical memo entries",
13487 &error.to_string(),
13488 "MemoUpserted history contains invalid replay identity entries",
13489 )
13490 })?;
13491 let merged = payload.get("merged").cloned().ok_or_else(|| {
13492 invalid_recorded_history(
13493 "memo_merged_projection_missing",
13494 sequence,
13495 "merged memo projection",
13496 "missing merged",
13497 "MemoUpserted history is missing its merged projection",
13498 )
13499 })?;
13500 decode_memo_history_map(&merged, false).map_err(|error| {
13501 invalid_recorded_history(
13502 "memo_merged_projection_invalid",
13503 sequence,
13504 "valid merged memo projection",
13505 &error.to_string(),
13506 "MemoUpserted history contains an invalid merged projection",
13507 )
13508 })?;
13509
13510 return Ok(RecordedCommand::Memo { sequence, entries });
13511 }
13512 let scheduled: Vec<_> = timer_events
13513 .iter()
13514 .copied()
13515 .filter(|event| event.event_type == "TimerScheduled")
13516 .collect();
13517 let fired: Vec<_> = timer_events
13518 .iter()
13519 .copied()
13520 .filter(|event| event.event_type == "TimerFired")
13521 .collect();
13522 if scheduled.len() != 1 {
13523 return Err(invalid_recorded_history(
13524 "timer_schedule_missing_or_duplicate",
13525 sequence,
13526 "one TimerScheduled event",
13527 &format!("{} TimerScheduled events", scheduled.len()),
13528 "timer replay requires exactly one recorded schedule event",
13529 ));
13530 }
13531 if fired.len() > 1 {
13532 return Err(invalid_recorded_history(
13533 "duplicate_timer_fire",
13534 sequence,
13535 "at most one TimerFired event",
13536 "multiple TimerFired events",
13537 "timer history contains more than one fire event for a workflow sequence",
13538 ));
13539 }
13540
13541 let scheduled = scheduled[0];
13542 let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
13543 let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
13544 if let Some(fired) = fired.first() {
13545 let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
13546 if fired_timer_id != timer_id {
13547 return Err(invalid_recorded_history(
13548 "timer_identity_mismatch",
13549 sequence,
13550 &timer_id,
13551 &fired_timer_id,
13552 "TimerFired does not correspond to the recorded TimerScheduled event",
13553 ));
13554 }
13555 let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
13556 if fired_delay != delay_seconds {
13557 return Err(invalid_recorded_history(
13558 "timer_history_delay_mismatch",
13559 sequence,
13560 &delay_seconds.to_string(),
13561 &fired_delay.to_string(),
13562 "TimerScheduled and TimerFired record different delays",
13563 ));
13564 }
13565 }
13566
13567 Ok(RecordedCommand::Timer {
13568 sequence,
13569 delay_seconds,
13570 fired: !fired.is_empty(),
13571 parallel_group_path: recorded_parallel_group_path(&timer_events, sequence)?,
13572 })
13573 })
13574 .collect::<Result<_>>()?;
13575
13576 let mut marker_sequences = HashMap::new();
13577 for command in &commands {
13578 if let RecordedCommand::VersionMarker {
13579 sequence,
13580 change_id,
13581 ..
13582 } = command
13583 {
13584 if let Some(first_sequence) = marker_sequences.insert(change_id.clone(), *sequence) {
13585 return Err(invalid_recorded_history(
13586 "duplicate_version_marker",
13587 *sequence,
13588 &format!("one marker for change ID {change_id:?}"),
13589 &format!("markers at sequences {first_sequence} and {sequence}"),
13590 "workflow history contains duplicate markers for one stable change ID",
13591 ));
13592 }
13593 }
13594 }
13595
13596 Ok(commands)
13597}
13598
13599fn required_version_i32(payload: &Value, field: &str, sequence: u64) -> Result<i32> {
13600 payload
13601 .get(field)
13602 .and_then(Value::as_i64)
13603 .and_then(|value| i32::try_from(value).ok())
13604 .ok_or_else(|| {
13605 invalid_recorded_history(
13606 "version_marker_field_missing",
13607 sequence,
13608 &format!("integer {field}"),
13609 "missing or out-of-range integer",
13610 "version-marker history is missing a required integer field",
13611 )
13612 })
13613}
13614
13615fn durable_event_sequence(event: &HistoryEvent) -> Option<u64> {
13616 event
13617 .payload
13618 .get("sequence")
13619 .or_else(|| event.payload.get("workflow_sequence"))
13620 .or_else(|| event.raw.get("sequence"))
13621 .or_else(|| event.raw.get("workflow_sequence"))
13622 .and_then(value_as_u64)
13623}
13624
13625fn is_internal_timer_event(event: &HistoryEvent) -> bool {
13626 matches!(
13627 event
13628 .payload
13629 .get("timer_kind")
13630 .or_else(|| event.raw.get("timer_kind"))
13631 .and_then(Value::as_str),
13632 Some("condition_timeout" | "signal_timeout")
13633 )
13634}
13635
13636fn is_recorded_condition_wait_event(event: &HistoryEvent) -> bool {
13637 matches!(
13638 event.event_type.as_str(),
13639 "ConditionWaitOpened" | "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
13640 )
13641}
13642
13643fn recorded_condition_wait(
13644 sequence: u64,
13645 condition_events: &[&HistoryEvent],
13646 all_events: &[HistoryEvent],
13647) -> Result<RecordedCommand> {
13648 let opened = condition_events
13649 .iter()
13650 .copied()
13651 .filter(|event| event.event_type == "ConditionWaitOpened")
13652 .collect::<Vec<_>>();
13653 if opened.len() != 1 {
13654 return Err(invalid_recorded_history(
13655 "condition_wait_open_missing_or_duplicate",
13656 sequence,
13657 "one ConditionWaitOpened event",
13658 &format!("{} ConditionWaitOpened events", opened.len()),
13659 "condition replay requires exactly one canonical wait-open event",
13660 ));
13661 }
13662 let terminal = condition_events
13663 .iter()
13664 .copied()
13665 .filter(|event| {
13666 matches!(
13667 event.event_type.as_str(),
13668 "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
13669 )
13670 })
13671 .collect::<Vec<_>>();
13672 if terminal.len() > 1 {
13673 return Err(invalid_recorded_history(
13674 "duplicate_condition_wait_terminal_event",
13675 sequence,
13676 "at most one condition terminal event",
13677 "multiple condition terminal events",
13678 "condition history settles one durable wait more than once",
13679 ));
13680 }
13681
13682 let opened = opened[0];
13683 let condition_wait_id = required_condition_wait_id(opened, sequence)?;
13684 let occurrence_id = required_condition_wait_occurrence_id(opened, sequence)?;
13685 for event in condition_events
13686 .iter()
13687 .copied()
13688 .filter(|event| !std::ptr::eq(*event, opened))
13689 {
13690 let event_wait_id = required_condition_wait_id(event, sequence)?;
13691 if event_wait_id != condition_wait_id {
13692 return Err(invalid_recorded_history(
13693 "condition_wait_id_mismatch",
13694 sequence,
13695 &condition_wait_id,
13696 &event_wait_id,
13697 "condition lifecycle events at one sequence disagree on wait identity",
13698 ));
13699 }
13700 let event_occurrence_id = required_condition_wait_occurrence_id(event, sequence)?;
13701 if event_occurrence_id != occurrence_id {
13702 return Err(invalid_recorded_history(
13703 "condition_wait_occurrence_history_mismatch",
13704 sequence,
13705 &occurrence_id,
13706 &event_occurrence_id,
13707 "condition lifecycle events at one sequence disagree on authored occurrence identity",
13708 ));
13709 }
13710 }
13711
13712 let condition_key = optional_non_empty_history_string(opened, "condition_key");
13713 let predicate_identity = opened
13714 .payload
13715 .get("condition_definition_fingerprint")
13716 .and_then(Value::as_str)
13717 .filter(|value| !value.is_empty())
13718 .map(str::to_string)
13719 .ok_or_else(|| {
13720 invalid_recorded_history(
13721 "condition_wait_predicate_fingerprint_missing",
13722 sequence,
13723 "non-empty condition_definition_fingerprint",
13724 &opened.event_type,
13725 "canonical condition history is missing its predicate identity",
13726 )
13727 })?;
13728 let timeout_seconds = optional_history_u64(opened, "timeout_seconds", sequence)?;
13729 for event in condition_events
13730 .iter()
13731 .copied()
13732 .filter(|event| !std::ptr::eq(*event, opened))
13733 {
13734 for (field, opened_value) in [
13735 ("condition_key", condition_key.as_deref()),
13736 (
13737 "condition_definition_fingerprint",
13738 Some(predicate_identity.as_str()),
13739 ),
13740 ] {
13741 if let Some(value) = optional_non_empty_history_string(event, field) {
13742 if opened_value.is_some_and(|opened_value| opened_value != value) {
13743 return Err(invalid_recorded_history(
13744 "condition_wait_definition_history_mismatch",
13745 sequence,
13746 opened_value.unwrap_or_default(),
13747 &value,
13748 "condition lifecycle events disagree on the recorded definition",
13749 ));
13750 }
13751 }
13752 }
13753 if let Some(event_timeout) = optional_history_u64(event, "timeout_seconds", sequence)? {
13754 if timeout_seconds.is_some_and(|opened_timeout| opened_timeout != event_timeout) {
13755 return Err(invalid_recorded_history(
13756 "condition_wait_definition_history_mismatch",
13757 sequence,
13758 &format!("{}s", timeout_seconds.unwrap_or_default()),
13759 &format!("{event_timeout}s"),
13760 "condition lifecycle events disagree on the recorded timeout",
13761 ));
13762 }
13763 }
13764 }
13765
13766 let timeout_timer_events = all_events
13767 .iter()
13768 .filter(|event| {
13769 matches!(
13770 event.event_type.as_str(),
13771 "TimerScheduled" | "TimerCancelled" | "TimerFired"
13772 ) && event.payload.get("timer_kind").and_then(Value::as_str)
13773 == Some("condition_timeout")
13774 && event
13775 .payload
13776 .get("condition_wait_id")
13777 .and_then(Value::as_str)
13778 == Some(condition_wait_id.as_str())
13779 })
13780 .collect::<Vec<_>>();
13781 let scheduled = timeout_timer_events
13782 .iter()
13783 .copied()
13784 .filter(|event| event.event_type == "TimerScheduled")
13785 .collect::<Vec<_>>();
13786 let fired = timeout_timer_events
13787 .iter()
13788 .copied()
13789 .filter(|event| event.event_type == "TimerFired")
13790 .collect::<Vec<_>>();
13791 if scheduled.len() > 1 || fired.len() > 1 || (!fired.is_empty() && scheduled.len() != 1) {
13792 return Err(invalid_recorded_history(
13793 "condition_wait_timeout_history_invalid",
13794 sequence,
13795 "one timeout schedule and at most one fire",
13796 &format!("{} schedules and {} fires", scheduled.len(), fired.len()),
13797 "condition timeout history has a missing or duplicate lifecycle event",
13798 ));
13799 }
13800 if let Some(scheduled) = scheduled.first() {
13801 let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
13802 let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
13803 if timeout_seconds.is_some_and(|timeout| timeout != delay_seconds) {
13804 return Err(invalid_recorded_history(
13805 "condition_wait_timeout_delay_mismatch",
13806 sequence,
13807 &format!("{}s", timeout_seconds.unwrap_or_default()),
13808 &format!("{delay_seconds}s"),
13809 "condition timeout timer differs from the wait definition",
13810 ));
13811 }
13812 if let Some(fired) = fired.first() {
13813 let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
13814 let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
13815 if fired_timer_id != timer_id || fired_delay != delay_seconds {
13816 return Err(invalid_recorded_history(
13817 "condition_wait_timeout_identity_mismatch",
13818 sequence,
13819 &format!("{timer_id}:{delay_seconds}s"),
13820 &format!("{fired_timer_id}:{fired_delay}s"),
13821 "condition timeout fire does not match its durable schedule",
13822 ));
13823 }
13824 }
13825 }
13826
13827 let result = terminal.first().map(|event| {
13828 if event.event_type == "ConditionWaitTimedOut" {
13829 ConditionWaitResult::TimedOut
13830 } else {
13831 ConditionWaitResult::Satisfied
13832 }
13833 });
13834 let result = if !fired.is_empty() {
13835 if result == Some(ConditionWaitResult::Satisfied) {
13836 return Err(invalid_recorded_history(
13837 "condition_wait_terminal_conflict",
13838 sequence,
13839 "one satisfied or timed-out outcome",
13840 "satisfied event and fired timeout",
13841 "condition history records conflicting terminal outcomes",
13842 ));
13843 }
13844 Some(ConditionWaitResult::TimedOut)
13845 } else {
13846 result
13847 };
13848
13849 Ok(RecordedCommand::ConditionWait {
13850 sequence,
13851 occurrence_id,
13852 condition_key,
13853 predicate_identity,
13854 timeout_seconds,
13855 result,
13856 parallel_group_path: recorded_parallel_group_path(condition_events, sequence)?,
13857 })
13858}
13859
13860fn required_condition_wait_occurrence_id(event: &HistoryEvent, sequence: u64) -> Result<String> {
13861 event
13862 .payload
13863 .get("condition_wait_occurrence_id")
13864 .and_then(Value::as_str)
13865 .filter(|value| !value.is_empty())
13866 .map(str::to_string)
13867 .ok_or_else(|| {
13868 invalid_recorded_history(
13869 "condition_wait_occurrence_id_missing",
13870 sequence,
13871 "non-empty condition_wait_occurrence_id",
13872 &event.event_type,
13873 "condition history is missing authored occurrence identity",
13874 )
13875 })
13876}
13877
13878fn required_condition_wait_id(event: &HistoryEvent, sequence: u64) -> Result<String> {
13879 event
13880 .payload
13881 .get("condition_wait_id")
13882 .and_then(Value::as_str)
13883 .filter(|value| !value.is_empty())
13884 .map(str::to_string)
13885 .ok_or_else(|| {
13886 invalid_recorded_history(
13887 "condition_wait_id_missing",
13888 sequence,
13889 "non-empty condition_wait_id",
13890 &event.event_type,
13891 "canonical condition history is missing its durable wait identity",
13892 )
13893 })
13894}
13895
13896fn optional_non_empty_history_string(event: &HistoryEvent, field: &str) -> Option<String> {
13897 event
13898 .payload
13899 .get(field)
13900 .and_then(Value::as_str)
13901 .filter(|value| !value.is_empty())
13902 .map(str::to_string)
13903}
13904
13905fn optional_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<Option<u64>> {
13906 match event.payload.get(field) {
13907 None | Some(Value::Null) => Ok(None),
13908 Some(value) => value_as_u64(value).map(Some).ok_or_else(|| {
13909 invalid_recorded_history(
13910 "condition_wait_definition_invalid",
13911 sequence,
13912 &format!("non-negative integer {field}"),
13913 &value.to_string(),
13914 "condition history contains an invalid numeric definition field",
13915 )
13916 }),
13917 }
13918}
13919
13920fn required_signal_wait_name(event: &HistoryEvent, sequence: u64) -> Result<String> {
13921 event
13922 .payload
13923 .get("signal_name")
13924 .or_else(|| event.raw.get("signal_name"))
13925 .and_then(Value::as_str)
13926 .filter(|value| !value.is_empty())
13927 .map(str::to_string)
13928 .ok_or_else(|| {
13929 invalid_recorded_history(
13930 "signal_wait_name_missing",
13931 sequence,
13932 "non-empty signal_name",
13933 &event.event_type,
13934 "canonical signal-wait history is missing its signal identity",
13935 )
13936 })
13937}
13938
13939fn is_recorded_signal_wait_event(event: &HistoryEvent) -> bool {
13940 matches!(
13941 event.event_type.as_str(),
13942 "SignalWaitOpened" | "SignalApplied"
13943 )
13944}
13945
13946fn required_history_string(event: &HistoryEvent, field: &str, sequence: u64) -> Result<String> {
13947 event
13948 .payload
13949 .get(field)
13950 .and_then(Value::as_str)
13951 .filter(|value| !value.is_empty())
13952 .map(str::to_string)
13953 .ok_or_else(|| {
13954 invalid_recorded_history(
13955 "timer_history_field_missing",
13956 sequence,
13957 field,
13958 &event.event_type,
13959 "timer history is missing a required identity field",
13960 )
13961 })
13962}
13963
13964fn required_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<u64> {
13965 event
13966 .payload
13967 .get(field)
13968 .and_then(value_as_u64)
13969 .ok_or_else(|| {
13970 invalid_recorded_history(
13971 "timer_history_field_missing",
13972 sequence,
13973 field,
13974 &event.event_type,
13975 "timer history is missing a required numeric field",
13976 )
13977 })
13978}
13979
13980fn recorded_search_attribute_types(
13981 payload: &Value,
13982 attributes: &Value,
13983 sequence: u64,
13984) -> Result<RecordedSnapshotValue<BTreeMap<String, String>>> {
13985 let Some(raw_types) = payload.get("attribute_types") else {
13986 return Ok(RecordedSnapshotValue::Unknown);
13990 };
13991 let Some(raw_types) = raw_types.as_object() else {
13992 return Err(invalid_recorded_history(
13993 "search_attribute_types_malformed",
13994 sequence,
13995 "canonical attribute type map",
13996 &raw_types.to_string(),
13997 "search-attribute history contains malformed type identity",
13998 ));
13999 };
14000 let attribute_keys = attributes
14001 .as_object()
14002 .expect("recorded search attributes were validated as an object");
14003 let mut types = BTreeMap::new();
14004 for (key, value) in raw_types {
14005 let Some(attribute_type) = value.as_str() else {
14006 return Err(invalid_recorded_history(
14007 "search_attribute_types_malformed",
14008 sequence,
14009 "canonical string type name",
14010 &value.to_string(),
14011 "search-attribute history contains a non-string type identity",
14012 ));
14013 };
14014 if !attribute_keys.contains_key(key)
14015 || !matches!(
14016 attribute_type,
14017 "string" | "keyword" | "keyword_list" | "int" | "float" | "bool" | "datetime"
14018 )
14019 {
14020 return Err(invalid_recorded_history(
14021 "search_attribute_types_malformed",
14022 sequence,
14023 "canonical types for keys present in attributes",
14024 &format!("{key}:{attribute_type}"),
14025 "search-attribute history contains unsupported or orphaned type identity",
14026 ));
14027 }
14028 types.insert(key.clone(), attribute_type.to_string());
14029 }
14030 Ok(RecordedSnapshotValue::Known(types))
14031}
14032
14033fn invalid_recorded_history(
14034 reason: &str,
14035 sequence: u64,
14036 expected: &str,
14037 actual: &str,
14038 message: &str,
14039) -> Error {
14040 Error::NonDeterministicReplay(ReplayFailure::new(
14041 reason,
14042 Some(sequence),
14043 Some(expected.to_string()),
14044 Some(actual.to_string()),
14045 message,
14046 ))
14047}
14048
14049type ActivityOutcome = std::result::Result<AvroValue, ActivityFailure>;
14050
14051fn activity_outcome(
14052 event: &HistoryEvent,
14053 fallback_codec: &str,
14054 recorded_activity_type: Option<String>,
14055) -> Result<ActivityOutcome> {
14056 if event.event_type == "ActivityCompleted" {
14057 let codec = event
14058 .payload
14059 .get("payload_codec")
14060 .and_then(Value::as_str)
14061 .unwrap_or(fallback_codec);
14062 return Ok(Ok(decode_wire_avro_value(
14063 event.payload.get("result").unwrap_or(&Value::Null),
14064 codec,
14065 )?));
14066 }
14067
14068 let payload = &event.payload;
14069 let (kind, fallback_reason, fallback_message) = match event.event_type.as_str() {
14070 "ActivityFailed" => (ActivityFailureKind::Failed, "activity", "activity failed"),
14071 "ActivityCancelled" => (
14072 ActivityFailureKind::Cancelled,
14073 "cancelled",
14074 "activity was cancelled",
14075 ),
14076 "ActivityTimedOut" => (
14077 ActivityFailureKind::TimedOut,
14078 "timeout",
14079 "activity timed out",
14080 ),
14081 _ => unreachable!("activity_outcome is called only for terminal activity events"),
14082 };
14083 let exception = payload
14084 .get("exception")
14085 .filter(|value| !value.is_null())
14086 .cloned();
14087 let failure_category = payload_string(payload, "failure_category");
14088 let timeout_kind = payload_string(payload, "timeout_kind");
14089 let reason = payload_string(payload, "reason").unwrap_or_else(|| match kind {
14090 ActivityFailureKind::Failed => failure_category
14091 .clone()
14092 .unwrap_or_else(|| fallback_reason.to_string()),
14093 ActivityFailureKind::Cancelled => fallback_reason.to_string(),
14094 ActivityFailureKind::TimedOut => timeout_kind
14095 .clone()
14096 .unwrap_or_else(|| fallback_reason.to_string()),
14097 });
14098 let message = payload_string(payload, "message")
14099 .or_else(|| {
14100 exception
14101 .as_ref()
14102 .and_then(|value| payload_string(value, "message"))
14103 })
14104 .unwrap_or_else(|| fallback_message.to_string());
14105
14106 Ok(Err(ActivityFailure {
14107 kind,
14108 reason,
14109 message,
14110 activity_execution_id: payload_string(payload, "activity_execution_id"),
14111 activity_attempt_id: payload_string(payload, "activity_attempt_id"),
14112 activity_type: payload_string(payload, "activity_type")
14113 .or_else(|| payload_string(payload, "activity_name"))
14114 .or(recorded_activity_type),
14115 activity_class: payload_string(payload, "activity_class"),
14116 attempt_number: payload.get("attempt_number").and_then(value_as_u64),
14117 failure_id: payload_string(payload, "failure_id"),
14118 failure_category,
14119 timeout_kind,
14120 non_retryable: payload
14121 .get("non_retryable")
14122 .and_then(Value::as_bool)
14123 .unwrap_or(false),
14124 exception_type: payload_string(payload, "exception_type").or_else(|| {
14125 exception
14126 .as_ref()
14127 .and_then(|value| payload_string(value, "type"))
14128 }),
14129 exception_class: payload_string(payload, "exception_class").or_else(|| {
14130 exception
14131 .as_ref()
14132 .and_then(|value| payload_string(value, "class"))
14133 }),
14134 code: payload
14135 .get("code")
14136 .filter(|value| !value.is_null())
14137 .cloned(),
14138 exception,
14139 }))
14140}
14141
14142type ChildWorkflowOutcome = std::result::Result<ChildWorkflowAvroResult, ChildWorkflowFailure>;
14143
14144fn child_workflow_outcomes(
14145 events: &[HistoryEvent],
14146 fallback_codec: &str,
14147 parent: WorkflowIdentity,
14148) -> Result<Vec<ChildWorkflowOutcome>> {
14149 let mut outcomes = Vec::new();
14150
14151 for event in events {
14152 let kind = match event.event_type.as_str() {
14153 "ChildRunCompleted" => None,
14154 "ChildRunFailed" => Some((
14155 ChildWorkflowFailureKind::Failed,
14156 "child_workflow",
14157 "child workflow failed",
14158 )),
14159 "ChildRunCancelled" => Some((
14160 ChildWorkflowFailureKind::Cancelled,
14161 "cancelled",
14162 "child workflow was cancelled",
14163 )),
14164 "ChildRunTerminated" => Some((
14165 ChildWorkflowFailureKind::Terminated,
14166 "terminated",
14167 "child workflow was terminated",
14168 )),
14169 _ => continue,
14170 };
14171 let payload = &event.payload;
14172 let child_workflow_id = payload_string(payload, "child_workflow_instance_id");
14173 let child_workflow_run_id = payload_string(payload, "child_workflow_run_id");
14174 let child_workflow_type = payload_string(payload, "child_workflow_type");
14175
14176 if let Some((kind, reason, fallback_message)) = kind {
14177 let exception = payload
14178 .get("exception")
14179 .filter(|value| !value.is_null())
14180 .cloned();
14181 let message = payload_string(payload, "message")
14182 .or_else(|| {
14183 exception
14184 .as_ref()
14185 .and_then(|value| payload_string(value, "message"))
14186 })
14187 .unwrap_or_else(|| fallback_message.to_string());
14188 let exception_type = payload_string(payload, "exception_type").or_else(|| {
14189 exception
14190 .as_ref()
14191 .and_then(|value| payload_string(value, "type"))
14192 });
14193 let exception_class = payload_string(payload, "exception_class").or_else(|| {
14194 exception
14195 .as_ref()
14196 .and_then(|value| payload_string(value, "class"))
14197 });
14198 outcomes.push(Err(ChildWorkflowFailure {
14199 kind,
14200 reason: reason.to_string(),
14201 message,
14202 parent_workflow_id: parent.workflow_id.clone(),
14203 parent_workflow_run_id: parent.run_id.clone(),
14204 child_workflow_id,
14205 child_workflow_run_id,
14206 child_workflow_type,
14207 failure_id: payload_string(payload, "failure_id"),
14208 failure_category: payload_string(payload, "failure_category"),
14209 exception_type,
14210 exception_class,
14211 non_retryable: payload
14212 .get("non_retryable")
14213 .and_then(Value::as_bool)
14214 .unwrap_or(false),
14215 code: payload
14216 .get("code")
14217 .filter(|value| !value.is_null())
14218 .cloned(),
14219 exception,
14220 }));
14221 continue;
14222 }
14223
14224 let codec = payload
14225 .get("payload_codec")
14226 .and_then(Value::as_str)
14227 .unwrap_or(fallback_codec);
14228 let result = payload
14229 .get("result")
14230 .or_else(|| payload.get("output"))
14231 .unwrap_or(&Value::Null);
14232 outcomes.push(Ok(ChildWorkflowAvroResult {
14233 parent: parent.clone(),
14234 child: WorkflowIdentity {
14235 workflow_id: child_workflow_id,
14236 run_id: child_workflow_run_id,
14237 },
14238 child_workflow_type,
14239 result: decode_wire_avro_value(result, codec)?,
14240 }));
14241 }
14242
14243 Ok(outcomes)
14244}
14245
14246fn payload_string(payload: &Value, key: &str) -> Option<String> {
14247 payload
14248 .get(key)
14249 .and_then(Value::as_str)
14250 .filter(|value| !value.is_empty())
14251 .map(str::to_string)
14252}
14253
14254fn workflow_failure_command(error: &Error) -> Value {
14255 let (exception_type, exception_class, properties) = match error {
14256 Error::ActivityFailed(failure) => (
14257 match failure.kind {
14258 ActivityFailureKind::Failed => "ActivityFailed",
14259 ActivityFailureKind::Cancelled => "ActivityCancelled",
14260 ActivityFailureKind::TimedOut => "ActivityTimedOut",
14261 },
14262 "durable_workflow::ActivityFailure",
14263 json!({
14264 "reason": failure.reason,
14265 "activity_execution_id": failure.activity_execution_id,
14266 "activity_attempt_id": failure.activity_attempt_id,
14267 "activity_type": failure.activity_type,
14268 "activity_class": failure.activity_class,
14269 "attempt_number": failure.attempt_number,
14270 "failure_id": failure.failure_id,
14271 "failure_category": failure.failure_category,
14272 "timeout_kind": failure.timeout_kind,
14273 "activity_non_retryable": failure.non_retryable,
14274 "activity_exception_type": failure.exception_type,
14275 "activity_exception_class": failure.exception_class,
14276 "activity_code": failure.code,
14277 "activity_exception": failure.exception,
14278 }),
14279 ),
14280 Error::ChildWorkflowFailed(failure) => (
14281 match failure.kind {
14282 ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
14283 ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
14284 ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
14285 },
14286 "durable_workflow::ChildWorkflowFailure",
14287 json!({
14288 "reason": failure.reason,
14289 "parent_workflow_id": failure.parent_workflow_id,
14290 "parent_workflow_run_id": failure.parent_workflow_run_id,
14291 "child_workflow_id": failure.child_workflow_id,
14292 "child_workflow_run_id": failure.child_workflow_run_id,
14293 "child_workflow_type": failure.child_workflow_type,
14294 "failure_id": failure.failure_id,
14295 "failure_category": failure.failure_category,
14296 "child_exception_type": failure.exception_type,
14297 "child_exception_class": failure.exception_class,
14298 "child_non_retryable": failure.non_retryable,
14299 "child_code": failure.code,
14300 "child_exception": failure.exception,
14301 }),
14302 ),
14303 Error::ParallelFailed(failure) => (
14304 "ParallelFailed",
14305 "durable_workflow::ParallelFailure",
14306 json!({
14307 "parallel_group_id": failure.group_id,
14308 "parallel_member_path": failure.member_path,
14309 "parallel_group_path": failure.group_path,
14310 "completed_members": failure.completed.iter().map(|completion| &completion.member_path).collect::<Vec<_>>(),
14311 "cause_type": workflow_error_type(&failure.cause),
14312 "cause_message": failure.cause.to_string(),
14313 }),
14314 ),
14315 Error::SagaCompensationFailed(failure) => (
14316 "SagaCompensationFailed",
14317 "durable_workflow::SagaCompensationFailure",
14318 json!({
14319 "initiating_failure_type": workflow_error_type(&failure.initiating_failure),
14320 "initiating_failure_message": failure.initiating_failure.to_string(),
14321 "compensation_activity_type": failure.compensation_activity_type,
14322 "compensation_registration_order": failure.compensation_registration_order,
14323 "compensation_failure_type": workflow_error_type(&failure.compensation_failure),
14324 "compensation_failure_message": failure.compensation_failure.to_string(),
14325 }),
14326 ),
14327 Error::WorkflowCancellationRequested(_) => (
14328 "WorkflowCancellationRequested",
14329 "durable_workflow::WorkflowCancellationRequested",
14330 json!({"reason": "cancelled"}),
14331 ),
14332 Error::NonDeterministicReplay(_) => (
14333 "NonDeterministicReplay",
14334 "durable_workflow::Error",
14335 Value::Null,
14336 ),
14337 _ => ("RustWorkflowError", "durable_workflow::Error", Value::Null),
14338 };
14339 let non_retryable = match error {
14340 Error::ActivityFailed(failure) => failure.non_retryable,
14341 Error::ChildWorkflowFailed(failure) => failure.non_retryable,
14342 Error::ParallelFailed(failure) => workflow_error_non_retryable(&failure.cause),
14343 Error::SagaCompensationFailed(failure) => {
14344 workflow_error_non_retryable(&failure.compensation_failure)
14345 }
14346 Error::WorkflowCancellationRequested(_) => true,
14347 Error::NonDeterministicReplay(_) => true,
14348 _ => false,
14349 };
14350
14351 json!({
14352 "type": "fail_workflow",
14353 "message": error.to_string(),
14354 "exception_type": exception_type,
14355 "exception_class": exception_class,
14356 "non_retryable": non_retryable,
14357 "exception": {
14358 "type": exception_type,
14359 "class": exception_class,
14360 "message": error.to_string(),
14361 "properties": properties,
14362 }
14363 })
14364}
14365
14366fn workflow_error_type(error: &Error) -> &'static str {
14367 match error {
14368 Error::ActivityFailed(failure) => match failure.kind {
14369 ActivityFailureKind::Failed => "ActivityFailed",
14370 ActivityFailureKind::Cancelled => "ActivityCancelled",
14371 ActivityFailureKind::TimedOut => "ActivityTimedOut",
14372 },
14373 Error::ChildWorkflowFailed(failure) => match failure.kind {
14374 ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
14375 ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
14376 ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
14377 },
14378 Error::ParallelFailed(_) => "ParallelFailed",
14379 Error::SagaCompensationFailed(_) => "SagaCompensationFailed",
14380 Error::WorkflowCancellationRequested(_) => "WorkflowCancellationRequested",
14381 Error::NonDeterministicReplay(_) => "NonDeterministicReplay",
14382 _ => "RustWorkflowError",
14383 }
14384}
14385
14386fn workflow_error_non_retryable(error: &Error) -> bool {
14387 match error {
14388 Error::ActivityFailed(failure) => failure.non_retryable,
14389 Error::ChildWorkflowFailed(failure) => failure.non_retryable,
14390 Error::ParallelFailed(failure) => workflow_error_non_retryable(&failure.cause),
14391 Error::SagaCompensationFailed(failure) => {
14392 workflow_error_non_retryable(&failure.compensation_failure)
14393 }
14394 Error::WorkflowCancellationRequested(_) | Error::NonDeterministicReplay(_) => true,
14395 _ => false,
14396 }
14397}
14398
14399fn workflow_task_integrity_error(error: &Error) -> bool {
14400 matches!(
14401 error,
14402 Error::NonDeterministicReplay(_)
14403 | Error::Protocol(_)
14404 | Error::MissingWorkflowCommandIdentity
14405 | Error::WorkflowStatePoisoned
14406 )
14407}
14408
14409fn decode_signal_event_arguments(
14410 event: &HistoryEvent,
14411 fallback_codec: &str,
14412) -> Result<Vec<AvroValue>> {
14413 let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
14414 validate_payload_codec(codec)?;
14415 let raw = signal_history_payload(&event.payload);
14416 let decoded = match raw.filter(|value| !value.is_null()) {
14417 Some(value) => decode_wire_avro_value(value, codec)?,
14418 None => AvroValue::Array(Vec::new()),
14419 };
14420 let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
14421 unreachable!("normalize_avro_arguments always returns an array");
14422 };
14423 Ok(arguments)
14424}
14425
14426fn decode_update_event_arguments(
14427 event: &HistoryEvent,
14428 fallback_codec: &str,
14429) -> Result<Vec<AvroValue>> {
14430 let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
14431 validate_payload_codec(codec)?;
14432 let decoded = match event
14433 .payload
14434 .get("arguments")
14435 .filter(|value| !value.is_null())
14436 {
14437 Some(value) => decode_wire_avro_value(value, codec)?,
14438 None => AvroValue::Array(Vec::new()),
14439 };
14440 let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
14441 unreachable!("normalize_avro_arguments always returns an array");
14442 };
14443 Ok(arguments)
14444}
14445
14446fn hydrate_query_history_from_export(task: &mut QueryTask) -> Result<()> {
14447 let Some(export_events) = task
14448 .history_export
14449 .as_ref()
14450 .and_then(|export| export.get("history_events"))
14451 .and_then(Value::as_array)
14452 else {
14453 return Ok(());
14454 };
14455
14456 if export_events.len() > task.history_events.len() {
14457 task.history_events = serde_json::from_value(Value::Array(export_events.clone()))?;
14458 }
14459
14460 Ok(())
14461}
14462
14463fn enrich_query_history_from_export(task: &mut QueryTask) -> Result<()> {
14464 let Some(export) = task.history_export.as_ref() else {
14465 return Ok(());
14466 };
14467 let signals = export
14468 .get("signals")
14469 .and_then(Value::as_array)
14470 .cloned()
14471 .unwrap_or_default();
14472 let activities = export
14473 .get("activities")
14474 .and_then(Value::as_array)
14475 .cloned()
14476 .unwrap_or_default();
14477 let export_codec = export
14478 .get("payloads")
14479 .and_then(|payloads| payloads.get("codec"))
14480 .and_then(Value::as_str)
14481 .unwrap_or(&task.payload_codec)
14482 .to_string();
14483 let mut signal_name_offsets: HashMap<String, usize> = HashMap::new();
14484
14485 for event in &mut task.history_events {
14486 if event.event_type == "ActivityCompleted" {
14487 let sequence = event
14488 .payload
14489 .get("sequence")
14490 .or_else(|| event.payload.get("workflow_sequence"))
14491 .and_then(value_as_u64);
14492 let Some(activity) = sequence.and_then(|sequence| {
14493 activities.iter().find(|activity| {
14494 activity.get("sequence").and_then(value_as_u64) == Some(sequence)
14495 })
14496 }) else {
14497 continue;
14498 };
14499 let Some(payload) = event.payload.as_object_mut() else {
14500 continue;
14501 };
14502 if missing_payload(payload.get("result")) {
14503 if let Some(result) = activity
14504 .get("result")
14505 .filter(|value| !missing_payload(Some(value)))
14506 {
14507 payload.insert("result".to_string(), result.clone());
14508 }
14509 }
14510 for field in ["payload_codec", "activity_type"] {
14511 if payload
14512 .get(field)
14513 .and_then(Value::as_str)
14514 .unwrap_or_default()
14515 .is_empty()
14516 {
14517 if let Some(value) = activity.get(field) {
14518 payload.insert(field.to_string(), value.clone());
14519 }
14520 }
14521 }
14522 continue;
14523 }
14524
14525 if event.event_type != "SignalReceived" && event.event_type != "SignalApplied" {
14526 continue;
14527 }
14528 let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
14529 let command_id = event
14530 .payload
14531 .get("workflow_command_id")
14532 .or_else(|| event.raw.get("workflow_command_id"))
14533 .and_then(Value::as_str);
14534 let signal_name = event
14535 .payload
14536 .get("signal_name")
14537 .and_then(Value::as_str)
14538 .unwrap_or_default()
14539 .to_string();
14540 let matched = signals
14541 .iter()
14542 .find(|signal| {
14543 signal_id.is_some() && signal.get("id").and_then(Value::as_str) == signal_id
14544 })
14545 .or_else(|| {
14546 signals.iter().find(|signal| {
14547 command_id.is_some()
14548 && signal.get("command_id").and_then(Value::as_str) == command_id
14549 })
14550 })
14551 .or_else(|| {
14552 let offset = signal_name_offsets.entry(signal_name.clone()).or_default();
14553 let signal = signals
14554 .iter()
14555 .filter(|signal| {
14556 signal.get("name").and_then(Value::as_str) == Some(signal_name.as_str())
14557 })
14558 .nth(*offset);
14559 if signal.is_some() {
14560 *offset += 1;
14561 }
14562 signal
14563 });
14564 let Some(signal) = matched else {
14565 continue;
14566 };
14567 let signal_codec = signal
14568 .get("payload_codec")
14569 .and_then(Value::as_str)
14570 .unwrap_or(&export_codec);
14571 let Some(payload) = event.payload.as_object_mut() else {
14572 continue;
14573 };
14574 if missing_payload(payload.get("arguments")) {
14575 if let Some(arguments) = signal
14576 .get("arguments")
14577 .filter(|value| !missing_payload(Some(value)))
14578 {
14579 let envelope = match arguments {
14580 Value::String(blob) => json!({"codec": signal_codec, "blob": blob}),
14581 other => other.clone(),
14582 };
14583 payload.insert("arguments".to_string(), envelope);
14584 }
14585 }
14586 if payload
14587 .get("payload_codec")
14588 .and_then(Value::as_str)
14589 .unwrap_or_default()
14590 .is_empty()
14591 {
14592 payload.insert("payload_codec".to_string(), json!(signal_codec));
14593 }
14594 }
14595
14596 Ok(())
14597}
14598
14599fn missing_payload(value: Option<&Value>) -> bool {
14600 match value {
14601 None | Some(Value::Null) => true,
14602 Some(Value::String(value)) => value.is_empty(),
14603 Some(_) => false,
14604 }
14605}
14606
14607fn query_signal_events(task: &QueryTask) -> Result<Vec<QuerySignal>> {
14608 let export_signals = task
14609 .history_export
14610 .as_ref()
14611 .and_then(|export| export.get("signals"))
14612 .and_then(Value::as_array)
14613 .cloned()
14614 .unwrap_or_default();
14615 let export_codec = task
14616 .history_export
14617 .as_ref()
14618 .and_then(|export| export.get("payloads"))
14619 .and_then(|payloads| payloads.get("codec"))
14620 .and_then(Value::as_str)
14621 .unwrap_or(&task.payload_codec);
14622 let mut name_offsets: HashMap<String, usize> = HashMap::new();
14623 let mut signals = Vec::new();
14624
14625 for event in &task.history_events {
14626 if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
14627 continue;
14628 }
14629
14630 let name = event
14631 .payload
14632 .get("signal_name")
14633 .and_then(Value::as_str)
14634 .unwrap_or_default();
14635 if name.is_empty() {
14636 continue;
14637 }
14638 let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
14639 let command_id = event
14640 .payload
14641 .get("workflow_command_id")
14642 .or_else(|| event.raw.get("workflow_command_id"))
14643 .and_then(Value::as_str);
14644 let matched_export = export_signals
14645 .iter()
14646 .find(|candidate| {
14647 signal_id.is_some() && candidate.get("id").and_then(Value::as_str) == signal_id
14648 })
14649 .or_else(|| {
14650 export_signals.iter().find(|candidate| {
14651 command_id.is_some()
14652 && candidate.get("command_id").and_then(Value::as_str) == command_id
14653 })
14654 })
14655 .or_else(|| {
14656 let offset = name_offsets.entry(name.to_string()).or_default();
14657 let candidate = export_signals
14658 .iter()
14659 .filter(|candidate| candidate.get("name").and_then(Value::as_str) == Some(name))
14660 .nth(*offset);
14661 if candidate.is_some() {
14662 *offset += 1;
14663 }
14664 candidate
14665 });
14666 let codec = event
14667 .payload
14668 .get("payload_codec")
14669 .and_then(Value::as_str)
14670 .or_else(|| {
14671 matched_export
14672 .and_then(|signal| signal.get("payload_codec"))
14673 .and_then(Value::as_str)
14674 })
14675 .unwrap_or(export_codec);
14676 let raw_arguments = signal_history_payload(&event.payload)
14677 .filter(|value| !value.is_null())
14678 .or_else(|| matched_export.and_then(|signal| signal.get("arguments")));
14679 let (arguments, avro_arguments) = decode_query_signal_arguments(raw_arguments, codec)?;
14680 let workflow_sequence = event
14681 .payload
14682 .get("workflow_sequence")
14683 .and_then(value_as_u64)
14684 .or_else(|| {
14685 matched_export
14686 .and_then(|signal| signal.get("workflow_sequence"))
14687 .and_then(value_as_u64)
14688 });
14689
14690 signals.push(QuerySignal {
14691 id: signal_id.map(str::to_string).or_else(|| {
14692 matched_export
14693 .and_then(|signal| signal.get("id"))
14694 .and_then(Value::as_str)
14695 .map(str::to_string)
14696 }),
14697 name: name.to_string(),
14698 arguments,
14699 avro_arguments,
14700 workflow_sequence,
14701 });
14702 }
14703
14704 if signals.is_empty() {
14705 for signal in export_signals {
14706 if signal.get("status").and_then(Value::as_str) == Some("rejected") {
14707 continue;
14708 }
14709 let Some(name) = signal.get("name").and_then(Value::as_str) else {
14710 continue;
14711 };
14712 let codec = signal
14713 .get("payload_codec")
14714 .and_then(Value::as_str)
14715 .unwrap_or(export_codec);
14716 let (arguments, avro_arguments) =
14717 decode_query_signal_arguments(signal.get("arguments"), codec)?;
14718 signals.push(QuerySignal {
14719 id: signal.get("id").and_then(Value::as_str).map(str::to_string),
14720 name: name.to_string(),
14721 arguments,
14722 avro_arguments,
14723 workflow_sequence: signal.get("workflow_sequence").and_then(value_as_u64),
14724 });
14725 }
14726 signals.sort_by_key(|signal| signal.workflow_sequence.unwrap_or(u64::MAX));
14727 }
14728
14729 Ok(signals)
14730}
14731
14732fn decode_query_signal_arguments(
14733 raw: Option<&Value>,
14734 codec: &str,
14735) -> Result<(Vec<Value>, Vec<AvroValue>)> {
14736 validate_payload_codec(codec)?;
14737 let decoded = match raw.filter(|value| !value.is_null()) {
14738 Some(value) => decode_wire_avro_value(value, codec)?,
14739 None => AvroValue::Array(Vec::new()),
14740 };
14741 let AvroValue::Array(avro_arguments) = normalize_avro_arguments(decoded) else {
14742 unreachable!("normalize_avro_arguments always returns an array");
14743 };
14744 let arguments = avro_arguments
14745 .iter()
14746 .cloned()
14747 .map(AvroValue::into_json)
14748 .collect::<Result<Vec<_>>>()?;
14749 Ok((arguments, avro_arguments))
14750}
14751
14752fn value_as_u64(value: &Value) -> Option<u64> {
14753 value
14754 .as_u64()
14755 .or_else(|| value.as_str().and_then(|value| value.parse().ok()))
14756}
14757
14758#[cfg(test)]
14759mod tests {
14760 use super::*;
14761 mod runtime_payloads;
14762 mod runtime_uploads;
14763 use std::{
14764 fs,
14765 io::{Read, Write},
14766 net::{SocketAddr, TcpListener, TcpStream},
14767 process::Command as ProcessCommand,
14768 sync::atomic::AtomicUsize,
14769 thread,
14770 };
14771
14772 #[derive(Clone, Copy, Debug)]
14773 enum InvalidTaskPayloadCodec {
14774 Missing,
14775 Null,
14776 NonString,
14777 }
14778
14779 impl InvalidTaskPayloadCodec {
14780 fn label(self) -> &'static str {
14781 match self {
14782 Self::Missing => "missing",
14783 Self::Null => "null",
14784 Self::NonString => "non-string",
14785 }
14786 }
14787
14788 fn apply(self, task: &mut Value) {
14789 let task = task.as_object_mut().expect("task fixture object");
14790 match self {
14791 Self::Missing => {
14792 task.remove("payload_codec");
14793 }
14794 Self::Null => {
14795 task.insert("payload_codec".to_string(), Value::Null);
14796 }
14797 Self::NonString => {
14798 task.insert("payload_codec".to_string(), json!(42));
14799 }
14800 }
14801 }
14802 }
14803
14804 fn fixture_envelope(value: Value) -> Value {
14805 encode_value_envelope(&value, DEFAULT_CODEC).expect("encode Avro test fixture")
14806 }
14807
14808 fn fixture_blob(value: Value) -> String {
14809 encode_payload(&value, DEFAULT_CODEC)
14810 .expect("encode Avro test fixture")
14811 .blob
14812 }
14813
14814 #[test]
14815 fn client_builder_rejects_the_sdk_owned_api_suffix() {
14816 for base_url in [
14817 "http://127.0.0.1:8080/api",
14818 "http://localhost:8080/api/",
14819 "https://runtime.example.test/namespaces/orders/api",
14820 ] {
14821 let error = Client::builder(base_url)
14822 .build()
14823 .expect_err("SDK-owned /api suffix must be rejected during build");
14824
14825 assert!(matches!(error, Error::InvalidBaseUrl), "{base_url}");
14826 assert!(
14827 error.to_string().contains("SDK appends /api automatically"),
14828 "the validation error must explain how to fix the endpoint"
14829 );
14830 }
14831 }
14832
14833 #[test]
14834 fn client_builder_preserves_self_hosted_and_managed_runtime_prefixes() {
14835 for (base_url, expected) in [
14836 ("http://127.0.0.1:8080", "http://127.0.0.1:8080"),
14837 (
14838 "http://localhost:8080/durable-workflow/",
14839 "http://localhost:8080/durable-workflow",
14840 ),
14841 (
14842 "https://runtime.example.test/namespaces/orders",
14843 "https://runtime.example.test/namespaces/orders",
14844 ),
14845 (
14846 "https://runtime.example.test/gateway/api/namespaces/orders",
14847 "https://runtime.example.test/gateway/api/namespaces/orders",
14848 ),
14849 (
14850 "https://api.example.test/runtime/orders/",
14851 "https://api.example.test/runtime/orders",
14852 ),
14853 ] {
14854 let client = Client::builder(base_url)
14855 .build()
14856 .expect("Server and Cloud runtime base URL must remain valid");
14857
14858 assert_eq!(client.base_url, expected);
14859 }
14860 }
14861
14862 #[test]
14863 fn workflow_completion_uses_the_additive_command_protocol_floor() {
14864 assert_eq!(
14865 workflow_completion_protocol_version(&[json!({"type": "complete_workflow"})]),
14866 WORKER_PROTOCOL_VERSION
14867 );
14868 assert_eq!(
14869 workflow_completion_protocol_version(&[json!({
14870 "type": "upsert_search_attributes",
14871 "attributes": {"OrderStatus": "waiting"},
14872 })]),
14873 SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION
14874 );
14875 assert_eq!(
14876 workflow_completion_protocol_version(&[json!({
14877 "type": "upsert_search_attributes",
14878 "attributes": {"OrderStatus": "waiting"},
14879 "attribute_types": {"OrderStatus": "keyword"},
14880 })]),
14881 TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION
14882 );
14883 assert_eq!(
14884 workflow_completion_protocol_version(&[
14885 json!({"type": "upsert_memo", "entries": {"status": "waiting"}}),
14886 json!({"type": "open_condition_wait", "condition_key": "ready"}),
14887 ]),
14888 MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION
14889 );
14890 assert_eq!(
14891 workflow_completion_protocol_version(&[
14892 json!({"type": "upsert_search_attributes", "attributes": {"State": "waiting"}}),
14893 json!({"type": "open_condition_wait", "condition_key": "ready"}),
14894 ]),
14895 CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION
14896 );
14897 assert_eq!(
14898 workflow_completion_protocol_version(&[json!({
14899 "type": "open_condition_wait",
14900 "condition_wait_occurrence_id": "rust:condition-wait:0",
14901 "condition_key": "ready",
14902 })]),
14903 CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
14904 );
14905 assert_eq!(
14906 workflow_completion_protocol_version_with_message_streams(
14907 &[json!({"type": "upsert_memo", "entries": {"status": "waiting"}})],
14908 true,
14909 ),
14910 MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION
14911 );
14912 assert_eq!(
14913 workflow_completion_protocol_version_with_message_streams(
14914 &[json!({
14915 "type": "open_condition_wait",
14916 "condition_wait_occurrence_id": "rust:condition-wait:0",
14917 "condition_key": "ready",
14918 })],
14919 true,
14920 ),
14921 CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
14922 );
14923 }
14924
14925 #[test]
14926 fn portable_worker_affinity_manifest_explicitly_refuses_unimplemented_features() {
14927 let manifest = portable_worker_affinity_capability_manifest();
14928
14929 for capability in ["local_activities", "worker_sessions", "sticky_execution"] {
14930 assert_eq!(manifest[capability]["supported"], json!(false));
14931 assert_eq!(
14932 manifest[capability]["minimum_protocol_version"],
14933 json!(PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION)
14934 );
14935 assert!(manifest[capability]["reason"]
14936 .as_str()
14937 .is_some_and(|reason| !reason.is_empty()));
14938 }
14939 }
14940
14941 fn typed_fidelity_probe() -> AvroValue {
14942 AvroValue::Map(BTreeMap::from([
14943 ("bytes".to_string(), AvroValue::Bytes(vec![0, 0xff])),
14944 ("empty".to_string(), AvroValue::Map(BTreeMap::new())),
14945 (
14946 "numeric".to_string(),
14947 AvroValue::Map(BTreeMap::from([
14948 ("0".to_string(), AvroValue::String("zero".to_string())),
14949 ("1".to_string(), AvroValue::String("one".to_string())),
14950 ])),
14951 ),
14952 (
14953 "nested".to_string(),
14954 AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([(
14955 "enabled".to_string(),
14956 AvroValue::Boolean(true),
14957 )]))]),
14958 ),
14959 (
14960 "projection_collisions".to_string(),
14961 AvroValue::Array(projection_collision_probe()),
14962 ),
14963 ]))
14964 }
14965
14966 fn projection_collision_probe() -> Vec<AvroValue> {
14967 vec![
14968 AvroValue::Map(BTreeMap::from([
14969 ("$type".to_string(), AvroValue::String("bytes".to_string())),
14970 (
14971 "base64".to_string(),
14972 AvroValue::String("ordinary user text".to_string()),
14973 ),
14974 ])),
14975 AvroValue::Map(BTreeMap::from([
14976 ("$type".to_string(), AvroValue::String("map".to_string())),
14977 (
14978 "entries".to_string(),
14979 AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([
14980 ("key".to_string(), AvroValue::String("ordinary".to_string())),
14981 (
14982 "value".to_string(),
14983 AvroValue::String("user map".to_string()),
14984 ),
14985 ]))]),
14986 ),
14987 ])),
14988 ]
14989 }
14990
14991 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
14992 struct TypedContract {
14993 nested: TypedNested,
14994 mode: TypedMode,
14995 optional: Option<String>,
14996 absent: Option<String>,
14997 items: Vec<i64>,
14998 labels: BTreeMap<String, String>,
14999 bytes: serde_bytes::ByteBuf,
15000 signed: i64,
15001 finite: f64,
15002 }
15003
15004 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
15005 struct TypedNested {
15006 enabled: bool,
15007 }
15008
15009 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
15010 enum TypedMode {
15011 Detailed { label: String },
15012 }
15013
15014 fn typed_contract() -> TypedContract {
15015 TypedContract {
15016 nested: TypedNested { enabled: true },
15017 mode: TypedMode::Detailed {
15018 label: "compiler-checked".to_string(),
15019 },
15020 optional: Some("present".to_string()),
15021 absent: None,
15022 items: vec![i64::MIN, 0, i64::MAX],
15023 labels: BTreeMap::from([
15024 ("language".to_string(), "rust".to_string()),
15025 ("wire".to_string(), "avro".to_string()),
15026 ]),
15027 bytes: serde_bytes::ByteBuf::from(vec![0, 0xff, 7]),
15028 signed: -9_223_372_036_854_775_000,
15029 finite: 12.5,
15030 }
15031 }
15032
15033 #[derive(Clone, Debug, Default, PartialEq)]
15034 struct ReplayCounterState {
15035 loaded: Option<String>,
15036 count: i64,
15037 finished: bool,
15038 }
15039
15040 fn replay_counter_worker() -> Worker {
15041 let client = Client::new("http://127.0.0.1:8080").expect("client");
15042 let mut worker = Worker::new(client, "rust-workers");
15043 worker.register_replayed_workflow(
15044 "replay-counter",
15045 ReplayCounterState::default,
15046 |ctx, _input, state| async move {
15047 let loaded = ctx.activity("load-counter", json!([])).await?;
15048 state.update(|current| {
15049 current.loaded = loaded.as_str().map(str::to_string);
15050 })?;
15051 for _ in 0..2 {
15052 let signal = ctx.wait_signal("increment").await?;
15053 let amount = signal.first().and_then(Value::as_i64).unwrap_or_default();
15054 state.update(|current| current.count += amount)?;
15055 }
15056 state.update(|current| current.finished = true)?;
15057 state.read(|current| Ok(json!(current.count)))?
15058 },
15059 );
15060 worker.register_replayed_query::<ReplayCounterState, _, _>(
15061 "replay-counter",
15062 "current",
15063 |_ctx, state, _args| async move {
15064 Ok(json!({
15065 "loaded": state.loaded,
15066 "count": state.count,
15067 "finished": state.finished,
15068 }))
15069 },
15070 );
15071 worker.register_replayed_query::<ReplayCounterState, _, _>(
15072 "replay-counter",
15073 "detached-mutation",
15074 |_ctx, state, _args| async move {
15075 let mut detached = (*state).clone();
15076 detached.count = 999;
15077 Ok(json!(detached.count))
15078 },
15079 );
15080 worker.register_replayed_query::<ReplayCounterState, _, _>(
15081 "replay-counter",
15082 "failed-mutation",
15083 |_ctx, state, _args| async move {
15084 let mut detached = (*state).clone();
15085 detached.count = 999;
15086 Err(Error::WorkerLoop("query refused".to_string()))
15087 },
15088 );
15089 worker
15090 }
15091
15092 fn replay_counter_query(
15093 query_name: &str,
15094 history_events: Value,
15095 run_status: &str,
15096 ) -> QueryTask {
15097 let arguments = fixture_envelope(json!([]));
15098 serde_json::from_value(json!({
15099 "query_task_id": format!("query-{query_name}"),
15100 "workflow_type": "replay-counter",
15101 "query_name": query_name,
15102 "payload_codec": DEFAULT_CODEC,
15103 "workflow_arguments": arguments.clone(),
15104 "query_arguments": arguments,
15105 "history_events": history_events,
15106 "run_status": run_status,
15107 }))
15108 .expect("query task")
15109 }
15110
15111 fn workflow_context(history: Vec<HistoryEvent>) -> WorkflowContext {
15112 workflow_context_with_codec(history, DEFAULT_CODEC)
15113 }
15114
15115 fn workflow_context_with_codec(
15116 history: Vec<HistoryEvent>,
15117 payload_codec: &str,
15118 ) -> WorkflowContext {
15119 WorkflowContext {
15120 state: Arc::new(Mutex::new(
15121 WorkflowState::new_with_identity(
15122 history,
15123 None,
15124 None,
15125 "rust-workers".to_string(),
15126 payload_codec.to_string(),
15127 None,
15128 )
15129 .expect("valid workflow history"),
15130 )),
15131 }
15132 }
15133
15134 fn history_event(event_type: &str, payload: Value) -> HistoryEvent {
15135 HistoryEvent {
15136 event_type: event_type.to_string(),
15137 payload,
15138 raw: HashMap::new(),
15139 }
15140 }
15141
15142 fn parallel_path_entry(
15143 kind: &str,
15144 base: u64,
15145 size: usize,
15146 index: usize,
15147 ) -> ParallelGroupMetadata {
15148 parallel_group_entry(base, size, index, kind)
15149 }
15150
15151 fn parallel_history_event(
15152 event_type: &str,
15153 sequence: u64,
15154 identity_field: &str,
15155 identity: &str,
15156 path: Vec<ParallelGroupMetadata>,
15157 result: Option<Value>,
15158 ) -> HistoryEvent {
15159 let mut payload = serde_json::Map::from_iter([
15160 ("sequence".to_string(), json!(sequence)),
15161 (identity_field.to_string(), json!(identity)),
15162 ]);
15163 let inner = path.last().expect("parallel history path");
15164 apply_parallel_group_path(&mut payload, std::slice::from_ref(inner));
15165 payload.insert("parallel_group_path".to_string(), json!(path));
15166 if let Some(result) = result {
15167 let field = if event_type == "ChildRunCompleted" {
15168 "result"
15169 } else {
15170 "result"
15171 };
15172 payload.insert(field.to_string(), fixture_envelope(result));
15173 payload.insert("payload_codec".to_string(), json!(DEFAULT_CODEC));
15174 }
15175 history_event(event_type, Value::Object(payload))
15176 }
15177
15178 fn nested_parallel_operations() -> Vec<ParallelOperation> {
15179 vec![
15180 ParallelOperation::activity("first", json!([])),
15181 ParallelOperation::group(vec![
15182 ParallelOperation::child_workflow(
15183 "second",
15184 ChildWorkflowOptions::new("child-workers"),
15185 json!([]),
15186 ),
15187 ParallelOperation::activity("third", json!([])),
15188 ]),
15189 ]
15190 }
15191
15192 fn nested_parallel_paths() -> [Vec<ParallelGroupMetadata>; 3] {
15193 let outer = [
15194 parallel_path_entry("mixed", 1, 3, 0),
15195 parallel_path_entry("mixed", 1, 3, 1),
15196 parallel_path_entry("mixed", 1, 3, 2),
15197 ];
15198 [
15199 vec![outer[0].clone()],
15200 vec![outer[1].clone(), parallel_path_entry("mixed", 2, 2, 0)],
15201 vec![outer[2].clone(), parallel_path_entry("mixed", 2, 2, 1)],
15202 ]
15203 }
15204
15205 #[test]
15206 fn parallel_schedules_every_nested_mixed_leaf_with_stable_metadata() {
15207 let ctx = workflow_context(Vec::new());
15208 let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15209 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15210
15211 assert!(matches!(
15212 call.as_mut().poll(&mut task_context),
15213 Poll::Pending
15214 ));
15215 let commands = ctx.take_commands().expect("parallel commands");
15216 assert_eq!(
15217 commands
15218 .iter()
15219 .map(|command| command["type"].as_str().unwrap_or_default())
15220 .collect::<Vec<_>>(),
15221 [
15222 "schedule_activity",
15223 "start_child_workflow",
15224 "schedule_activity"
15225 ]
15226 );
15227 let paths = nested_parallel_paths();
15228 for (command, path) in commands.iter().zip(paths) {
15229 assert_eq!(command["parallel_group_path"], json!(path));
15230 assert_eq!(
15231 command["parallel_group_id"],
15232 json!(path.last().expect("inner group").parallel_group_id)
15233 );
15234 }
15235 }
15236
15237 fn completed_nested_parallel_history() -> Vec<HistoryEvent> {
15238 let paths = nested_parallel_paths();
15239 let third = parallel_history_event(
15240 "ActivityCompleted",
15241 3,
15242 "activity_type",
15243 "third",
15244 paths[2].clone(),
15245 Some(json!("three")),
15246 );
15247 vec![
15248 parallel_history_event(
15249 "ActivityCompleted",
15250 1,
15251 "activity_type",
15252 "first",
15253 paths[0].clone(),
15254 Some(json!("one")),
15255 ),
15256 parallel_history_event(
15257 "ChildWorkflowScheduled",
15258 2,
15259 "child_workflow_type",
15260 "second",
15261 paths[1].clone(),
15262 None,
15263 ),
15264 parallel_history_event(
15265 "ChildRunCompleted",
15266 2,
15267 "child_workflow_type",
15268 "second",
15269 paths[1].clone(),
15270 Some(json!("two")),
15271 ),
15272 third.clone(),
15273 third,
15274 ]
15275 }
15276
15277 #[test]
15278 fn parallel_replay_rebuilds_input_order_and_tolerates_duplicate_delivery() {
15279 for _restart_or_completed_replay in 0..2 {
15280 let ctx = workflow_context(completed_nested_parallel_history());
15281 let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15282 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15283 let Poll::Ready(Ok(results)) = call.as_mut().poll(&mut task_context) else {
15284 panic!("completed nested parallel history must replay");
15285 };
15286 assert_eq!(
15287 results,
15288 vec![
15289 ParallelResult::Activity(json!("one")),
15290 ParallelResult::Group(vec![
15291 ParallelResult::ChildWorkflow(ChildWorkflowResult {
15292 parent: WorkflowIdentity {
15293 workflow_id: None,
15294 run_id: None,
15295 },
15296 child: WorkflowIdentity {
15297 workflow_id: None,
15298 run_id: None,
15299 },
15300 child_workflow_type: Some("second".to_string()),
15301 result: json!("two"),
15302 }),
15303 ParallelResult::Activity(json!("three")),
15304 ]),
15305 ]
15306 );
15307 assert!(ctx.take_commands().expect("commands").is_empty());
15308 ctx.ensure_history_consumed().expect("history consumed");
15309 }
15310 }
15311
15312 #[test]
15313 fn parallel_failure_keeps_typed_cause_path_and_late_completions() {
15314 let paths = nested_parallel_paths();
15315 let history = vec![
15316 parallel_history_event(
15317 "ActivityCompleted",
15318 1,
15319 "activity_type",
15320 "first",
15321 paths[0].clone(),
15322 Some(json!("one")),
15323 ),
15324 parallel_history_event(
15325 "ChildWorkflowScheduled",
15326 2,
15327 "child_workflow_type",
15328 "second",
15329 paths[1].clone(),
15330 None,
15331 ),
15332 parallel_history_event(
15333 "ChildRunFailed",
15334 2,
15335 "child_workflow_type",
15336 "second",
15337 paths[1].clone(),
15338 None,
15339 ),
15340 parallel_history_event(
15341 "ActivityCompleted",
15342 3,
15343 "activity_type",
15344 "third",
15345 paths[2].clone(),
15346 Some(json!("late")),
15347 ),
15348 ];
15349 let ctx = workflow_context(history);
15350 let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15351 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15352 let outcome = call.as_mut().poll(&mut task_context);
15353 let Poll::Ready(Err(Error::ParallelFailed(failure))) = outcome else {
15354 panic!("one failed child must return a typed partial failure: {outcome:?}");
15355 };
15356 assert_eq!(failure.member_path, [1, 0]);
15357 assert_eq!(failure.group_id, "parallel-calls:1:3");
15358 assert!(matches!(*failure.cause, Error::ChildWorkflowFailed(_)));
15359 assert_eq!(
15360 failure
15361 .completed
15362 .iter()
15363 .map(|completion| completion.member_path.clone())
15364 .collect::<Vec<_>>(),
15365 [vec![0], vec![1, 1]]
15366 );
15367 }
15368
15369 #[test]
15370 fn pending_parallel_history_restarts_without_rescheduling_any_leaf() {
15371 let paths = nested_parallel_paths();
15372 let history = vec![
15373 parallel_history_event(
15374 "ActivityScheduled",
15375 1,
15376 "activity_type",
15377 "first",
15378 paths[0].clone(),
15379 None,
15380 ),
15381 parallel_history_event(
15382 "ChildWorkflowScheduled",
15383 2,
15384 "child_workflow_type",
15385 "second",
15386 paths[1].clone(),
15387 None,
15388 ),
15389 parallel_history_event(
15390 "ActivityScheduled",
15391 3,
15392 "activity_type",
15393 "third",
15394 paths[2].clone(),
15395 None,
15396 ),
15397 ];
15398 for _restart in 0..2 {
15399 let ctx = workflow_context(history.clone());
15400 let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15401 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15402 let outcome = call.as_mut().poll(&mut task_context);
15403 assert!(matches!(outcome, Poll::Pending), "{outcome:?}");
15404 assert!(ctx.take_commands().expect("commands").is_empty());
15405 }
15406 }
15407
15408 fn selection_path(index: usize, key: &str) -> Vec<ParallelGroupMetadata> {
15409 vec![selection_group_entry(
15410 1,
15411 2,
15412 index,
15413 "activity",
15414 &SelectionMemberMetadata {
15415 key: SelectionKey::Name(key.to_string()),
15416 index,
15417 base_sequence: index as u64 + 1,
15418 size: 1,
15419 kind: "activity".to_string(),
15420 },
15421 )]
15422 }
15423
15424 fn selection_activity_event(
15425 event_type: &str,
15426 index: usize,
15427 key: &str,
15428 result: Option<Value>,
15429 ) -> HistoryEvent {
15430 let sequence = index as u64 + 1;
15431 let mut event = parallel_history_event(
15432 event_type,
15433 sequence,
15434 "activity_type",
15435 &format!("{key}-activity"),
15436 selection_path(index, key),
15437 result,
15438 );
15439 event.payload["activity_execution_id"] = json!(format!("activity-{key}"));
15440 event.raw.insert(
15441 "id".to_string(),
15442 json!(if event_type == "ActivityCompleted" {
15443 format!("event-{key}")
15444 } else {
15445 format!("{event_type}-{key}")
15446 }),
15447 );
15448 event
15449 }
15450
15451 fn selection_winner_marker() -> HistoryEvent {
15452 history_event(
15453 "SelectionResolved",
15454 json!({
15455 "selection_group_id": "select-calls:1:2",
15456 "selection_group_base_sequence": 1,
15457 "selection_group_size": 2,
15458 "member_key": "fast",
15459 "member_index": 1,
15460 "member_base_sequence": 2,
15461 "member_size": 1,
15462 "operation_kind": "activity",
15463 "operation_identity": "activity-fast",
15464 "outcome": "completed",
15465 "resolution_event_id": "event-fast",
15466 "resolution_event_type": "ActivityCompleted",
15467 }),
15468 )
15469 }
15470
15471 fn keyed_activity_selection(ctx: &WorkflowContext) -> SelectCall {
15472 ctx.select_keyed(vec![
15473 (
15474 "slow",
15475 ParallelOperation::activity_with_options(
15476 "slow-activity",
15477 ActivityOptions::new().task_queue("default"),
15478 json!([]),
15479 ),
15480 ),
15481 (
15482 "fast",
15483 ParallelOperation::activity_with_options(
15484 "fast-activity",
15485 ActivityOptions::new().task_queue("default"),
15486 json!([]),
15487 ),
15488 ),
15489 ])
15490 }
15491
15492 fn assert_persisted_selection_replay(history: Vec<HistoryEvent>) {
15493 let ctx = workflow_context(history);
15494 let mut call = Box::pin(keyed_activity_selection(&ctx));
15495 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15496 let selected = match call.as_mut().poll(&mut task_context) {
15497 Poll::Ready(Ok(selected)) => selected,
15498 Poll::Ready(Err(error)) => panic!("persisted selection winner must replay: {error:?}"),
15499 Poll::Pending => panic!("persisted selection winner must replay without pending"),
15500 };
15501 assert_eq!(selected.key, SelectionKey::Name("fast".to_string()));
15502 assert_eq!(
15503 selected.value,
15504 Some(ParallelResult::Activity(json!("winner-value")))
15505 );
15506 let slow = selected
15507 .handle(&SelectionKey::Name("slow".to_string()))
15508 .expect("slow handle")
15509 .clone();
15510 let mut await_slow = Box::pin(slow.await_result());
15511 assert!(matches!(
15512 await_slow.as_mut().poll(&mut task_context),
15513 Poll::Ready(Ok(ParallelResult::Activity(value))) if value == json!("loser-value")
15514 ));
15515 assert!(ctx.take_commands().expect("commands").is_empty());
15516 }
15517
15518 const SELECTION_COLD_REPLAY_HISTORY: &str = "DURABLE_WORKFLOW_SELECTION_COLD_REPLAY_HISTORY";
15519
15520 fn canonical_selection_history() -> Vec<HistoryEvent> {
15521 const FIXTURE: &[u8] =
15522 include_bytes!("../tests/fixtures/durable_selection_runtime_history.json");
15523 assert_eq!(
15524 format!("{:x}", Sha256::digest(FIXTURE)),
15525 "51fd8b9c16e978dcef536a5c727b9fdc0ae724d9afc17d9a7837d219f41ee3ba",
15526 );
15527 let fixture: Value = serde_json::from_slice(FIXTURE).expect("canonical selection fixture");
15528
15529 serde_json::from_value(fixture["history"].clone()).expect("canonical selection history")
15530 }
15531
15532 #[test]
15533 fn selection_fresh_process_entrypoint() {
15534 let Ok(path) = std::env::var(SELECTION_COLD_REPLAY_HISTORY) else {
15535 return;
15536 };
15537 let persisted = fs::read(path).expect("persisted selection history");
15538 assert_eq!(
15539 format!("{:x}", Sha256::digest(&persisted)),
15540 "51fd8b9c16e978dcef536a5c727b9fdc0ae724d9afc17d9a7837d219f41ee3ba",
15541 );
15542 let fixture: Value =
15543 serde_json::from_slice(&persisted).expect("valid persisted selection fixture");
15544 let history: Vec<HistoryEvent> = serde_json::from_value(fixture["history"].clone())
15545 .expect("valid persisted selection history");
15546
15547 assert_persisted_selection_replay(history);
15548 }
15549
15550 #[test]
15551 fn selection_starts_every_member_with_stable_keys_and_group_identity() {
15552 let ctx = workflow_context(Vec::new());
15553 let mut call = Box::pin(keyed_activity_selection(&ctx));
15554 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15555
15556 assert!(matches!(
15557 call.as_mut().poll(&mut task_context),
15558 Poll::Pending
15559 ));
15560 let commands = ctx.take_commands().expect("selection commands");
15561 assert_eq!(commands.len(), 2);
15562 assert_eq!(commands[0]["selection_member_key"], json!("slow"));
15563 assert_eq!(commands[1]["selection_member_key"], json!("fast"));
15564 assert!(commands.iter().all(|command| {
15565 command["parallel_group_id"] == json!("select-calls:1:2")
15566 && command["parallel_group_mode"] == json!("select")
15567 }));
15568 }
15569
15570 #[test]
15571 fn selection_key_domain_rejects_empty_authoring_and_malformed_history() {
15572 let ctx = workflow_context(Vec::new());
15573 let mut invalid = Box::pin(ctx.select_keyed(vec![(
15574 "",
15575 ParallelOperation::activity("invalid", json!([])),
15576 )]));
15577 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15578 assert!(matches!(
15579 invalid.as_mut().poll(&mut task_context),
15580 Poll::Ready(Err(Error::InvalidParallelGroup(ParallelGroupError {
15581 reason: "selection_key_invalid",
15582 ..
15583 })))
15584 ));
15585
15586 for invalid_key in [json!(""), json!(-1)] {
15587 let mut event = selection_activity_event("ActivityScheduled", 0, "slow", None);
15588 event.payload["selection_member_key"] = invalid_key.clone();
15589 event.payload["parallel_group_path"][0]["selection_member_key"] = invalid_key;
15590 assert!(matches!(
15591 WorkflowState::new_with_identity(
15592 vec![event],
15593 None,
15594 None,
15595 "rust-workers".to_string(),
15596 DEFAULT_CODEC.to_string(),
15597 None,
15598 ),
15599 Err(Error::NonDeterministicReplay(_))
15600 ));
15601 }
15602 }
15603
15604 #[test]
15605 fn selection_preserves_valid_named_and_numeric_keys() {
15606 let ctx = workflow_context(Vec::new());
15607 let mut selection = Box::pin(ctx.select_keyed(vec![
15608 (
15609 SelectionKey::Index(0),
15610 ParallelOperation::activity("numeric", json!([])),
15611 ),
15612 (
15613 SelectionKey::Name("named".to_string()),
15614 ParallelOperation::timer(Duration::from_secs(1)),
15615 ),
15616 ]));
15617 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15618
15619 assert!(matches!(
15620 selection.as_mut().poll(&mut task_context),
15621 Poll::Pending
15622 ));
15623 let commands = ctx.take_commands().expect("selection commands");
15624 assert_eq!(commands[0]["selection_member_key"], json!(0));
15625 assert_eq!(commands[1]["selection_member_key"], json!("named"));
15626 }
15627
15628 #[test]
15629 fn selection_replays_persisted_winner_and_loser_can_be_awaited_later() {
15630 let history = canonical_selection_history();
15631 assert_persisted_selection_replay(history.clone());
15632
15633 let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
15634 .join("tests/fixtures/durable_selection_runtime_history.json");
15635 let output =
15636 ProcessCommand::new(std::env::current_exe().expect("current Rust test binary"))
15637 .args([
15638 "--exact",
15639 "tests::selection_fresh_process_entrypoint",
15640 "--nocapture",
15641 ])
15642 .env(SELECTION_COLD_REPLAY_HISTORY, &path)
15643 .output()
15644 .expect("run fresh selection replay process");
15645
15646 assert!(
15647 output.status.success(),
15648 "fresh selection replay failed:\nstdout:\n{}\nstderr:\n{}",
15649 String::from_utf8_lossy(&output.stdout),
15650 String::from_utf8_lossy(&output.stderr),
15651 );
15652 }
15653
15654 #[test]
15655 fn selection_waits_durably_when_terminal_members_precede_the_winner_marker() {
15656 let mut history = canonical_selection_history();
15657 history.retain(|event| event.event_type != "SelectionResolved");
15658 let ctx = workflow_context(history);
15659 let mut selection = Box::pin(keyed_activity_selection(&ctx));
15660 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15661
15662 assert!(matches!(
15663 selection.as_mut().poll(&mut task_context),
15664 Poll::Pending
15665 ));
15666 assert!(ctx.take_commands().expect("commands").is_empty());
15667 assert!(
15668 ctx.matched_recorded_pending()
15669 .expect("selection pending state"),
15670 "terminal member history must keep the workflow durably pending until SelectionResolved commits"
15671 );
15672 }
15673
15674 #[test]
15675 fn selection_terminal_condition_history_waits_durably_for_its_winner_marker() {
15676 for (terminal_event, predicate_satisfied, timeout_seconds) in [
15677 ("ConditionWaitSatisfied", true, None),
15678 ("ConditionWaitTimedOut", false, Some(0)),
15679 ] {
15680 let member = SelectionMemberMetadata {
15681 key: SelectionKey::Name("condition".to_string()),
15682 index: 0,
15683 base_sequence: 1,
15684 size: 1,
15685 kind: "condition".to_string(),
15686 };
15687 let path = vec![selection_group_entry(1, 1, 0, "condition", &member)];
15688 let mut payload = json!({
15689 "sequence": 1,
15690 "condition_wait_id": "condition-1",
15691 "condition_wait_occurrence_id": "rust:condition-wait:0",
15692 "condition_key": "ready",
15693 "condition_definition_fingerprint": "sha256:ready-v1",
15694 "parallel_group_path": path,
15695 });
15696 payload
15697 .as_object_mut()
15698 .expect("condition history payload")
15699 .extend(
15700 serde_json::to_value(&path[0])
15701 .expect("condition selection metadata")
15702 .as_object()
15703 .expect("condition selection metadata object")
15704 .clone(),
15705 );
15706 if let Some(timeout_seconds) = timeout_seconds {
15707 payload["timeout_seconds"] = json!(timeout_seconds);
15708 }
15709 let history = vec![
15710 history_event("ConditionWaitOpened", payload.clone()),
15711 history_event(terminal_event, payload),
15712 ];
15713 let ctx = workflow_context(history);
15714 let mut options = ConditionWaitOptions::new("ready", "sha256:ready-v1");
15715 if timeout_seconds.is_some() {
15716 options = options.timeout(Duration::ZERO);
15717 }
15718 let mut selection = Box::pin(ctx.select_keyed(vec![(
15719 "condition",
15720 ParallelOperation::condition(options, move || Ok(predicate_satisfied)),
15721 )]));
15722 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15723
15724 assert!(matches!(
15725 selection.as_mut().poll(&mut task_context),
15726 Poll::Pending
15727 ));
15728 assert!(ctx.take_commands().expect("commands").is_empty());
15729 assert!(
15730 ctx.matched_recorded_pending()
15731 .expect("condition selection pending state"),
15732 "{terminal_event} must keep the workflow durably pending until SelectionResolved commits"
15733 );
15734 }
15735 }
15736
15737 #[test]
15738 fn selection_immediate_condition_members_open_a_durable_wait() {
15739 for predicate_satisfied in [true, false] {
15740 let ctx = workflow_context(Vec::new());
15741 let mut selection = Box::pin(ctx.select_keyed(vec![(
15742 "condition",
15743 ParallelOperation::condition(
15744 ConditionWaitOptions::new("ready", "sha256:ready-v1").timeout(Duration::ZERO),
15745 move || Ok(predicate_satisfied),
15746 ),
15747 )]));
15748 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15749
15750 assert!(matches!(
15751 selection.as_mut().poll(&mut task_context),
15752 Poll::Pending
15753 ));
15754 let commands = ctx.take_commands().expect("condition selection command");
15755 assert_eq!(commands.len(), 1);
15756 assert_eq!(commands[0]["type"], json!("open_condition_wait"));
15757 assert_eq!(commands[0]["timeout_seconds"], json!(0));
15758 assert_eq!(
15759 commands[0]["parallel_group_path"][0]["parallel_group_mode"],
15760 json!("select")
15761 );
15762 }
15763 }
15764
15765 #[test]
15766 fn selection_loser_cancellation_is_explicit_and_idempotent() {
15767 let history = vec![
15768 selection_activity_event("ActivityScheduled", 0, "slow", None),
15769 selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
15770 selection_winner_marker(),
15771 ];
15772 let ctx = workflow_context(history.clone());
15773 let mut call = Box::pin(keyed_activity_selection(&ctx));
15774 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15775 let Poll::Ready(Ok(selected)) = call.as_mut().poll(&mut task_context) else {
15776 panic!("winner must replay");
15777 };
15778 let slow = selected
15779 .handle(&SelectionKey::Name("slow".to_string()))
15780 .expect("slow handle")
15781 .clone();
15782 let mut cancel = Box::pin(slow.cancel());
15783 assert!(matches!(
15784 cancel.as_mut().poll(&mut task_context),
15785 Poll::Pending
15786 ));
15787 assert!(matches!(
15788 cancel.as_mut().poll(&mut task_context),
15789 Poll::Pending
15790 ));
15791 let commands = ctx.take_commands().expect("cancel command");
15792 assert_eq!(commands.len(), 1);
15793 assert_eq!(commands[0]["type"], json!("cancel_selection_operation"));
15794 assert_eq!(commands[0]["member_key"], json!("slow"));
15795
15796 let mut cancelled_history = history;
15797 cancelled_history.push(history_event(
15798 "SelectionOperationCancelled",
15799 json!({
15800 "selection_group_id": "select-calls:1:2",
15801 "member_key": "slow",
15802 "member_index": 0,
15803 "member_base_sequence": 1,
15804 "member_size": 1,
15805 "operation_kind": "activity",
15806 "operation_identity": "activity-slow",
15807 "cancelled_at": "2026-08-27T00:00:00Z",
15808 }),
15809 ));
15810 let replayed = workflow_context(cancelled_history);
15811 let mut call = Box::pin(keyed_activity_selection(&replayed));
15812 let Poll::Ready(Ok(selected)) = call.as_mut().poll(&mut task_context) else {
15813 panic!("winner must replay after cancellation");
15814 };
15815 let slow = selected
15816 .handle(&SelectionKey::Name("slow".to_string()))
15817 .expect("slow handle")
15818 .clone();
15819 let mut cancel = Box::pin(slow.cancel());
15820 assert!(matches!(
15821 cancel.as_mut().poll(&mut task_context),
15822 Poll::Ready(Ok(()))
15823 ));
15824 assert!(replayed.take_commands().expect("commands").is_empty());
15825 }
15826
15827 #[test]
15828 fn selection_cancellation_marker_is_bound_to_every_authored_handle_field() {
15829 let base_history = vec![
15830 selection_activity_event("ActivityScheduled", 0, "slow", None),
15831 selection_activity_event("ActivityScheduled", 1, "fast", None),
15832 selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
15833 selection_winner_marker(),
15834 ];
15835 for (field, corrupt) in [
15836 ("member_key", json!("fast")),
15837 ("member_index", json!(1)),
15838 ("member_base_sequence", json!(3)),
15839 ("member_size", json!(2)),
15840 ("operation_kind", json!("timer")),
15841 ("operation_identity", json!("forged")),
15842 ] {
15843 let mut cancellation = json!({
15844 "selection_group_id": "select-calls:1:2",
15845 "member_key": "slow",
15846 "member_index": 0,
15847 "member_base_sequence": 1,
15848 "member_size": 1,
15849 "operation_kind": "activity",
15850 "operation_identity": "activity-slow",
15851 });
15852 cancellation[field] = corrupt;
15853 let mut history = base_history.clone();
15854 history.push(history_event("SelectionOperationCancelled", cancellation));
15855 let ctx = workflow_context(history);
15856 let mut selection = Box::pin(keyed_activity_selection(&ctx));
15857 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15858
15859 assert!(matches!(
15860 selection.as_mut().poll(&mut task_context),
15861 Poll::Ready(Err(Error::NonDeterministicReplay(_)))
15862 ));
15863 }
15864 }
15865
15866 #[test]
15867 fn selection_child_identity_prefers_the_durable_run_id() {
15868 let ctx = workflow_context(vec![history_event(
15869 "ChildWorkflowScheduled",
15870 json!({
15871 "sequence": 1,
15872 "child_workflow_type": "child",
15873 "child_workflow_instance_id": "child-instance",
15874 "child_workflow_run_id": "child-run",
15875 }),
15876 )]);
15877 let state = ctx.state.lock().expect("workflow state");
15878
15879 assert_eq!(
15880 selection_operation_identity(&state, "child", 1, 1),
15881 "child-run"
15882 );
15883 }
15884
15885 #[test]
15886 fn selection_activity_identity_requires_canonical_execution_id() {
15887 let slow = selection_activity_event("ActivityScheduled", 0, "slow", None);
15888 let mut fast_open = selection_activity_event("ActivityScheduled", 1, "fast", None);
15889 let mut fast_completed =
15890 selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner")));
15891 for event in [&mut fast_open, &mut fast_completed] {
15892 event
15893 .payload
15894 .as_object_mut()
15895 .expect("activity payload")
15896 .remove("activity_execution_id");
15897 event.payload["activity_id"] = json!("forged-activity-id");
15898 }
15899 let mut marker = selection_winner_marker();
15900 marker.payload["operation_identity"] = json!("forged-activity-id");
15901 let ctx = workflow_context(vec![slow, fast_open, fast_completed, marker]);
15902 let mut selection = Box::pin(keyed_activity_selection(&ctx));
15903 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15904
15905 assert!(matches!(
15906 selection.as_mut().poll(&mut task_context),
15907 Poll::Ready(Err(Error::NonDeterministicReplay(_)))
15908 ));
15909 }
15910
15911 #[test]
15912 fn selection_completion_before_cancellation_remains_awaitable() {
15913 let history = vec![
15914 selection_activity_event("ActivityScheduled", 0, "slow", None),
15915 selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
15916 selection_winner_marker(),
15917 selection_activity_event(
15918 "ActivityCompleted",
15919 0,
15920 "slow",
15921 Some(json!("completed-first")),
15922 ),
15923 ];
15924 let ctx = workflow_context(history);
15925 let mut selection = Box::pin(keyed_activity_selection(&ctx));
15926 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15927 let Poll::Ready(Ok(selected)) = selection.as_mut().poll(&mut task_context) else {
15928 panic!("winner must replay");
15929 };
15930 let slow = selected
15931 .handle(&SelectionKey::Name("slow".to_string()))
15932 .expect("slow handle")
15933 .clone();
15934 let mut cancel = Box::pin(slow.cancel());
15935 assert!(matches!(
15936 cancel.as_mut().poll(&mut task_context),
15937 Poll::Ready(Ok(()))
15938 ));
15939 let mut await_slow = Box::pin(slow.await_result());
15940 assert!(matches!(
15941 await_slow.as_mut().poll(&mut task_context),
15942 Poll::Ready(Ok(ParallelResult::Activity(value))) if value == json!("completed-first")
15943 ));
15944 let commands = ctx.take_commands().expect("commands");
15945 assert!(commands.is_empty());
15946 }
15947
15948 #[test]
15949 fn selection_nested_later_failure_before_cancel_remains_the_awaited_failure() {
15950 let nested_member = SelectionMemberMetadata {
15951 key: SelectionKey::Name("nested".to_string()),
15952 index: 0,
15953 base_sequence: 1,
15954 size: 2,
15955 kind: "group".to_string(),
15956 };
15957 let deadline_member = SelectionMemberMetadata {
15958 key: SelectionKey::Name("deadline".to_string()),
15959 index: 1,
15960 base_sequence: 3,
15961 size: 1,
15962 kind: "timer".to_string(),
15963 };
15964 let nested_paths = [
15965 vec![
15966 selection_group_entry(1, 3, 0, "mixed", &nested_member),
15967 parallel_group_entry(1, 2, 0, "activity"),
15968 ],
15969 vec![
15970 selection_group_entry(1, 3, 1, "mixed", &nested_member),
15971 parallel_group_entry(1, 2, 1, "activity"),
15972 ],
15973 ];
15974 let deadline_path = vec![selection_group_entry(1, 3, 2, "mixed", &deadline_member)];
15975 let mut timer_fired = parallel_history_event(
15976 "TimerFired",
15977 3,
15978 "timer_id",
15979 "timer-3",
15980 deadline_path.clone(),
15981 None,
15982 );
15983 timer_fired.payload["delay_seconds"] = json!(0);
15984 timer_fired
15985 .raw
15986 .insert("id".to_string(), json!("timer-fired"));
15987 let mut timer_scheduled = parallel_history_event(
15988 "TimerScheduled",
15989 3,
15990 "timer_id",
15991 "timer-3",
15992 deadline_path,
15993 None,
15994 );
15995 timer_scheduled.payload["delay_seconds"] = json!(0);
15996 let history = vec![
15997 parallel_history_event(
15998 "ActivityScheduled",
15999 1,
16000 "activity_type",
16001 "nested-first",
16002 nested_paths[0].clone(),
16003 None,
16004 ),
16005 parallel_history_event(
16006 "ActivityScheduled",
16007 2,
16008 "activity_type",
16009 "nested-second",
16010 nested_paths[1].clone(),
16011 None,
16012 ),
16013 timer_scheduled,
16014 timer_fired,
16015 history_event(
16016 "SelectionResolved",
16017 json!({
16018 "selection_group_id": "select-calls:1:3",
16019 "selection_group_base_sequence": 1,
16020 "selection_group_size": 3,
16021 "member_key": "deadline",
16022 "member_index": 1,
16023 "member_base_sequence": 3,
16024 "member_size": 1,
16025 "operation_kind": "timer",
16026 "operation_identity": "timer-3",
16027 "outcome": "completed",
16028 "resolution_event_id": "timer-fired",
16029 "resolution_event_type": "TimerFired",
16030 }),
16031 ),
16032 parallel_history_event(
16033 "ActivityFailed",
16034 2,
16035 "activity_type",
16036 "nested-second",
16037 nested_paths[1].clone(),
16038 None,
16039 ),
16040 ];
16041 let ctx = workflow_context(history);
16042 let mut selection = Box::pin(ctx.select_keyed(vec![
16043 (
16044 "nested",
16045 ParallelOperation::group(vec![
16046 ParallelOperation::activity("nested-first", json!([])),
16047 ParallelOperation::activity("nested-second", json!([])),
16048 ]),
16049 ),
16050 ("deadline", ParallelOperation::timer(Duration::ZERO)),
16051 ]));
16052 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16053 let Poll::Ready(Ok(selected)) = selection.as_mut().poll(&mut task_context) else {
16054 panic!("deadline winner must replay");
16055 };
16056 let nested = selected
16057 .handle(&SelectionKey::Name("nested".to_string()))
16058 .expect("nested handle")
16059 .clone();
16060 let mut cancel = Box::pin(nested.cancel());
16061 assert!(matches!(
16062 cancel.as_mut().poll(&mut task_context),
16063 Poll::Ready(Ok(()))
16064 ));
16065 let mut await_nested = Box::pin(nested.await_result());
16066
16067 assert!(matches!(
16068 await_nested.as_mut().poll(&mut task_context),
16069 Poll::Ready(Err(Error::ActivityFailed(_)))
16070 ));
16071 assert!(ctx.take_commands().expect("commands").is_empty());
16072 }
16073
16074 #[test]
16075 fn selection_supports_child_timer_signal_condition_and_nested_groups() {
16076 let ctx = workflow_context(Vec::new());
16077 let mut call = Box::pin(ctx.select(vec![
16078 ParallelOperation::child_workflow(
16079 "child",
16080 ChildWorkflowOptions::new("children"),
16081 json!([]),
16082 ),
16083 ParallelOperation::timer(Duration::from_secs(30)),
16084 ParallelOperation::signal("approval"),
16085 ParallelOperation::condition(
16086 ConditionWaitOptions::new("ready", "sha256:ready"),
16087 || Ok(false),
16088 ),
16089 ParallelOperation::group(vec![
16090 ParallelOperation::activity("nested-one", json!([])),
16091 ParallelOperation::activity("nested-two", json!([])),
16092 ]),
16093 ]));
16094 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16095 assert!(matches!(
16096 call.as_mut().poll(&mut task_context),
16097 Poll::Pending
16098 ));
16099 let commands = ctx.take_commands().expect("selection commands");
16100 assert_eq!(
16101 commands
16102 .iter()
16103 .map(|command| command["type"].as_str().unwrap_or_default())
16104 .collect::<Vec<_>>(),
16105 [
16106 "start_child_workflow",
16107 "start_timer",
16108 "open_signal_wait",
16109 "open_condition_wait",
16110 "schedule_activity",
16111 "schedule_activity",
16112 ]
16113 );
16114 assert!(commands.iter().all(|command| {
16115 command["parallel_group_path"][0]["parallel_group_mode"] == json!("select")
16116 }));
16117 assert_eq!(
16118 commands[4]["parallel_group_path"].as_array().map(Vec::len),
16119 Some(2)
16120 );
16121 assert_eq!(
16122 commands[4]["parallel_group_path"][0]["selection_member_kind"],
16123 json!("group")
16124 );
16125 assert_eq!(
16126 commands[5]["parallel_group_path"][0]["selection_member_kind"],
16127 json!("group")
16128 );
16129
16130 let one_leaf_ctx = workflow_context(Vec::new());
16131 let mut one_leaf = Box::pin(one_leaf_ctx.select(vec![ParallelOperation::group(vec![
16132 ParallelOperation::activity("nested-only", json!([])),
16133 ])]));
16134 assert!(matches!(
16135 one_leaf.as_mut().poll(&mut task_context),
16136 Poll::Pending
16137 ));
16138 let one_leaf_commands = one_leaf_ctx.take_commands().expect("one-leaf commands");
16139 assert_eq!(one_leaf_commands.len(), 1);
16140 assert_eq!(
16141 one_leaf_commands[0]["parallel_group_path"][0]["selection_member_kind"],
16142 json!("group")
16143 );
16144 assert_eq!(
16145 one_leaf_commands[0]["parallel_group_path"][0]["selection_member_size"],
16146 json!(1)
16147 );
16148 }
16149
16150 async fn trip_saga(ctx: WorkflowContext) -> Result<Value> {
16151 let mut saga = ctx.saga();
16152 let outcome = async {
16153 let flight = ctx.activity("trip.reserve-flight", json!([])).await?;
16154 saga.add_compensation("trip.cancel-flight", json!([flight]))?;
16155 let hotel = ctx.activity("trip.reserve-hotel", json!([])).await?;
16156 saga.add_compensation("trip.cancel-hotel", json!([hotel]))?;
16157 ctx.activity("trip.charge", json!([])).await?;
16158 Ok(json!({"status": "booked"}))
16159 }
16160 .await;
16161 saga.finish(outcome).await
16162 }
16163
16164 fn saga_activity(
16165 event_type: &str,
16166 sequence: u64,
16167 activity_type: &str,
16168 result: Option<Value>,
16169 ) -> HistoryEvent {
16170 let mut payload = json!({
16171 "sequence": sequence,
16172 "activity_type": activity_type,
16173 "message": format!("{activity_type} failed"),
16174 "exception_type": "PlannedFailure",
16175 "non_retryable": true,
16176 });
16177 if let Some(result) = result {
16178 payload["result"] = fixture_envelope(result);
16179 }
16180 history_event(event_type, payload)
16181 }
16182
16183 #[test]
16184 fn saga_replays_reverse_compensation_across_restart_and_duplicate_delivery() {
16185 let completed_hotel_compensation = saga_activity(
16186 "ActivityCompleted",
16187 4,
16188 "trip.cancel-hotel",
16189 Some(Value::Null),
16190 );
16191 let history = vec![
16192 saga_activity(
16193 "ActivityCompleted",
16194 1,
16195 "trip.reserve-flight",
16196 Some(json!("flight-1")),
16197 ),
16198 saga_activity(
16199 "ActivityCompleted",
16200 2,
16201 "trip.reserve-hotel",
16202 Some(json!("hotel-1")),
16203 ),
16204 saga_activity("ActivityFailed", 3, "trip.charge", None),
16205 completed_hotel_compensation.clone(),
16206 completed_hotel_compensation,
16207 ];
16208
16209 for _restart in 0..2 {
16210 let ctx = workflow_context(history.clone());
16211 let mut future = Box::pin(trip_saga(ctx.clone()));
16212 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16213 assert!(matches!(
16214 future.as_mut().poll(&mut task_context),
16215 Poll::Pending
16216 ));
16217 let commands = ctx.take_commands().expect("compensation command");
16218 assert_eq!(commands.len(), 1);
16219 assert_eq!(commands[0]["activity_type"], "trip.cancel-flight");
16220 }
16221 }
16222
16223 #[test]
16224 fn saga_compensation_failure_preserves_both_typed_failures() {
16225 let history = vec![
16226 saga_activity(
16227 "ActivityCompleted",
16228 1,
16229 "trip.reserve-flight",
16230 Some(json!("flight-1")),
16231 ),
16232 saga_activity(
16233 "ActivityCompleted",
16234 2,
16235 "trip.reserve-hotel",
16236 Some(json!("hotel-1")),
16237 ),
16238 saga_activity("ActivityFailed", 3, "trip.charge", None),
16239 saga_activity("ActivityFailed", 4, "trip.cancel-hotel", None),
16240 ];
16241 let ctx = workflow_context(history);
16242 let mut future = Box::pin(trip_saga(ctx));
16243 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16244 let Poll::Ready(Err(Error::SagaCompensationFailed(failure))) =
16245 future.as_mut().poll(&mut task_context)
16246 else {
16247 panic!("compensation failure must remain structured");
16248 };
16249 assert!(matches!(
16250 *failure.initiating_failure,
16251 Error::ActivityFailed(_)
16252 ));
16253 assert!(matches!(
16254 *failure.compensation_failure,
16255 Error::ActivityFailed(_)
16256 ));
16257 assert_eq!(failure.compensation_activity_type, "trip.cancel-hotel");
16258 assert_eq!(failure.compensation_registration_order, 2);
16259 }
16260
16261 #[test]
16262 fn saga_compensates_cooperative_cancellation() {
16263 let ctx = workflow_context(vec![saga_activity(
16264 "ActivityCompleted",
16265 1,
16266 "trip.reserve-flight",
16267 Some(json!("flight-1")),
16268 )]);
16269 ctx.state.lock().expect("state").cancel_requested = true;
16270 let run = {
16271 let ctx = ctx.clone();
16272 async move {
16273 let mut saga = ctx.saga();
16274 let outcome = async {
16275 let flight = ctx.activity("trip.reserve-flight", json!([])).await?;
16276 saga.add_compensation("trip.cancel-flight", json!([flight]))?;
16277 ctx.throw_if_cancellation_requested()?;
16278 Ok(json!("unexpected"))
16279 }
16280 .await;
16281 saga.finish(outcome).await
16282 }
16283 };
16284 let mut future = Box::pin(run);
16285 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16286 assert!(matches!(
16287 future.as_mut().poll(&mut task_context),
16288 Poll::Pending
16289 ));
16290 let commands = ctx.take_commands().expect("cancellation compensation");
16291 assert_eq!(commands[0]["activity_type"], "trip.cancel-flight");
16292 }
16293
16294 fn workflow_task(
16295 workflow_type: &str,
16296 history_events: Vec<HistoryEvent>,
16297 payload_codec: &str,
16298 ) -> WorkflowTask {
16299 WorkflowTask {
16300 task_id: format!("wft-{workflow_type}"),
16301 workflow_command_id: None,
16302 workflow_id: Some(format!("wf-{workflow_type}")),
16303 run_id: Some(format!("run-{workflow_type}")),
16304 workflow_type: workflow_type.to_string(),
16305 cancel_requested: false,
16306 payload_codec: payload_codec.to_string(),
16307 arguments: Some(
16308 encode_value_envelope(&json!([]), payload_codec).expect("workflow arguments"),
16309 ),
16310 total_history_events: Some(history_events.len() as u64),
16311 history_size_bytes: None,
16312 continue_as_new_recommended: None,
16313 history_budget_pressure: None,
16314 history_events,
16315 next_history_page_token: None,
16316 workflow_task_attempt: 1,
16317 workflow_signal_id: None,
16318 signal_name: None,
16319 signal_arguments: None,
16320 workflow_update_id: None,
16321 update_name: None,
16322 lease_owner: Some("rust-worker".to_string()),
16323 }
16324 }
16325
16326 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
16327 struct SideEffectProbe {
16328 request_id: String,
16329 attempt: u32,
16330 }
16331
16332 #[test]
16333 fn typed_side_effect_runs_callback_once_and_replay_skips_it() {
16334 let calls = AtomicUsize::new(0);
16335 let ctx = workflow_context(Vec::new());
16336 let value = ctx
16337 .side_effect(|| {
16338 calls.fetch_add(1, Ordering::SeqCst);
16339 SideEffectProbe {
16340 request_id: "request-42".to_string(),
16341 attempt: 3,
16342 }
16343 })
16344 .expect("first side effect");
16345 assert_eq!(value.attempt, 3);
16346 assert_eq!(calls.load(Ordering::SeqCst), 1);
16347 let commands = ctx.take_commands().expect("commands");
16348 assert_eq!(commands.len(), 1);
16349 assert_eq!(commands[0]["type"], "record_side_effect");
16350 assert_eq!(
16351 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
16352 serde_json::to_value(&value).expect("value")
16353 );
16354
16355 let replay = workflow_context(vec![history_event(
16356 "SideEffectRecorded",
16357 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16358 )]);
16359 let replayed: SideEffectProbe = replay
16360 .side_effect(|| {
16361 calls.fetch_add(1, Ordering::SeqCst);
16362 panic!("committed side-effect callbacks must not run during replay")
16363 })
16364 .expect("replayed side effect");
16365 assert_eq!(replayed, value);
16366 assert_eq!(calls.load(Ordering::SeqCst), 1);
16367 assert!(replay.take_commands().expect("commands").is_empty());
16368 replay.ensure_history_consumed().expect("history consumed");
16369 }
16370
16371 #[test]
16372 fn side_effect_uses_avro_envelope_and_uuid_is_replay_stable() {
16373 let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
16374 let value = ctx
16375 .side_effect(|| SideEffectProbe {
16376 request_id: "avro-request".to_string(),
16377 attempt: 1,
16378 })
16379 .expect("Avro side effect");
16380 let uuid = ctx.uuid_v4().expect("deterministic UUID");
16381 let commands = ctx.take_commands().expect("commands");
16382 assert_eq!(commands.len(), 2);
16383 assert_eq!(commands[0]["result"]["codec"], DEFAULT_CODEC);
16384 assert_eq!(commands[1]["result"]["codec"], DEFAULT_CODEC);
16385 assert_eq!(
16386 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
16387 serde_json::to_value(&value).expect("value")
16388 );
16389
16390 let replay = workflow_context_with_codec(
16391 vec![
16392 history_event(
16393 "SideEffectRecorded",
16394 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16395 ),
16396 history_event(
16397 "SideEffectRecorded",
16398 json!({"sequence": 2, "result": commands[1]["result"].clone()}),
16399 ),
16400 ],
16401 DEFAULT_CODEC,
16402 );
16403 let replayed: SideEffectProbe = replay
16404 .side_effect(|| panic!("Avro callback must not run"))
16405 .expect("replayed Avro value");
16406 let replayed_uuid = replay.uuid_v4().expect("replayed UUID");
16407 assert_eq!(replayed, value);
16408 assert_eq!(replayed_uuid, uuid);
16409 assert!(replay.take_commands().expect("commands").is_empty());
16410 }
16411
16412 #[test]
16413 fn typed_side_effect_replay_preserves_bytes_and_maps() {
16414 let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
16415 let value = ctx
16416 .side_effect_avro_value(typed_fidelity_probe)
16417 .expect("typed side effect");
16418 let commands = ctx.take_commands().expect("side-effect command");
16419 assert_eq!(
16420 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
16421 .expect("recorded side effect"),
16422 value
16423 );
16424
16425 let replay = workflow_context_with_codec(
16426 vec![history_event(
16427 "SideEffectRecorded",
16428 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16429 )],
16430 DEFAULT_CODEC,
16431 );
16432 assert_eq!(
16433 replay
16434 .side_effect_avro_value(|| panic!("replay must not invoke callback"))
16435 .expect("replayed typed side effect"),
16436 value
16437 );
16438 }
16439
16440 #[test]
16441 fn ordered_side_effects_share_the_durable_command_stream() {
16442 let first = encode_value_envelope(&json!("first"), DEFAULT_CODEC).expect("first");
16443 let second = encode_value_envelope(&json!(29), DEFAULT_CODEC).expect("second");
16444 let ctx = workflow_context(vec![
16445 history_event(
16446 "SideEffectRecorded",
16447 json!({"sequence": 1, "result": first}),
16448 ),
16449 history_event(
16450 "SideEffectRecorded",
16451 json!({"sequence": 2, "result": second}),
16452 ),
16453 ]);
16454 let first: String = ctx
16455 .side_effect(|| panic!("first callback must not run"))
16456 .expect("first replay");
16457 let second: i32 = ctx
16458 .side_effect(|| panic!("second callback must not run"))
16459 .expect("second replay");
16460 assert_eq!(first, "first");
16461 assert_eq!(second, 29);
16462 ctx.ensure_history_consumed().expect("ordered history");
16463
16464 let reordered = workflow_context(vec![history_event(
16465 "VersionMarkerRecorded",
16466 json!({
16467 "sequence": 1,
16468 "change_id": "before-side-effect",
16469 "version": 1,
16470 "min_supported": 1,
16471 "max_supported": 1,
16472 }),
16473 )]);
16474 let error = reordered
16475 .side_effect(|| "new".to_string())
16476 .expect_err("command reordering must fail");
16477 assert!(matches!(
16478 error,
16479 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16480 if reason == "recorded_command_mismatch"
16481 ));
16482 }
16483
16484 #[test]
16485 fn version_markers_replay_across_upgrades_and_do_not_duplicate() {
16486 let ctx = workflow_context(Vec::new());
16487 assert_eq!(ctx.get_version("checkout-v2", 1, 2).expect("version"), 2);
16488 assert_eq!(ctx.get_version("checkout-v2", 1, 3).expect("cached"), 2);
16489 assert!(ctx.patched("new-search").expect("patch"));
16490 ctx.deprecate_patch("new-search").expect("deprecate patch");
16491 let commands = ctx.take_commands().expect("commands");
16492 assert_eq!(commands.len(), 2);
16493 assert_eq!(commands[0]["type"], "record_version_marker");
16494 assert_eq!(commands[0]["version"], 2);
16495 assert_eq!(commands[1]["change_id"], "new-search");
16496
16497 let replay = workflow_context(vec![history_event(
16498 "VersionMarkerRecorded",
16499 json!({
16500 "sequence": 1,
16501 "change_id": "checkout-v2",
16502 "version": 2,
16503 "min_supported": 1,
16504 "max_supported": 2,
16505 }),
16506 )]);
16507 assert_eq!(replay.get_version("checkout-v2", 1, 4).expect("upgrade"), 2);
16508 assert_eq!(replay.get_version("checkout-v2", 2, 5).expect("repeat"), 2);
16509 assert!(replay.take_commands().expect("commands").is_empty());
16510 replay.ensure_history_consumed().expect("history consumed");
16511 }
16512
16513 #[test]
16514 fn version_markers_reject_incompatible_or_malformed_history() {
16515 let incompatible = workflow_context(vec![history_event(
16516 "VersionMarkerRecorded",
16517 json!({
16518 "sequence": 1,
16519 "change_id": "checkout-v2",
16520 "version": 1,
16521 "min_supported": 1,
16522 "max_supported": 2,
16523 }),
16524 )]);
16525 let error = incompatible
16526 .get_version("checkout-v2", 2, 3)
16527 .expect_err("old version is unsupported");
16528 assert!(matches!(
16529 error,
16530 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16531 if reason == "version_marker_incompatible_range"
16532 ));
16533
16534 for (history, reason) in [
16535 (
16536 vec![history_event("SideEffectRecorded", json!({"sequence": 1}))],
16537 "side_effect_result_missing",
16538 ),
16539 (
16540 vec![history_event(
16541 "SideEffectRecorded",
16542 json!({
16543 "sequence": 1,
16544 "result": {"codec": "avro", "blob": "not-base64"},
16545 }),
16546 )],
16547 "side_effect_payload_incompatible",
16548 ),
16549 (
16550 vec![history_event(
16551 "SideEffectRecorded",
16552 json!({"sequence": 1, "result": {"unwrapped": true}}),
16553 )],
16554 "side_effect_payload_malformed",
16555 ),
16556 (
16557 vec![history_event(
16558 "VersionMarkerRecorded",
16559 json!({
16560 "sequence": 1,
16561 "change_id": "change",
16562 "version": 1,
16563 "min_supported": 2,
16564 "max_supported": 1,
16565 }),
16566 )],
16567 "version_marker_history_range_invalid",
16568 ),
16569 ] {
16570 let error = WorkflowState::new(
16571 history,
16572 "rust-workers".to_string(),
16573 DEFAULT_CODEC.to_string(),
16574 None,
16575 )
16576 .expect_err("malformed history must fail");
16577 assert!(matches!(
16578 error,
16579 Error::NonDeterministicReplay(ReplayFailure { reason: actual, .. })
16580 if actual == reason
16581 ));
16582 }
16583 }
16584
16585 #[test]
16586 fn typed_search_attributes_replay_value_and_type_identity_after_restart() {
16587 let history = vec![history_event(
16588 "SearchAttributesUpserted",
16589 json!({
16590 "sequence": 1,
16591 "attributes": {"customer_tier": "gold"},
16592 "attribute_types": {"customer_tier": "keyword"},
16593 "merged": {"customer_tier": "gold"}
16594 }),
16595 )];
16596
16597 let matching = workflow_context(history.clone());
16598 matching
16599 .upsert_search_attributes(
16600 SearchAttributeUpdate::new()
16601 .keyword("customer_tier", "gold")
16602 .expect("keyword update"),
16603 )
16604 .expect("matching typed update must replay");
16605 matching
16606 .ensure_history_consumed()
16607 .expect("history consumed");
16608
16609 let changed_type = workflow_context(history.clone());
16610 let error = changed_type
16611 .upsert_search_attributes(
16612 SearchAttributeUpdate::new()
16613 .string("customer_tier", "gold")
16614 .expect("string update"),
16615 )
16616 .expect_err("same JSON value with a different declaration must be nondeterministic");
16617 let Error::NonDeterministicReplay(failure) = error else {
16618 panic!("typed identity drift must be a replay failure");
16619 };
16620 assert_eq!(failure.reason, "search_attribute_type_mismatch");
16621 assert_eq!(failure.sequence, Some(1));
16622
16623 let changed_value = workflow_context(history);
16624 let error = changed_value
16625 .upsert_search_attributes(
16626 SearchAttributeUpdate::new()
16627 .keyword("customer_tier", "platinum")
16628 .expect("keyword update"),
16629 )
16630 .expect_err("changed values must be nondeterministic");
16631 let Error::NonDeterministicReplay(failure) = error else {
16632 panic!("value drift must be a replay failure");
16633 };
16634 assert_eq!(failure.reason, "search_attribute_value_mismatch");
16635 }
16636
16637 #[test]
16638 fn legacy_search_attribute_history_keeps_type_identity_unknown() {
16639 let history = vec![history_event(
16640 "SearchAttributesUpserted",
16641 json!({
16642 "sequence": 1,
16643 "attributes": {"customer_tier": "gold"},
16644 "merged": {"customer_tier": "gold"}
16645 }),
16646 )];
16647
16648 for update in [
16649 SearchAttributeUpdate::new()
16650 .keyword("customer_tier", "gold")
16651 .expect("keyword update"),
16652 SearchAttributeUpdate::new()
16653 .string("customer_tier", "gold")
16654 .expect("string update"),
16655 ] {
16656 let restarted = workflow_context(history.clone());
16657 restarted
16658 .upsert_search_attributes(update)
16659 .expect("legacy history constrains values but has unknown type identity");
16660 restarted
16661 .ensure_history_consumed()
16662 .expect("history consumed");
16663 }
16664 }
16665
16666 #[test]
16667 fn search_attribute_command_emits_canonical_types() {
16668 let ctx = workflow_context(Vec::new());
16669 ctx.upsert_search_attributes(
16670 SearchAttributeUpdate::new()
16671 .keyword("customer_tier", "gold")
16672 .expect("keyword update")
16673 .int("attempts", 3)
16674 .expect("int update")
16675 .delete("obsolete")
16676 .expect("delete update"),
16677 )
16678 .expect("valid search attributes");
16679
16680 assert_eq!(
16681 ctx.take_commands().expect("commands"),
16682 vec![json!({
16683 "type": "upsert_search_attributes",
16684 "attributes": {
16685 "attempts": 3,
16686 "customer_tier": "gold",
16687 "obsolete": null
16688 },
16689 "attribute_types": {
16690 "attempts": "int",
16691 "customer_tier": "keyword"
16692 }
16693 })]
16694 );
16695 }
16696
16697 #[test]
16698 fn duplicate_side_effects_and_version_markers_are_rejected() {
16699 let duplicate_side_effect = WorkflowState::new(
16700 vec![
16701 history_event(
16702 "SideEffectRecorded",
16703 json!({"sequence": 1, "result": fixture_envelope(json!(1))}),
16704 ),
16705 history_event(
16706 "SideEffectRecorded",
16707 json!({"sequence": 1, "result": fixture_envelope(json!(2))}),
16708 ),
16709 ],
16710 "rust-workers".to_string(),
16711 DEFAULT_CODEC.to_string(),
16712 None,
16713 )
16714 .expect_err("duplicate side effect");
16715 assert!(matches!(
16716 duplicate_side_effect,
16717 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16718 if reason == "duplicate_side_effect_record"
16719 ));
16720
16721 let marker = |sequence| {
16722 history_event(
16723 "VersionMarkerRecorded",
16724 json!({
16725 "sequence": sequence,
16726 "change_id": "same-change",
16727 "version": 1,
16728 "min_supported": 1,
16729 "max_supported": 1,
16730 }),
16731 )
16732 };
16733 let duplicate_marker = WorkflowState::new(
16734 vec![marker(1), marker(3)],
16735 "rust-workers".to_string(),
16736 DEFAULT_CODEC.to_string(),
16737 None,
16738 )
16739 .expect_err("duplicate marker");
16740 assert!(matches!(
16741 duplicate_marker,
16742 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16743 if reason == "duplicate_version_marker"
16744 ));
16745 }
16746
16747 #[test]
16748 fn workflow_stream_authoring_derives_identity_and_replay_skips_duplicate_append() {
16749 let mut state = WorkflowState::new(
16750 Vec::new(),
16751 "rust-workers".to_string(),
16752 DEFAULT_CODEC.to_string(),
16753 None,
16754 )
16755 .expect("workflow state");
16756 state.workflow_command_identity = "command-7".to_string();
16757 let context = WorkflowContext {
16758 state: Arc::new(Mutex::new(state)),
16759 };
16760 let item =
16761 WorkflowStreamAppendItem::from_reference("s3://bucket/item.avro").item_type("receipt");
16762
16763 context
16764 .append_workflow_stream("output", &[item], Some(10))
16765 .expect("append command");
16766 context
16767 .error_workflow_stream("output", "producer failed", None)
16768 .expect("error command");
16769 let commands = context.take_commands().expect("commands");
16770
16771 assert_eq!(commands[0]["type"], "record_side_effect");
16772 assert_eq!(
16773 commands[0]["workflow_stream"]["command_identity"],
16774 "command-7"
16775 );
16776 assert_eq!(commands[0]["workflow_stream"]["command_ordinal"], 0);
16777 assert_eq!(
16778 commands[0]["workflow_stream"]["items"][0]["idempotency_key"],
16779 "dw-stream:command-7:0:0"
16780 );
16781 assert_eq!(commands[1]["workflow_stream"]["operation"], "error");
16782
16783 let recorded = history_event(
16784 "SideEffectRecorded",
16785 json!({"sequence": 1, "result": fixture_envelope(Value::Null)}),
16786 );
16787 let mut replay_state = WorkflowState::new(
16788 vec![recorded],
16789 "rust-workers".to_string(),
16790 DEFAULT_CODEC.to_string(),
16791 None,
16792 )
16793 .expect("replay state");
16794 replay_state.workflow_command_identity = "command-7".to_string();
16795 let replay_context = WorkflowContext {
16796 state: Arc::new(Mutex::new(replay_state)),
16797 };
16798 replay_context
16799 .append_workflow_stream(
16800 "output",
16801 &[WorkflowStreamAppendItem::from_reference(
16802 "s3://bucket/item.avro",
16803 )],
16804 Some(10),
16805 )
16806 .expect("replayed append");
16807 assert!(replay_context
16808 .take_commands()
16809 .expect("replayed commands")
16810 .is_empty());
16811 }
16812
16813 #[test]
16814 fn workflow_stream_authoring_requires_server_durable_command_identity() {
16815 let context = workflow_context(Vec::new());
16816 let error = context
16817 .append_workflow_stream(
16818 "output",
16819 &[WorkflowStreamAppendItem::from_reference(
16820 "s3://bucket/item.avro",
16821 )],
16822 None,
16823 )
16824 .expect_err("stream append without durable command identity must fail closed");
16825
16826 assert!(matches!(error, Error::MissingWorkflowCommandIdentity));
16827 assert!(context.take_commands().expect("commands").is_empty());
16828 }
16829
16830 #[test]
16831 fn cold_worker_replay_does_not_repeat_committed_side_effects_or_markers() {
16832 fn worker(calls: Arc<AtomicUsize>) -> Worker {
16833 let client = Client::new("http://127.0.0.1:8080").expect("client");
16834 let mut worker = Worker::new(client, "rust-workers");
16835 worker.register_workflow("rust.side-effect-version", move |ctx, _input| {
16836 let calls = Arc::clone(&calls);
16837 async move {
16838 let captured = ctx.side_effect(|| {
16839 calls.fetch_add(1, Ordering::SeqCst);
16840 "captured-once".to_string()
16841 })?;
16842 let version = ctx.get_version("cold-restart", 1, 2)?;
16843 Ok(json!({"captured": captured, "version": version}))
16844 }
16845 });
16846 worker
16847 }
16848
16849 fn task(history_events: Vec<HistoryEvent>) -> WorkflowTask {
16850 WorkflowTask {
16851 task_id: "wft-side-effect-version".to_string(),
16852 workflow_command_id: None,
16853 workflow_id: Some("wf-side-effect-version".to_string()),
16854 run_id: Some("run-side-effect-version".to_string()),
16855 workflow_type: "rust.side-effect-version".to_string(),
16856 cancel_requested: false,
16857 payload_codec: DEFAULT_CODEC.to_string(),
16858 arguments: Some(
16859 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("arguments"),
16860 ),
16861 history_events,
16862 total_history_events: None,
16863 history_size_bytes: None,
16864 continue_as_new_recommended: None,
16865 history_budget_pressure: None,
16866 next_history_page_token: None,
16867 workflow_task_attempt: 1,
16868 workflow_signal_id: None,
16869 signal_name: None,
16870 signal_arguments: None,
16871 workflow_update_id: None,
16872 update_name: None,
16873 lease_owner: Some("rust-worker".to_string()),
16874 }
16875 }
16876
16877 let calls = Arc::new(AtomicUsize::new(0));
16878 let initial = worker(Arc::clone(&calls))
16879 .execute_workflow_task(task(Vec::new()))
16880 .expect("initial execution");
16881 assert_eq!(
16882 initial
16883 .iter()
16884 .map(|command| &command["type"])
16885 .collect::<Vec<_>>(),
16886 vec![
16887 "record_side_effect",
16888 "record_version_marker",
16889 "complete_workflow"
16890 ]
16891 );
16892 assert_eq!(calls.load(Ordering::SeqCst), 1);
16893
16894 let restarted = worker(Arc::clone(&calls));
16895 let replayed = restarted
16896 .execute_workflow_task(task(vec![
16897 history_event(
16898 "SideEffectRecorded",
16899 json!({"sequence": 1, "result": initial[0]["result"].clone()}),
16900 ),
16901 history_event(
16902 "VersionMarkerRecorded",
16903 json!({
16904 "sequence": 2,
16905 "change_id": "cold-restart",
16906 "version": 2,
16907 "min_supported": 1,
16908 "max_supported": 2,
16909 }),
16910 ),
16911 ]))
16912 .expect("cold replay");
16913 assert_eq!(replayed.len(), 1);
16914 assert_eq!(replayed[0]["type"], "complete_workflow");
16915 assert_eq!(calls.load(Ordering::SeqCst), 1);
16916 }
16917
16918 #[test]
16919 fn side_effect_replay_rejects_changed_rust_value_type() {
16920 let result = encode_value_envelope(&json!({"value": 42}), DEFAULT_CODEC).expect("result");
16921 let ctx = workflow_context(vec![history_event(
16922 "SideEffectRecorded",
16923 json!({"sequence": 1, "result": result}),
16924 )]);
16925 let error = ctx
16926 .side_effect::<Vec<String>, _>(|| panic!("callback must not run"))
16927 .expect_err("changed type must fail replay");
16928 assert!(matches!(
16929 error,
16930 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16931 if reason == "side_effect_type_mismatch"
16932 ));
16933 }
16934
16935 fn completed_retry_activity_history() -> Vec<HistoryEvent> {
16936 vec![
16937 history_event(
16938 "ActivityScheduled",
16939 json!({
16940 "sequence": 1,
16941 "activity_type": "flaky",
16942 "activity_execution_id": "act-1",
16943 "activity": {
16944 "id": "act-1",
16945 "sequence": 1,
16946 "type": "flaky",
16947 "queue": "critical-activities",
16948 "execution_mode": null,
16949 "retry_policy": {
16950 "snapshot_version": 1,
16951 "max_attempts": 3,
16952 "backoff_seconds": [2, 4],
16953 "start_to_close_timeout": 30,
16954 "schedule_to_start_timeout": 5,
16955 "schedule_to_close_timeout": 90,
16956 "heartbeat_timeout": 10,
16957 "non_retryable_error_types": ["PermanentError"]
16958 }
16959 }
16960 }),
16961 ),
16962 history_event(
16963 "ActivityStarted",
16964 json!({
16965 "sequence": 1,
16966 "activity_type": "flaky",
16967 "activity_execution_id": "act-1",
16968 "activity_attempt_id": "attempt-1",
16969 "attempt_number": 1
16970 }),
16971 ),
16972 history_event(
16973 "ActivityRetryScheduled",
16974 json!({
16975 "sequence": 1,
16976 "activity_type": "flaky",
16977 "activity_execution_id": "act-1",
16978 "activity_attempt_id": "attempt-1",
16979 "attempt_number": 1,
16980 "retry_after_attempt": 1,
16981 "retry_backoff_seconds": 2,
16982 "failure_category": "activity",
16983 "exception_type": "TransientError"
16984 }),
16985 ),
16986 history_event(
16987 "ActivityStarted",
16988 json!({
16989 "sequence": 1,
16990 "activity_type": "flaky",
16991 "activity_execution_id": "act-1",
16992 "activity_attempt_id": "attempt-2",
16993 "attempt_number": 2
16994 }),
16995 ),
16996 history_event(
16997 "ActivityCompleted",
16998 json!({
16999 "sequence": 1,
17000 "activity_type": "flaky",
17001 "activity_execution_id": "act-1",
17002 "activity_attempt_id": "attempt-2",
17003 "attempt_number": 2,
17004 "payload_codec": DEFAULT_CODEC,
17005 "result": fixture_envelope(json!({"status":"recovered"}))
17006 }),
17007 ),
17008 ]
17009 }
17010
17011 fn retry_activity_options() -> ActivityOptions {
17012 ActivityOptions::new()
17013 .task_queue("critical-activities")
17014 .retry_policy(
17015 ActivityRetryPolicy::new(3)
17016 .backoff_intervals([Duration::from_secs(2), Duration::from_secs(4)])
17017 .non_retryable_error_type("PermanentError"),
17018 )
17019 .start_to_close_timeout(Duration::from_secs(30))
17020 .schedule_to_start_timeout(Duration::from_secs(5))
17021 .schedule_to_close_timeout(Duration::from_secs(90))
17022 .heartbeat_timeout(Duration::from_secs(10))
17023 }
17024
17025 #[test]
17026 fn fixed_avro_value_round_trips_json_values() {
17027 let value = json!({"greeting": "hello", "count": 3, "ok": true});
17028 let envelope = PayloadEnvelope::avro(&value).expect("encode");
17029 assert_eq!(envelope.codec, DEFAULT_CODEC);
17030 assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
17031 }
17032
17033 #[tokio::test]
17034 async fn typed_handler_adapters_round_trip_serde_contracts_on_the_fixed_wire() {
17035 let client = Client::new("http://127.0.0.1:8080").expect("client");
17036 let mut worker = Worker::new(client, "rust-workers");
17037 worker.register_typed_workflow(
17038 "typed.contract.workflow",
17039 |_ctx, input: TypedContract| async move { Ok(input) },
17040 );
17041 worker.register_typed_activity(
17042 "typed.contract.activity",
17043 |_ctx, input: TypedContract| async move { Ok(input) },
17044 );
17045
17046 let expected = typed_contract();
17047 let arguments = AvroValue::Array(vec![
17048 AvroValue::from_serialize(&expected).expect("typed request")
17049 ]);
17050 let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("arguments");
17051 let mut workflow = workflow_task("typed.contract.workflow", Vec::new(), DEFAULT_CODEC);
17052 workflow.arguments = Some(envelope.clone());
17053 let commands = worker
17054 .execute_workflow_task(workflow)
17055 .expect("typed workflow task");
17056 let workflow_result: TypedContract =
17057 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17058 .expect("workflow result envelope")
17059 .deserialize()
17060 .expect("workflow result type");
17061 assert_eq!(workflow_result, expected);
17062
17063 let activity = ActivityTask {
17064 task_id: "typed-contract-activity".to_string(),
17065 activity_attempt_id: Some("typed-contract-attempt".to_string()),
17066 attempt_id: None,
17067 activity_type: "typed.contract.activity".to_string(),
17068 payload_codec: DEFAULT_CODEC.to_string(),
17069 arguments: Some(envelope),
17070 attempt_number: 1,
17071 lease_owner: Some("rust-worker".to_string()),
17072 };
17073 let activity_result: TypedContract = worker
17074 .execute_activity_task(activity)
17075 .await
17076 .expect("typed activity task")
17077 .deserialize()
17078 .expect("activity result type");
17079 assert_eq!(activity_result, expected);
17080 }
17081
17082 #[tokio::test]
17083 async fn typed_handler_errors_include_handler_name_direction_and_rust_type() {
17084 let client = Client::new("http://127.0.0.1:8080").expect("client");
17085 let mut worker = Worker::new(client, "rust-workers");
17086 worker.register_typed_workflow(
17087 "typed.shape.workflow",
17088 |_ctx, input: TypedContract| async move { Ok(input) },
17089 );
17090 worker.register_typed_activity("typed.unsupported.activity", |_ctx, (): ()| async move {
17091 Ok(f64::NAN)
17092 });
17093
17094 let mut workflow = workflow_task("typed.shape.workflow", Vec::new(), DEFAULT_CODEC);
17095 workflow.arguments = Some(
17096 encode_typed_envelope(
17097 &AvroValue::Array(vec![
17098 AvroValue::String("first".to_string()),
17099 AvroValue::String("second".to_string()),
17100 ]),
17101 DEFAULT_CODEC,
17102 )
17103 .expect("malformed typed arguments"),
17104 );
17105 let commands = worker
17106 .execute_workflow_task(workflow)
17107 .expect("shape mismatch becomes a workflow failure");
17108 let message = commands[0]["message"].as_str().expect("failure message");
17109 assert!(message.contains("workflow handler \"typed.shape.workflow\" input type"));
17110 assert!(message.contains(type_name::<TypedContract>()));
17111 assert!(message.contains("task carried 2 arguments"));
17112
17113 let activity = ActivityTask {
17114 task_id: "typed-unsupported-activity".to_string(),
17115 activity_attempt_id: Some("typed-unsupported-attempt".to_string()),
17116 attempt_id: None,
17117 activity_type: "typed.unsupported.activity".to_string(),
17118 payload_codec: DEFAULT_CODEC.to_string(),
17119 arguments: Some(
17120 encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
17121 .expect("unit arguments"),
17122 ),
17123 attempt_number: 1,
17124 lease_owner: Some("rust-worker".to_string()),
17125 };
17126 let Error::HandlerType {
17127 handler_kind,
17128 handler_name,
17129 value_kind,
17130 rust_type,
17131 message,
17132 } = worker
17133 .execute_activity_task(activity)
17134 .await
17135 .expect_err("non-finite handler output must fail")
17136 else {
17137 panic!("expected contextual handler type failure");
17138 };
17139 assert_eq!(handler_kind, HandlerKind::Activity);
17140 assert_eq!(handler_name, "typed.unsupported.activity");
17141 assert_eq!(value_kind, HandlerValueKind::Result);
17142 assert_eq!(rust_type, type_name::<f64>());
17143 assert!(message.contains("non_finite_float"));
17144 }
17145
17146 #[tokio::test]
17147 async fn typed_replayed_workflow_decodes_input_and_activity_result_losslessly() {
17148 #[derive(Clone, Default)]
17149 struct State {
17150 observed: Option<TypedContract>,
17151 }
17152
17153 let client = Client::new("http://127.0.0.1:8080").expect("client");
17154 let mut worker = Worker::new(client, "rust-workers");
17155 worker.register_typed_replayed_workflow(
17156 "typed.contract.replayed",
17157 State::default,
17158 |ctx, input: TypedContract, state| async move {
17159 let result: TypedContract =
17160 ctx.activity_typed("typed.contract.activity", input).await?;
17161 state.update(|current| current.observed = Some(result.clone()))?;
17162 Ok(result)
17163 },
17164 );
17165 worker.register_replayed_query::<State, _, _>(
17166 "typed.contract.replayed",
17167 "observed",
17168 |_ctx, state, _args| async move {
17169 Ok(json!(state.observed.as_ref().map(|value| value.signed)))
17170 },
17171 );
17172
17173 let expected = typed_contract();
17174 let typed_value = AvroValue::from_serialize(&expected).expect("typed value");
17175 let workflow_arguments =
17176 encode_typed_envelope(&AvroValue::Array(vec![typed_value.clone()]), DEFAULT_CODEC)
17177 .expect("workflow arguments");
17178 let result = encode_typed_envelope(&typed_value, DEFAULT_CODEC).expect("activity result");
17179 let task = QueryTask {
17180 query_task_id: "typed-replay-query".to_string(),
17181 query_task_attempt: 1,
17182 lease_owner: Some("rust-worker".to_string()),
17183 workflow_id: Some("typed-replay".to_string()),
17184 run_id: Some("typed-replay-run".to_string()),
17185 workflow_type: "typed.contract.replayed".to_string(),
17186 query_name: "observed".to_string(),
17187 payload_codec: DEFAULT_CODEC.to_string(),
17188 workflow_arguments: Some(workflow_arguments),
17189 query_arguments: Some(
17190 encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
17191 .expect("query arguments"),
17192 ),
17193 history_events: vec![
17194 history_event(
17195 "ActivityScheduled",
17196 json!({
17197 "sequence": 1,
17198 "activity_type": "typed.contract.activity"
17199 }),
17200 ),
17201 history_event(
17202 "ActivityCompleted",
17203 json!({
17204 "sequence": 1,
17205 "activity_type": "typed.contract.activity",
17206 "payload_codec": DEFAULT_CODEC,
17207 "result": result
17208 }),
17209 ),
17210 ],
17211 history_export: None,
17212 run_status: Some("completed".to_string()),
17213 };
17214
17215 assert_eq!(
17216 worker
17217 .execute_query_task(task)
17218 .await
17219 .expect("typed replay query")
17220 .deserialize::<i64>()
17221 .expect("query result"),
17222 expected.signed
17223 );
17224 }
17225
17226 #[tokio::test]
17227 async fn typed_worker_surfaces_preserve_bytes_and_map_list_identity() {
17228 let client = Client::new("http://127.0.0.1:8080").expect("client");
17229 let mut worker = Worker::new(client, "rust-workers");
17230 worker.register_workflow_avro_value("typed.echo", |_ctx, input| async move { Ok(input) });
17231 worker
17232 .register_activity_avro_value("typed.activity", |_ctx, input| async move { Ok(input) });
17233 worker.register_query_avro_value("typed.echo", "inspect", |_ctx, input| async move {
17234 Ok(input)
17235 });
17236 worker.register_update_avro_value("typed.echo", "replace", |_ctx, input| async move {
17237 Ok(input)
17238 });
17239 worker.register_workflow_avro_value("typed.signal", |ctx, _input| async move {
17240 Ok(AvroValue::Array(
17241 ctx.wait_signal_avro_value("changed").await?,
17242 ))
17243 });
17244
17245 let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
17246 let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed envelope");
17247
17248 let mut workflow = workflow_task("typed.echo", Vec::new(), DEFAULT_CODEC);
17249 workflow.arguments = Some(envelope.clone());
17250 let commands = worker
17251 .execute_workflow_task(workflow)
17252 .expect("typed workflow task");
17253 assert_eq!(commands[0]["type"], "complete_workflow");
17254 assert_eq!(
17255 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17256 .expect("typed workflow result"),
17257 arguments
17258 );
17259
17260 let activity = ActivityTask {
17261 task_id: "activity-typed".to_string(),
17262 activity_attempt_id: Some("attempt-typed".to_string()),
17263 attempt_id: None,
17264 activity_type: "typed.activity".to_string(),
17265 payload_codec: DEFAULT_CODEC.to_string(),
17266 arguments: Some(envelope.clone()),
17267 attempt_number: 1,
17268 lease_owner: Some("rust-worker".to_string()),
17269 };
17270 assert_eq!(
17271 worker
17272 .execute_activity_task(activity)
17273 .await
17274 .expect("typed activity result"),
17275 arguments
17276 );
17277
17278 let query = QueryTask {
17279 query_task_id: "query-typed".to_string(),
17280 query_task_attempt: 1,
17281 lease_owner: Some("rust-worker".to_string()),
17282 workflow_id: Some("typed-1".to_string()),
17283 run_id: Some("run-typed".to_string()),
17284 workflow_type: "typed.echo".to_string(),
17285 query_name: "inspect".to_string(),
17286 payload_codec: DEFAULT_CODEC.to_string(),
17287 workflow_arguments: Some(
17288 encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
17289 .expect("workflow input"),
17290 ),
17291 query_arguments: Some(envelope.clone()),
17292 history_events: Vec::new(),
17293 history_export: None,
17294 run_status: Some("running".to_string()),
17295 };
17296 assert_eq!(
17297 worker
17298 .execute_query_task(query)
17299 .await
17300 .expect("typed query result"),
17301 arguments
17302 );
17303
17304 let mut update = workflow_task(
17305 "typed.echo",
17306 vec![history_event(
17307 "UpdateAccepted",
17308 json!({
17309 "update_id": "update-typed",
17310 "update_name": "replace",
17311 "arguments": envelope.clone(),
17312 }),
17313 )],
17314 DEFAULT_CODEC,
17315 );
17316 update.workflow_update_id = Some("update-typed".to_string());
17317 update.update_name = Some("replace".to_string());
17318 let commands = worker
17319 .execute_workflow_task(update)
17320 .expect("typed update task");
17321 assert_eq!(commands[0]["type"], "complete_update");
17322 assert_eq!(
17323 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17324 .expect("typed update result"),
17325 arguments
17326 );
17327
17328 let mut signal = workflow_task(
17329 "typed.signal",
17330 vec![history_event(
17331 "SignalReceived",
17332 json!({
17333 "signal_id": "signal-typed",
17334 "signal_name": "changed",
17335 "arguments": envelope.clone(),
17336 }),
17337 )],
17338 DEFAULT_CODEC,
17339 );
17340 signal.workflow_signal_id = Some("signal-typed".to_string());
17341 signal.signal_name = Some("changed".to_string());
17342 signal.signal_arguments = Some(envelope);
17343 let commands = worker
17344 .execute_workflow_task(signal)
17345 .expect("typed signal resume");
17346 assert_eq!(
17347 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17348 .expect("typed signal result"),
17349 arguments
17350 );
17351 }
17352
17353 #[tokio::test]
17354 async fn typed_helpers_never_parse_json_inspection_projection() {
17355 let collision_values = projection_collision_probe();
17356 let expected = AvroValue::Array(collision_values.clone());
17357 let envelope = encode_typed_envelope(&expected, DEFAULT_CODEC).expect("collision envelope");
17358
17359 let activity_context = workflow_context_with_codec(
17360 vec![history_event(
17361 "ActivityCompleted",
17362 json!({
17363 "sequence": 1,
17364 "activity_type": "collision.activity",
17365 "payload_codec": DEFAULT_CODEC,
17366 "result": envelope.clone(),
17367 }),
17368 )],
17369 DEFAULT_CODEC,
17370 );
17371 assert_eq!(
17372 activity_context
17373 .activity_avro_value("collision.activity", AvroValue::Array(Vec::new()))
17374 .await
17375 .expect("typed activity collision result"),
17376 expected
17377 );
17378
17379 let signal_context = workflow_context_with_codec(
17380 vec![
17381 history_event(
17382 "SignalWaitOpened",
17383 json!({"sequence": 1, "signal_name": "collision"}),
17384 ),
17385 history_event(
17386 "SignalApplied",
17387 json!({
17388 "sequence": 1,
17389 "signal_name": "collision",
17390 "payload_codec": DEFAULT_CODEC,
17391 "value": envelope.clone(),
17392 }),
17393 ),
17394 ],
17395 DEFAULT_CODEC,
17396 );
17397 assert_eq!(
17398 signal_context
17399 .wait_signal_avro_value("collision")
17400 .await
17401 .expect("typed signal collision arguments"),
17402 collision_values
17403 );
17404
17405 let child_context = workflow_context_with_codec(
17406 vec![
17407 history_event(
17408 "ChildWorkflowScheduled",
17409 json!({
17410 "sequence": 1,
17411 "child_workflow_instance_id": "collision-child",
17412 "child_workflow_run_id": "collision-run",
17413 "child_workflow_type": "collision.child",
17414 }),
17415 ),
17416 history_event(
17417 "ChildRunCompleted",
17418 json!({
17419 "sequence": 1,
17420 "child_workflow_instance_id": "collision-child",
17421 "child_workflow_run_id": "collision-run",
17422 "child_workflow_type": "collision.child",
17423 "payload_codec": DEFAULT_CODEC,
17424 "result": envelope,
17425 }),
17426 ),
17427 ],
17428 DEFAULT_CODEC,
17429 );
17430 let child = child_context
17431 .start_child_workflow_avro_value(
17432 "collision.child",
17433 ChildWorkflowOptions::new("collision-workers"),
17434 AvroValue::Array(Vec::new()),
17435 )
17436 .await
17437 .expect("typed child collision result");
17438 assert_eq!(child.result, expected);
17439 }
17440
17441 #[tokio::test]
17442 async fn replayed_typed_query_keeps_lossless_workflow_and_query_inputs() {
17443 let client = Client::new("http://127.0.0.1:8080").expect("client");
17444 let mut worker = Worker::new(client, "rust-workers");
17445 worker.register_replayed_workflow_avro_value(
17446 "typed.replayed",
17447 || (),
17448 |_ctx, input, _state| async move { Ok(input) },
17449 );
17450 worker.register_replayed_query_avro_value::<(), _, _>(
17451 "typed.replayed",
17452 "inspect",
17453 |ctx, _state, args| async move {
17454 let mut signals = ctx.signals_avro_value("collision");
17455 let signal = signals
17456 .pop()
17457 .map(AvroValue::Array)
17458 .unwrap_or_else(|| AvroValue::Array(Vec::new()));
17459 Ok(AvroValue::Array(vec![
17460 ctx.workflow_input_avro_value().clone(),
17461 signal,
17462 args,
17463 ]))
17464 },
17465 );
17466 let arguments = AvroValue::Array(projection_collision_probe());
17467 let signal_arguments =
17468 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed query signal arguments");
17469 let task = QueryTask {
17470 query_task_id: "query-typed-replay".to_string(),
17471 query_task_attempt: 1,
17472 lease_owner: Some("rust-worker".to_string()),
17473 workflow_id: Some("typed-replay".to_string()),
17474 run_id: Some("run-typed-replay".to_string()),
17475 workflow_type: "typed.replayed".to_string(),
17476 query_name: "inspect".to_string(),
17477 payload_codec: DEFAULT_CODEC.to_string(),
17478 workflow_arguments: Some(
17479 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("workflow arguments"),
17480 ),
17481 query_arguments: Some(
17482 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("query arguments"),
17483 ),
17484 history_events: vec![history_event(
17485 "SignalReceived",
17486 json!({
17487 "signal_id": "collision-signal",
17488 "signal_name": "collision",
17489 "workflow_sequence": 1,
17490 "payload_codec": DEFAULT_CODEC,
17491 "arguments": signal_arguments,
17492 }),
17493 )],
17494 history_export: None,
17495 run_status: Some("completed".to_string()),
17496 };
17497
17498 assert_eq!(
17499 worker
17500 .execute_query_task(task)
17501 .await
17502 .expect("typed replay query"),
17503 AvroValue::Array(vec![arguments.clone(), arguments.clone(), arguments])
17504 );
17505 }
17506
17507 #[test]
17508 fn public_avro_adapter_rejects_non_string_map_keys_before_json_conversion() {
17509 let value = BTreeMap::from([(1_i32, "integer key")]);
17510 let error = PayloadEnvelope::avro(&value)
17511 .expect_err("integer map keys must fail")
17512 .to_string();
17513
17514 assert!(error.contains("invalid_map_key"));
17515 }
17516
17517 #[test]
17518 fn json_tagged_payload_fails_closed_with_actionable_diagnostic() {
17519 let envelope = PayloadEnvelope {
17520 codec: "json".to_string(),
17521 blob: r#"{"greeting":"hello"}"#.to_string(),
17522 };
17523
17524 let error = decode_payload::<Value>(&envelope).expect_err("JSON payload must fail");
17525 let diagnostic = error.to_string();
17526 assert!(diagnostic.contains("unsupported_payload_codec"));
17527 assert!(diagnostic.contains("codec=\"avro\""));
17528 assert!(diagnostic.contains("HTTP document transport"));
17529 }
17530
17531 #[test]
17532 fn untagged_json_payload_value_fails_closed() {
17533 let error = decode_wire_value(&json!({"stale": true}), DEFAULT_CODEC)
17534 .expect_err("untagged JSON payload values must fail");
17535 let diagnostic = error.to_string();
17536 assert!(diagnostic.contains("unsupported_payload_codec"));
17537 assert!(diagnostic.contains("untagged durable payload"));
17538 assert!(diagnostic.contains("HTTP document transport"));
17539 }
17540
17541 #[test]
17542 fn prerelease_avro_payload_without_single_object_frame_is_rejected() {
17543 let envelope = PayloadEnvelope {
17544 codec: DEFAULT_CODEC.to_string(),
17545 blob: BASE64.encode([0x01]),
17546 };
17547
17548 let error = decode_payload::<Value>(&envelope).expect_err("prerelease payload must fail");
17549 assert!(error.to_string().contains("invalid_payload_framing"));
17550 }
17551
17552 #[tokio::test]
17553 async fn workflow_completion_rejects_invalid_payload_slots_without_transport() {
17554 let server = MockWorkerServer::start();
17555 let client = Client::builder(server.base_url())
17556 .timeout(Duration::from_secs(2))
17557 .build()
17558 .expect("client");
17559 let invalid_commands = [
17560 json!({
17561 "type": "complete_workflow",
17562 "result": {"codec": "json", "blob": null}
17563 }),
17564 json!({
17565 "type": "schedule_activity",
17566 "arguments": {"codec": "yaml", "blob": "ignored"}
17567 }),
17568 json!({
17569 "type": "start_child_workflow",
17570 "arguments": {"codec": DEFAULT_CODEC, "blob": null}
17571 }),
17572 json!({"type": "continue_as_new", "arguments": []}),
17573 json!({"type": "complete_update"}),
17574 json!({"type": "record_side_effect", "result": null}),
17575 json!({
17576 "type": "start_service_operation",
17577 "payload_codec": DEFAULT_CODEC,
17578 "request_payload": "raw-avro-bytes"
17579 }),
17580 ];
17581
17582 for command in invalid_commands {
17583 let error = client
17584 .complete_workflow_task("invalid-codec", "rust-worker", 1, vec![command])
17585 .await
17586 .expect_err("invalid durable payload must fail locally");
17587 let diagnostic = error.to_string();
17588 assert!(
17589 diagnostic.contains("unsupported_payload_codec")
17590 || diagnostic.contains("invalid_payload_envelope")
17591 || diagnostic.contains("untagged durable payload"),
17592 "unexpected validation diagnostic: {diagnostic}"
17593 );
17594 }
17595
17596 assert_eq!(
17597 server.request_count("/api/worker/workflow-tasks/invalid-codec/complete"),
17598 0,
17599 "invalid command payloads must not reach HTTP transport"
17600 );
17601 }
17602
17603 #[test]
17604 fn workflow_completion_validates_only_protocol_owned_payload_slots() {
17605 let envelope = fixture_envelope(json!({"codec": "customer-value"}));
17606 let commands = [
17607 json!({"type": "complete_workflow", "result": envelope.clone()}),
17608 json!({"type": "schedule_activity", "arguments": envelope.clone()}),
17609 json!({"type": "start_child_workflow", "arguments": envelope.clone()}),
17610 json!({"type": "continue_as_new", "arguments": envelope.clone()}),
17611 json!({"type": "complete_update", "result": envelope.clone()}),
17612 json!({"type": "record_side_effect", "result": envelope.clone()}),
17613 json!({
17614 "type": "start_service_operation",
17615 "payload_codec": DEFAULT_CODEC,
17616 "request_payload": envelope.clone()
17617 }),
17618 json!({
17619 "type": "complete_workflow",
17620 "result": envelope,
17621 "metadata": {
17622 "codec": "json",
17623 "payload_codec": "customer-codec",
17624 "result": {"codec": "yaml", "blob": null}
17625 }
17626 }),
17627 ];
17628
17629 validate_workflow_task_commands(&commands)
17630 .expect("customer metadata must not become a protocol codec declaration");
17631 }
17632
17633 #[test]
17634 fn valid_avro_tasks_normalize_absent_and_null_arguments_to_empty_lists() {
17635 assert_eq!(
17636 decode_task_avro_arguments(None, DEFAULT_CODEC).expect("absent arguments"),
17637 AvroValue::Array(Vec::new())
17638 );
17639 assert_eq!(
17640 decode_task_avro_arguments(Some(&Value::Null), DEFAULT_CODEC).expect("null arguments"),
17641 AvroValue::Array(Vec::new())
17642 );
17643
17644 let mut signal = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
17645 signal.signal_name = Some("empty-signal".to_string());
17646 signal.signal_arguments = None;
17647 let decoded = decode_resume_signal(&signal)
17648 .expect("valid Avro signal")
17649 .expect("named signal resumes the workflow");
17650 assert!(decoded.arguments.is_empty());
17651 }
17652
17653 #[tokio::test]
17654 async fn malformed_task_level_codecs_become_pre_handler_failures() {
17655 let client = Client::new("http://127.0.0.1:8080").expect("client");
17656 let mut worker = Worker::new(client, "rust-workers");
17657 let handler_calls = Arc::new(AtomicUsize::new(0));
17658
17659 let calls = Arc::clone(&handler_calls);
17660 worker.register_workflow("codec.workflow", move |_ctx, _args| {
17661 calls.fetch_add(1, Ordering::SeqCst);
17662 async move { Ok(Value::Null) }
17663 });
17664 let calls = Arc::clone(&handler_calls);
17665 worker.register_activity("codec.activity", move |_ctx, _args| {
17666 calls.fetch_add(1, Ordering::SeqCst);
17667 async move { Ok(Value::Null) }
17668 });
17669 let calls = Arc::clone(&handler_calls);
17670 worker.register_query("codec.workflow", "known", move |_ctx, _args| {
17671 calls.fetch_add(1, Ordering::SeqCst);
17672 async move { Ok(Value::Null) }
17673 });
17674
17675 let mut failures = Vec::new();
17676 for codec_case in [
17677 InvalidTaskPayloadCodec::Missing,
17678 InvalidTaskPayloadCodec::Null,
17679 InvalidTaskPayloadCodec::NonString,
17680 ] {
17681 let mut workflow = json!({
17682 "task_id": format!("workflow-{}", codec_case.label()),
17683 "workflow_type": "codec.workflow"
17684 });
17685 codec_case.apply(&mut workflow);
17686 match serde_json::from_value::<WorkflowTask>(workflow) {
17687 Ok(task) => match worker.execute_workflow_task(task) {
17688 Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
17689 outcome => failures.push(format!(
17690 "workflow {} codec returned {outcome:?}",
17691 codec_case.label()
17692 )),
17693 },
17694 Err(error) => failures.push(format!(
17695 "workflow {} codec failed transport deserialization: {error}",
17696 codec_case.label()
17697 )),
17698 }
17699
17700 let mut activity = json!({
17701 "task_id": format!("activity-{}", codec_case.label()),
17702 "activity_attempt_id": format!("attempt-{}", codec_case.label()),
17703 "activity_type": "codec.activity",
17704 "attempt_number": 1
17705 });
17706 codec_case.apply(&mut activity);
17707 match serde_json::from_value::<ActivityTask>(activity) {
17708 Ok(task) => match worker.execute_activity_task(task).await {
17709 Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
17710 outcome => failures.push(format!(
17711 "activity {} codec returned {outcome:?}",
17712 codec_case.label()
17713 )),
17714 },
17715 Err(error) => failures.push(format!(
17716 "activity {} codec failed transport deserialization: {error}",
17717 codec_case.label()
17718 )),
17719 }
17720
17721 let mut query = json!({
17722 "query_task_id": format!("query-{}", codec_case.label()),
17723 "workflow_type": "codec.workflow",
17724 "query_name": "known"
17725 });
17726 codec_case.apply(&mut query);
17727 match serde_json::from_value::<QueryTask>(query) {
17728 Ok(task) => match worker.execute_query_task(task).await {
17729 Err(failure) if failure.message.contains("unsupported_payload_codec") => {}
17730 outcome => failures.push(format!(
17731 "query {} codec returned {outcome:?}",
17732 codec_case.label()
17733 )),
17734 },
17735 Err(error) => failures.push(format!(
17736 "query {} codec failed transport deserialization: {error}",
17737 codec_case.label()
17738 )),
17739 }
17740 }
17741
17742 assert!(failures.is_empty(), "{}", failures.join("\n"));
17743 assert_eq!(
17744 handler_calls.load(Ordering::SeqCst),
17745 0,
17746 "invalid task codecs must not invoke a handler"
17747 );
17748 }
17749
17750 #[tokio::test]
17751 async fn polled_malformed_task_codecs_are_settled_without_handler_execution() {
17752 for codec_case in [
17753 InvalidTaskPayloadCodec::Missing,
17754 InvalidTaskPayloadCodec::Null,
17755 InvalidTaskPayloadCodec::NonString,
17756 ] {
17757 let server = MockWorkerServer::invalid_task_payload_codec(codec_case);
17758 let client = Client::builder(server.base_url())
17759 .timeout(Duration::from_secs(2))
17760 .build()
17761 .expect("client");
17762 let mut worker = Worker::new(client, "rust-workers")
17763 .worker_id("codec-worker")
17764 .poll_timeout(Duration::from_millis(10));
17765 let handler_calls = Arc::new(AtomicUsize::new(0));
17766
17767 let calls = Arc::clone(&handler_calls);
17768 worker.register_workflow("codec.workflow", move |_ctx, _args| {
17769 calls.fetch_add(1, Ordering::SeqCst);
17770 async move { Ok(Value::Null) }
17771 });
17772 let calls = Arc::clone(&handler_calls);
17773 worker.register_activity("codec.activity", move |_ctx, _args| {
17774 calls.fetch_add(1, Ordering::SeqCst);
17775 async move { Ok(Value::Null) }
17776 });
17777 let calls = Arc::clone(&handler_calls);
17778 worker.register_query("codec.workflow", "known", move |_ctx, _args| {
17779 calls.fetch_add(1, Ordering::SeqCst);
17780 async move { Ok(Value::Null) }
17781 });
17782
17783 assert_eq!(
17784 worker.run_once().await.expect("invalid tasks are settled"),
17785 3,
17786 "all {} codec tasks must be handled",
17787 codec_case.label()
17788 );
17789 assert_eq!(
17790 handler_calls.load(Ordering::SeqCst),
17791 0,
17792 "{} task codecs must fail before every handler",
17793 codec_case.label()
17794 );
17795
17796 for path in [
17797 "/api/worker/workflow-tasks/codec-workflow/fail",
17798 "/api/worker/activity-tasks/codec-activity/fail",
17799 "/api/worker/query-tasks/codec-query/fail",
17800 ] {
17801 let body = server.request_body(path);
17802 assert!(
17803 body["failure"]["message"]
17804 .as_str()
17805 .is_some_and(|message| message.contains("unsupported_payload_codec")),
17806 "{path} must receive the stable codec diagnostic for the {} case: {body}",
17807 codec_case.label()
17808 );
17809 }
17810 assert_eq!(
17811 server.request_body("/api/worker/query-tasks/codec-query/fail")["failure"]
17812 ["reason"],
17813 "query_payload_decode_failed"
17814 );
17815 for path in [
17816 "/api/worker/workflow-tasks/codec-workflow/complete",
17817 "/api/worker/activity-tasks/codec-activity/complete",
17818 "/api/worker/query-tasks/codec-query/complete",
17819 ] {
17820 assert_eq!(
17821 server.request_count(path),
17822 0,
17823 "invalid {} codec task reached {path}",
17824 codec_case.label()
17825 );
17826 }
17827 }
17828 }
17829
17830 #[tokio::test]
17831 async fn invalid_inbound_codecs_precede_handlers_and_unrelated_outcomes() {
17832 let client = Client::new("http://127.0.0.1:8080").expect("client");
17833 let mut worker = Worker::new(client, "rust-workers");
17834 let handler_calls = Arc::new(AtomicUsize::new(0));
17835
17836 let calls = Arc::clone(&handler_calls);
17837 worker.register_workflow("codec.workflow", move |_ctx, _args| {
17838 calls.fetch_add(1, Ordering::SeqCst);
17839 async move { Ok(Value::Null) }
17840 });
17841 let calls = Arc::clone(&handler_calls);
17842 worker.register_activity("codec.activity", move |_ctx, _args| {
17843 calls.fetch_add(1, Ordering::SeqCst);
17844 async move { Ok(Value::Null) }
17845 });
17846 let calls = Arc::clone(&handler_calls);
17847 worker.register_update("codec.workflow", "known", move |_ctx, _args| {
17848 calls.fetch_add(1, Ordering::SeqCst);
17849 async move { Ok(Value::Null) }
17850 });
17851 let calls = Arc::clone(&handler_calls);
17852 worker.register_query("codec.workflow", "known", move |_ctx, _args| {
17853 calls.fetch_add(1, Ordering::SeqCst);
17854 async move { Ok(Value::Null) }
17855 });
17856
17857 let mut workflow = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
17858 workflow.payload_codec = "json".to_string();
17859 workflow.arguments = None;
17860 let error = worker
17861 .execute_workflow_task(workflow)
17862 .expect_err("task codec must be checked before workflow invocation");
17863 assert!(error.to_string().contains("unsupported_payload_codec"));
17864
17865 let activity = ActivityTask {
17866 task_id: "activity-invalid-codec".to_string(),
17867 activity_attempt_id: None,
17868 attempt_id: None,
17869 activity_type: "codec.activity".to_string(),
17870 payload_codec: "unknown".to_string(),
17871 arguments: None,
17872 attempt_number: 1,
17873 lease_owner: None,
17874 };
17875 let error = worker
17876 .execute_activity_task(activity)
17877 .await
17878 .expect_err("task codec must be checked before activity invocation");
17879 assert!(error.to_string().contains("unsupported_payload_codec"));
17880
17881 let mut update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
17882 update.workflow_update_id = Some("update-invalid-codec".to_string());
17883 update.update_name = Some("known".to_string());
17884 update.history_events.push(history_event(
17885 "UpdateAccepted",
17886 json!({
17887 "update_id": "update-invalid-codec",
17888 "update_name": "known",
17889 "arguments": {"codec": "json", "blob": null}
17890 }),
17891 ));
17892 let error = worker
17893 .execute_workflow_task(update)
17894 .expect_err("nested update codec must be checked before handler lookup");
17895 assert!(error.to_string().contains("unsupported_payload_codec"));
17896
17897 let query: QueryTask = serde_json::from_value(json!({
17898 "query_task_id": "query-invalid-codec",
17899 "workflow_type": "codec.workflow",
17900 "query_name": "known",
17901 "payload_codec": DEFAULT_CODEC,
17902 "workflow_arguments": null,
17903 "query_arguments": null,
17904 "history_export": {
17905 "payloads": {"codec": DEFAULT_CODEC},
17906 "signals": [{
17907 "name": "empty",
17908 "payload_codec": "json",
17909 "arguments": null
17910 }]
17911 }
17912 }))
17913 .expect("query task");
17914 let failure = worker
17915 .execute_query_task(query)
17916 .await
17917 .expect_err("exported signal codec must be checked before query invocation");
17918 assert_eq!(failure.reason, "query_payload_decode_failed");
17919 assert!(failure.message.contains("unsupported_payload_codec"));
17920
17921 let exported_history: QueryTask = serde_json::from_value(json!({
17922 "query_task_id": "query-invalid-history-codec",
17923 "workflow_type": "codec.workflow",
17924 "query_name": "known",
17925 "payload_codec": DEFAULT_CODEC,
17926 "history_export": {
17927 "payloads": {"codec": DEFAULT_CODEC},
17928 "history_events": [{
17929 "type": "ActivityCompleted",
17930 "payload": {"payload_codec": "unknown", "result": null}
17931 }]
17932 }
17933 }))
17934 .expect("query task");
17935 let failure = worker
17936 .execute_query_task(exported_history)
17937 .await
17938 .expect_err("exported history codec must be checked before query invocation");
17939 assert_eq!(failure.reason, "query_payload_decode_failed");
17940 assert!(failure.message.contains("unsupported_payload_codec"));
17941 assert_eq!(handler_calls.load(Ordering::SeqCst), 0);
17942
17943 let mut unknown_workflow = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
17944 unknown_workflow.arguments = None;
17945 unknown_workflow.history_events.push(history_event(
17946 "SignalReceived",
17947 json!({
17948 "signal_name": "empty",
17949 "payload_codec": "json",
17950 "arguments": null
17951 }),
17952 ));
17953 let error = worker
17954 .execute_workflow_task(unknown_workflow)
17955 .expect_err("history codec must precede unknown workflow outcome");
17956 assert!(error.to_string().contains("unsupported_payload_codec"));
17957
17958 let unknown_activity = ActivityTask {
17959 task_id: "activity-unknown".to_string(),
17960 activity_attempt_id: None,
17961 attempt_id: None,
17962 activity_type: "missing".to_string(),
17963 payload_codec: "json".to_string(),
17964 arguments: None,
17965 attempt_number: 1,
17966 lease_owner: None,
17967 };
17968 let error = worker
17969 .execute_activity_task(unknown_activity)
17970 .await
17971 .expect_err("codec must precede unknown activity outcome");
17972 assert!(error.to_string().contains("unsupported_payload_codec"));
17973
17974 let mut unknown_update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
17975 unknown_update.payload_codec = "json".to_string();
17976 unknown_update.arguments = None;
17977 unknown_update.workflow_update_id = Some("update-unknown".to_string());
17978 unknown_update.update_name = Some("missing".to_string());
17979 let error = worker
17980 .execute_workflow_task(unknown_update)
17981 .expect_err("codec must precede fail_update shortcut");
17982 assert!(error.to_string().contains("unsupported_payload_codec"));
17983
17984 let unknown_query: QueryTask = serde_json::from_value(json!({
17985 "query_task_id": "query-unknown",
17986 "workflow_type": "missing",
17987 "query_name": "missing",
17988 "payload_codec": "json",
17989 "workflow_arguments": null,
17990 "query_arguments": null
17991 }))
17992 .expect("query task");
17993 let failure = worker
17994 .execute_query_task(unknown_query)
17995 .await
17996 .expect_err("codec must precede unknown query outcome");
17997 assert_eq!(failure.reason, "query_payload_decode_failed");
17998 assert!(failure.message.contains("unsupported_payload_codec"));
17999 }
18000
18001 #[tokio::test]
18002 async fn invalid_signal_history_payload_aliases_precede_shortcuts() {
18003 let client = Client::new("http://127.0.0.1:8080").expect("client");
18004 let worker = Worker::new(client, "rust-workers");
18005
18006 for event_type in ["SignalReceived", "SignalApplied"] {
18007 for (payload_field, codec) in [
18008 ("value", "json"),
18009 ("input", "unknown"),
18010 ("arguments", "json"),
18011 ] {
18012 let payload = json!({
18013 "signal_name": "empty",
18014 payload_field: {"codec": codec, "blob": null}
18015 });
18016 let workflow = workflow_task(
18017 "missing",
18018 vec![history_event(event_type, payload.clone())],
18019 DEFAULT_CODEC,
18020 );
18021 let error = worker
18022 .execute_workflow_task(workflow)
18023 .expect_err("signal payload codec must precede unknown workflow outcome");
18024 assert!(
18025 error.to_string().contains("unsupported_payload_codec"),
18026 "{event_type}.{payload_field} returned an unrelated workflow error: {error}"
18027 );
18028
18029 let query: QueryTask = serde_json::from_value(json!({
18030 "query_task_id": format!("query-{event_type}-{payload_field}"),
18031 "workflow_type": "missing",
18032 "query_name": "missing",
18033 "payload_codec": DEFAULT_CODEC,
18034 "workflow_arguments": null,
18035 "query_arguments": null,
18036 "history_events": [{
18037 "event_type": event_type,
18038 "payload": payload
18039 }]
18040 }))
18041 .expect("query task");
18042 let failure = worker
18043 .execute_query_task(query)
18044 .await
18045 .expect_err("signal payload codec must precede unknown query outcome");
18046 assert_eq!(
18047 failure.reason, "query_payload_decode_failed",
18048 "{event_type}.{payload_field} returned an unrelated query outcome"
18049 );
18050 assert!(
18051 failure.message.contains("unsupported_payload_codec"),
18052 "{event_type}.{payload_field} returned an unrelated query error: {}",
18053 failure.message
18054 );
18055 }
18056 }
18057 }
18058
18059 #[test]
18060 fn workflow_context_schedules_activity_until_completion_is_in_history() {
18061 let ctx = WorkflowContext {
18062 state: Arc::new(Mutex::new(
18063 WorkflowState::new_with_identity(
18064 Vec::new(),
18065 Some("wf-parent".to_string()),
18066 Some("run-parent".to_string()),
18067 "rust-workers".to_string(),
18068 DEFAULT_CODEC.to_string(),
18069 None,
18070 )
18071 .expect("workflow state"),
18072 )),
18073 };
18074
18075 let mut call = Box::pin(ctx.activity("hello.activity", json!(["Ada"])));
18076 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18077 assert!(matches!(
18078 call.as_mut().poll(&mut task_context),
18079 Poll::Pending
18080 ));
18081
18082 let commands = ctx.take_commands().expect("commands");
18083 assert_eq!(commands[0]["type"], "schedule_activity");
18084 assert_eq!(commands[0]["activity_type"], "hello.activity");
18085 }
18086
18087 #[test]
18088 fn activity_options_encode_retry_policy_queue_and_every_timeout() {
18089 let ctx = workflow_context(Vec::new());
18090 let options = ActivityOptions::new()
18091 .task_queue("payments")
18092 .retry_policy(
18093 ActivityRetryPolicy::new(4)
18094 .exponential_backoff(Duration::from_secs(1), 3, Some(Duration::from_secs(10)))
18095 .non_retryable_error_type("ValidationError"),
18096 )
18097 .start_to_close_timeout(Duration::from_secs(120))
18098 .schedule_to_start_timeout(Duration::from_secs(10))
18099 .schedule_to_close_timeout(Duration::from_secs(300))
18100 .heartbeat_timeout(Duration::from_secs(15));
18101 let mut call = Box::pin(ctx.activity_with_options(
18102 "charge-card",
18103 options,
18104 json!([{"order_id": "o-1"}]),
18105 ));
18106 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18107
18108 assert!(matches!(
18109 call.as_mut().poll(&mut task_context),
18110 Poll::Pending
18111 ));
18112 assert!(matches!(
18113 call.as_mut().poll(&mut task_context),
18114 Poll::Pending
18115 ));
18116
18117 let commands = ctx.take_commands().expect("activity command");
18118 assert_eq!(commands.len(), 1, "one future emits one logical schedule");
18119 assert_eq!(commands[0]["queue"], "payments");
18120 assert_eq!(
18121 commands[0]["retry_policy"],
18122 json!({
18123 "max_attempts": 4,
18124 "backoff_seconds": [1, 3, 9],
18125 "non_retryable_error_types": ["ValidationError"],
18126 })
18127 );
18128 assert_eq!(commands[0]["start_to_close_timeout"], 120);
18129 assert_eq!(commands[0]["schedule_to_start_timeout"], 10);
18130 assert_eq!(commands[0]["schedule_to_close_timeout"], 300);
18131 assert_eq!(commands[0]["heartbeat_timeout"], 15);
18132 }
18133
18134 #[test]
18135 fn activity_options_encode_explicit_and_rounded_backoff_intervals() {
18136 let ctx = workflow_context(Vec::new());
18137 let options = ActivityOptions::new().retry_policy(
18138 ActivityRetryPolicy::new(3)
18139 .backoff_intervals([Duration::from_millis(1), Duration::from_millis(1_001)]),
18140 );
18141 let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
18142 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18143
18144 assert!(matches!(
18145 call.as_mut().poll(&mut task_context),
18146 Poll::Pending
18147 ));
18148 assert_eq!(
18149 ctx.take_commands().expect("command")[0]["retry_policy"]["backoff_seconds"],
18150 json!([1, 2])
18151 );
18152 }
18153
18154 #[test]
18155 fn invalid_activity_options_return_typed_errors_before_emitting_commands() {
18156 let cases = [
18157 (
18158 ActivityOptions::new().task_queue(" "),
18159 ActivityOptionsErrorKind::EmptyTaskQueue,
18160 ),
18161 (
18162 ActivityOptions::new().retry_policy(ActivityRetryPolicy::default()),
18163 ActivityOptionsErrorKind::EmptyRetryPolicy,
18164 ),
18165 (
18166 ActivityOptions::new().retry_policy(ActivityRetryPolicy::new(0)),
18167 ActivityOptionsErrorKind::InvalidMaxAttempts,
18168 ),
18169 (
18170 ActivityOptions::new().retry_policy(ActivityRetryPolicy {
18171 max_attempts: None,
18172 backoff: Some(ActivityBackoff::Explicit(vec![Duration::from_secs(1)])),
18173 non_retryable_error_types: Vec::new(),
18174 }),
18175 ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
18176 ),
18177 (
18178 ActivityOptions::new().retry_policy(
18179 ActivityRetryPolicy::new(2)
18180 .backoff_intervals([Duration::from_secs(1), Duration::from_secs(2)]),
18181 ),
18182 ActivityOptionsErrorKind::TooManyBackoffIntervals,
18183 ),
18184 (
18185 ActivityOptions::new().retry_policy(
18186 ActivityRetryPolicy::new(2).exponential_backoff(
18187 Duration::from_secs(1),
18188 0,
18189 None,
18190 ),
18191 ),
18192 ActivityOptionsErrorKind::InvalidBackoffCoefficient,
18193 ),
18194 (
18195 ActivityOptions::new()
18196 .retry_policy(ActivityRetryPolicy::new(2).non_retryable_error_type(" ")),
18197 ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
18198 ),
18199 (
18200 ActivityOptions::new().retry_policy(
18201 ActivityRetryPolicy::new(10_002).exponential_backoff(
18202 Duration::from_secs(1),
18203 1,
18204 None,
18205 ),
18206 ),
18207 ActivityOptionsErrorKind::BackoffGenerationTooLarge,
18208 ),
18209 (
18210 ActivityOptions::new().retry_policy(
18211 ActivityRetryPolicy::new(2)
18212 .backoff_intervals([Duration::from_secs(i64::MAX as u64 + 1)]),
18213 ),
18214 ActivityOptionsErrorKind::BackoffOverflow,
18215 ),
18216 ];
18217
18218 for (options, expected_kind) in cases {
18219 let ctx = workflow_context(Vec::new());
18220 let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
18221 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18222 let Poll::Ready(Err(Error::InvalidActivityOptions(error))) =
18223 call.as_mut().poll(&mut task_context)
18224 else {
18225 panic!("expected typed activity validation error");
18226 };
18227 assert_eq!(error.kind, expected_kind);
18228 assert!(ctx.take_commands().expect("commands").is_empty());
18229 }
18230 }
18231
18232 #[test]
18233 fn activity_options_validate_positive_and_ordered_timeouts() {
18234 let zero_timeout_cases = [
18235 ActivityOptions::new().start_to_close_timeout(Duration::ZERO),
18236 ActivityOptions::new().schedule_to_start_timeout(Duration::ZERO),
18237 ActivityOptions::new().schedule_to_close_timeout(Duration::ZERO),
18238 ActivityOptions::new().heartbeat_timeout(Duration::ZERO),
18239 ];
18240 for options in zero_timeout_cases {
18241 assert_eq!(
18242 options.validate().expect_err("zero timeout").kind,
18243 ActivityOptionsErrorKind::TimeoutNotPositive
18244 );
18245 }
18246
18247 let ordering_cases = [
18248 ActivityOptions::new()
18249 .heartbeat_timeout(Duration::from_secs(11))
18250 .start_to_close_timeout(Duration::from_secs(10)),
18251 ActivityOptions::new()
18252 .start_to_close_timeout(Duration::from_secs(31))
18253 .schedule_to_close_timeout(Duration::from_secs(30)),
18254 ActivityOptions::new()
18255 .schedule_to_start_timeout(Duration::from_secs(31))
18256 .schedule_to_close_timeout(Duration::from_secs(30)),
18257 ];
18258 for options in ordering_cases {
18259 assert_eq!(
18260 options.validate().expect_err("timeout order").kind,
18261 ActivityOptionsErrorKind::TimeoutOrder
18262 );
18263 }
18264
18265 assert_eq!(
18266 ActivityOptions::new()
18267 .start_to_close_timeout(Duration::from_secs(i64::MAX as u64 + 1))
18268 .validate()
18269 .expect_err("protocol integer overflow")
18270 .kind,
18271 ActivityOptionsErrorKind::TimeoutOverflow
18272 );
18273 }
18274
18275 #[test]
18276 fn replayed_activity_retry_history_completes_without_duplicate_schedule() {
18277 let ctx = workflow_context(completed_retry_activity_history());
18278 let mut call =
18279 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18280 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18281
18282 assert!(matches!(
18283 call.as_mut().poll(&mut task_context),
18284 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18285 ));
18286 assert!(ctx.take_commands().expect("commands").is_empty());
18287 ctx.ensure_history_consumed().expect("history consumed");
18288 }
18289
18290 #[test]
18291 fn duplicate_non_retryable_types_use_one_command_and_replay_representation() {
18292 let mut options = retry_activity_options();
18293 options
18294 .retry_policy
18295 .as_mut()
18296 .expect("retry policy")
18297 .non_retryable_error_types
18298 .extend([" PermanentError ".to_string(), "PermanentError".to_string()]);
18299
18300 let new_ctx = workflow_context(Vec::new());
18301 let mut new_call =
18302 Box::pin(new_ctx.activity_with_options("flaky", options.clone(), json!([])));
18303 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18304 assert!(matches!(
18305 new_call.as_mut().poll(&mut task_context),
18306 Poll::Pending
18307 ));
18308 let commands = new_ctx.take_commands().expect("commands");
18309 assert_eq!(commands.len(), 1);
18310 assert_eq!(
18311 commands[0]["retry_policy"]["non_retryable_error_types"],
18312 json!(["PermanentError"])
18313 );
18314
18315 let replay_ctx = workflow_context(completed_retry_activity_history());
18316 let mut replay_call =
18317 Box::pin(replay_ctx.activity_with_options("flaky", options, json!([])));
18318 assert!(matches!(
18319 replay_call.as_mut().poll(&mut task_context),
18320 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18321 ));
18322 assert!(replay_ctx.take_commands().expect("commands").is_empty());
18323 replay_ctx
18324 .ensure_history_consumed()
18325 .expect("history consumed");
18326 }
18327
18328 #[test]
18329 fn replayed_intermediate_retry_remains_pending_across_restarts() {
18330 let history = completed_retry_activity_history()
18331 .into_iter()
18332 .take(3)
18333 .collect::<Vec<_>>();
18334
18335 for _restart in 0..2 {
18336 let ctx = workflow_context(history.clone());
18337 let mut call =
18338 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18339 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18340 assert!(matches!(
18341 call.as_mut().poll(&mut task_context),
18342 Poll::Pending
18343 ));
18344 assert!(ctx.take_commands().expect("commands").is_empty());
18345 }
18346 }
18347
18348 #[test]
18349 fn replayed_activity_rejects_changed_queue_retry_and_every_timeout_field() {
18350 let mut changed_queue = retry_activity_options();
18351 changed_queue.task_queue = Some("different-queue".to_string());
18352
18353 let mut changed_max_attempts = retry_activity_options();
18354 let retry_policy = changed_max_attempts
18355 .retry_policy
18356 .as_mut()
18357 .expect("retry policy");
18358 retry_policy.max_attempts = Some(4);
18359
18360 let mut changed_backoff = retry_activity_options();
18361 let retry_policy = changed_backoff.retry_policy.as_mut().expect("retry policy");
18362 retry_policy.backoff = Some(ActivityBackoff::Explicit(vec![
18363 Duration::from_secs(3),
18364 Duration::from_secs(4),
18365 ]));
18366
18367 let mut changed_non_retryable_types = retry_activity_options();
18368 let retry_policy = changed_non_retryable_types
18369 .retry_policy
18370 .as_mut()
18371 .expect("retry policy");
18372 retry_policy.non_retryable_error_types = vec!["AnotherPermanentError".to_string()];
18373
18374 let mut changed_start_to_close = retry_activity_options();
18375 changed_start_to_close.start_to_close_timeout = Some(Duration::from_secs(31));
18376 let mut changed_schedule_to_start = retry_activity_options();
18377 changed_schedule_to_start.schedule_to_start_timeout = Some(Duration::from_secs(6));
18378 let mut changed_schedule_to_close = retry_activity_options();
18379 changed_schedule_to_close.schedule_to_close_timeout = Some(Duration::from_secs(91));
18380 let mut changed_heartbeat = retry_activity_options();
18381 changed_heartbeat.heartbeat_timeout = Some(Duration::from_secs(11));
18382
18383 let cases = [
18384 (changed_queue, "activity_task_queue_mismatch"),
18385 (changed_max_attempts, "activity_retry_policy_mismatch"),
18386 (changed_backoff, "activity_retry_policy_mismatch"),
18387 (
18388 changed_non_retryable_types,
18389 "activity_retry_policy_mismatch",
18390 ),
18391 (changed_start_to_close, "activity_retry_policy_mismatch"),
18392 (changed_schedule_to_start, "activity_retry_policy_mismatch"),
18393 (changed_schedule_to_close, "activity_retry_policy_mismatch"),
18394 (changed_heartbeat, "activity_retry_policy_mismatch"),
18395 ];
18396
18397 for (options, expected_reason) in cases {
18398 let ctx = workflow_context(completed_retry_activity_history());
18399 let mut call = Box::pin(ctx.activity_with_options("flaky", options, json!([])));
18400 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18401 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18402 call.as_mut().poll(&mut task_context)
18403 else {
18404 panic!("changed activity options must fail replay");
18405 };
18406 assert_eq!(failure.reason, expected_reason);
18407 assert_eq!(failure.sequence, Some(1));
18408 assert!(ctx.take_commands().expect("commands").is_empty());
18409 }
18410 }
18411
18412 #[test]
18413 fn replayed_activity_rejects_changed_execution_mode_and_snapshot_version() {
18414 let cases = [
18415 (
18416 "execution_mode",
18417 json!("local"),
18418 "activity_execution_mode_mismatch",
18419 ),
18420 (
18421 "snapshot_version",
18422 json!(2),
18423 "activity_retry_policy_mismatch",
18424 ),
18425 ];
18426
18427 for (field, value, expected_reason) in cases {
18428 let mut history = completed_retry_activity_history();
18429 let activity = history[0].payload["activity"]
18430 .as_object_mut()
18431 .expect("activity snapshot");
18432 if field == "execution_mode" {
18433 activity.insert(field.to_string(), value);
18434 } else {
18435 activity["retry_policy"]
18436 .as_object_mut()
18437 .expect("retry snapshot")
18438 .insert(field.to_string(), value);
18439 }
18440
18441 let ctx = workflow_context(history);
18442 let mut call =
18443 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18444 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18445 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18446 call.as_mut().poll(&mut task_context)
18447 else {
18448 panic!("changed {field} must fail replay");
18449 };
18450 assert_eq!(failure.reason, expected_reason);
18451 assert_eq!(failure.sequence, Some(1));
18452 assert!(ctx.take_commands().expect("commands").is_empty());
18453 }
18454 }
18455
18456 #[test]
18457 fn replayed_legacy_activity_treats_missing_option_snapshot_as_unknown() {
18458 let mut history = completed_retry_activity_history();
18459 let activity = history[0].payload["activity"]
18460 .as_object_mut()
18461 .expect("activity snapshot");
18462 activity.remove("execution_mode");
18463 activity.remove("retry_policy");
18464
18465 let mut current = retry_activity_options();
18466 current.start_to_close_timeout = Some(Duration::from_secs(45));
18467 current.schedule_to_start_timeout = Some(Duration::from_secs(8));
18468 current.schedule_to_close_timeout = Some(Duration::from_secs(120));
18469 current.heartbeat_timeout = Some(Duration::from_secs(12));
18470
18471 let ctx = workflow_context(history);
18472 let mut call = Box::pin(ctx.activity_with_options("flaky", current, json!([])));
18473 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18474 assert!(matches!(
18475 call.as_mut().poll(&mut task_context),
18476 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18477 ));
18478 assert!(ctx.take_commands().expect("commands").is_empty());
18479 ctx.ensure_history_consumed().expect("history consumed");
18480 }
18481
18482 #[test]
18483 fn terminal_activity_failed_after_start_returns_typed_failure() {
18484 let history = vec![
18485 history_event(
18486 "ActivityScheduled",
18487 json!({
18488 "sequence": 1,
18489 "activity_type": "flaky",
18490 "activity_execution_id": "act-terminal",
18491 "activity": {
18492 "id": "act-terminal",
18493 "sequence": 1,
18494 "type": "flaky",
18495 "queue": "critical-activities",
18496 "retry_policy": {
18497 "snapshot_version": 1,
18498 "max_attempts": 3,
18499 "backoff_seconds": [2, 4],
18500 "non_retryable_error_types": ["PermanentError"]
18501 }
18502 }
18503 }),
18504 ),
18505 history_event(
18506 "ActivityStarted",
18507 json!({
18508 "sequence": 1,
18509 "activity_type": "flaky",
18510 "activity_execution_id": "act-terminal",
18511 "activity_attempt_id": "attempt-1",
18512 "attempt_number": 1
18513 }),
18514 ),
18515 history_event(
18516 "ActivityFailed",
18517 json!({
18518 "sequence": 1,
18519 "activity_type": "flaky",
18520 "activity_execution_id": "act-terminal",
18521 "activity_attempt_id": "attempt-1",
18522 "attempt_number": 1,
18523 "failure_id": "failure-terminal",
18524 "failure_category": "activity",
18525 "exception_type": "PermanentError",
18526 "message": "cannot retry",
18527 "non_retryable": true
18528 }),
18529 ),
18530 ];
18531 let ctx = workflow_context(history);
18532 let mut call =
18533 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18534 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18535
18536 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18537 call.as_mut().poll(&mut task_context)
18538 else {
18539 panic!("terminal ActivityFailed must settle the activity future");
18540 };
18541 assert_eq!(failure.kind, ActivityFailureKind::Failed);
18542 assert_eq!(
18543 failure.activity_execution_id.as_deref(),
18544 Some("act-terminal")
18545 );
18546 assert_eq!(failure.exception_type.as_deref(), Some("PermanentError"));
18547 assert!(failure.non_retryable);
18548 assert!(ctx.take_commands().expect("commands").is_empty());
18549 ctx.ensure_history_consumed().expect("history consumed");
18550 }
18551
18552 #[test]
18553 fn activity_terminal_events_return_machine_readable_failures() {
18554 let cases = [
18555 (
18556 "ActivityFailed",
18557 json!({
18558 "sequence": 1,
18559 "activity_type": "charge-card",
18560 "activity_execution_id": "act-1",
18561 "activity_attempt_id": "attempt-2",
18562 "attempt_number": 2,
18563 "failure_id": "failure-1",
18564 "failure_category": "activity",
18565 "exception_type": "PaymentDeclined",
18566 "exception_class": "payments.PaymentDeclined",
18567 "message": "card declined",
18568 "non_retryable": true
18569 }),
18570 ActivityFailureKind::Failed,
18571 "activity",
18572 ),
18573 (
18574 "ActivityCancelled",
18575 json!({
18576 "sequence": 1,
18577 "activity_type": "charge-card",
18578 "activity_execution_id": "act-1",
18579 "activity_attempt_id": "attempt-1"
18580 }),
18581 ActivityFailureKind::Cancelled,
18582 "cancelled",
18583 ),
18584 ];
18585
18586 for (event_type, payload, expected_kind, expected_reason) in cases {
18587 let ctx = workflow_context(vec![history_event(event_type, payload)]);
18588 let mut call = Box::pin(ctx.activity("charge-card", json!([])));
18589 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18590 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18591 call.as_mut().poll(&mut task_context)
18592 else {
18593 panic!("expected terminal activity failure");
18594 };
18595 assert_eq!(failure.kind, expected_kind);
18596 assert_eq!(failure.reason, expected_reason);
18597 assert_eq!(failure.activity_execution_id.as_deref(), Some("act-1"));
18598 assert_eq!(failure.activity_type.as_deref(), Some("charge-card"));
18599 }
18600 }
18601
18602 #[test]
18603 fn every_activity_timeout_class_is_typed() {
18604 for timeout_kind in [
18605 "start_to_close",
18606 "schedule_to_start",
18607 "schedule_to_close",
18608 "heartbeat",
18609 ] {
18610 let ctx = workflow_context(vec![history_event(
18611 "ActivityTimedOut",
18612 json!({
18613 "sequence": 1,
18614 "activity_type": "slow",
18615 "activity_execution_id": "act-timeout",
18616 "activity_attempt_id": "attempt-timeout",
18617 "failure_category": "timeout",
18618 "timeout_kind": timeout_kind,
18619 "message": "deadline expired"
18620 }),
18621 )]);
18622 let mut call = Box::pin(ctx.activity("slow", json!([])));
18623 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18624 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18625 call.as_mut().poll(&mut task_context)
18626 else {
18627 panic!("expected timeout failure");
18628 };
18629 assert_eq!(failure.kind, ActivityFailureKind::TimedOut);
18630 assert_eq!(failure.reason, timeout_kind);
18631 assert_eq!(failure.timeout_kind.as_deref(), Some(timeout_kind));
18632 assert_eq!(failure.failure_category.as_deref(), Some("timeout"));
18633 }
18634 }
18635
18636 #[test]
18637 fn workflow_sleep_emits_one_durable_timer_and_rounds_up() {
18638 let ctx = workflow_context(Vec::new());
18639 let mut sleep = Box::pin(ctx.sleep(Duration::from_millis(1_001)));
18640 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18641
18642 assert!(matches!(
18643 sleep.as_mut().poll(&mut task_context),
18644 Poll::Pending
18645 ));
18646 assert!(matches!(
18647 sleep.as_mut().poll(&mut task_context),
18648 Poll::Pending
18649 ));
18650
18651 let commands = ctx.take_commands().expect("timer command");
18652 assert_eq!(
18653 commands,
18654 vec![json!({
18655 "type": "start_timer",
18656 "delay_seconds": 2,
18657 })]
18658 );
18659 }
18660
18661 #[test]
18662 fn workflow_sleep_replays_matching_schedule_and_fire_without_a_command() {
18663 let history = vec![
18664 history_event(
18665 "TimerScheduled",
18666 json!({
18667 "sequence": 1,
18668 "timer_id": "timer-1",
18669 "delay_seconds": 5,
18670 "fire_at": "2026-07-11T12:00:05Z",
18671 }),
18672 ),
18673 history_event(
18674 "TimerFired",
18675 json!({
18676 "sequence": 1,
18677 "timer_id": "timer-1",
18678 "delay_seconds": 5,
18679 "fire_at": "2026-07-11T12:00:05Z",
18680 "fired_at": "2026-07-11T12:00:05Z",
18681 }),
18682 ),
18683 ];
18684
18685 for _restart in 0..2 {
18686 let ctx = workflow_context(history.clone());
18687 let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(5)));
18688 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18689 assert!(matches!(
18690 sleep.as_mut().poll(&mut task_context),
18691 Poll::Ready(Ok(()))
18692 ));
18693 assert!(ctx.take_commands().expect("commands").is_empty());
18694 ctx.ensure_history_consumed().expect("history consumed");
18695 }
18696 }
18697
18698 #[test]
18699 fn workflow_sleep_rejects_changed_delay_during_replay() {
18700 let ctx = workflow_context(vec![
18701 history_event(
18702 "TimerScheduled",
18703 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
18704 ),
18705 history_event(
18706 "TimerFired",
18707 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
18708 ),
18709 ]);
18710 let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(500)));
18711 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18712
18713 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18714 sleep.as_mut().poll(&mut task_context)
18715 else {
18716 panic!("changed timer delay must be rejected");
18717 };
18718 assert_eq!(failure.reason, "timer_delay_mismatch");
18719 assert_eq!(failure.sequence, Some(1));
18720 }
18721
18722 #[test]
18723 fn workflow_condition_wait_emits_published_identity_and_timeout_contract() {
18724 let ctx = workflow_context(Vec::new());
18725 let mut wait = Box::pin(
18726 ctx.wait_condition(
18727 ConditionWaitOptions::new("approval.ready", "sha256:approval-v1")
18728 .timeout(Duration::from_millis(60_001)),
18729 || Ok(false),
18730 ),
18731 );
18732 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18733
18734 assert!(matches!(
18735 wait.as_mut().poll(&mut task_context),
18736 Poll::Pending
18737 ));
18738 assert!(matches!(
18739 wait.as_mut().poll(&mut task_context),
18740 Poll::Pending
18741 ));
18742 assert_eq!(
18743 ctx.take_commands().expect("condition command"),
18744 vec![json!({
18745 "type": "open_condition_wait",
18746 "condition_wait_occurrence_id": "rust:condition-wait:0",
18747 "condition_key": "approval.ready",
18748 "condition_definition_fingerprint": "sha256:approval-v1",
18749 "timeout_seconds": 61,
18750 })]
18751 );
18752 }
18753
18754 #[test]
18755 fn workflow_condition_wait_returns_explicit_immediate_results_without_commands() {
18756 let ctx = workflow_context(Vec::new());
18757 let mut satisfied = Box::pin(wait_condition!(ctx, "already-ready", || Ok(true)));
18758 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18759 assert!(matches!(
18760 satisfied.as_mut().poll(&mut task_context),
18761 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18762 ));
18763
18764 let mut timed_out = Box::pin(wait_condition!(
18765 ctx,
18766 "no-wait",
18767 timeout: Duration::ZERO,
18768 || Ok(false),
18769 ));
18770 assert!(matches!(
18771 timed_out.as_mut().poll(&mut task_context),
18772 Poll::Ready(Ok(ConditionWaitResult::TimedOut))
18773 ));
18774 assert!(ctx.take_commands().expect("commands").is_empty());
18775 }
18776
18777 #[test]
18778 fn signal_and_update_history_reevaluate_open_conditions_after_restart() {
18779 let signal_history = vec![
18780 history_event(
18781 "ConditionWaitOpened",
18782 json!({
18783 "sequence": 4,
18784 "condition_wait_id": "condition:4",
18785 "condition_wait_occurrence_id": "rust:condition-wait:0",
18786 "condition_key": "approval",
18787 "condition_definition_fingerprint": "sha256:approval-v1",
18788 "timeout_seconds": 30,
18789 }),
18790 ),
18791 history_event(
18792 "SignalReceived",
18793 json!({
18794 "workflow_sequence": 4,
18795 "signal_name": "approve",
18796 "arguments": fixture_envelope(json!(["Ada"])),
18797 }),
18798 ),
18799 ];
18800 for _worker_before_or_after_restart in 0..2 {
18801 let ctx = workflow_context(signal_history.clone());
18802 let predicate_ctx = ctx.clone();
18803 let mut wait = Box::pin(
18804 ctx.wait_condition(
18805 ConditionWaitOptions::new("approval", "sha256:approval-v1")
18806 .timeout(Duration::from_secs(30)),
18807 move || Ok(!predicate_ctx.signals("approve")?.is_empty()),
18808 ),
18809 );
18810 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18811 assert!(matches!(
18812 wait.as_mut().poll(&mut task_context),
18813 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18814 ));
18815 assert!(ctx.take_commands().expect("commands").is_empty());
18816 ctx.ensure_history_consumed().expect("condition consumed");
18817 }
18818
18819 let update_history = vec![
18820 history_event(
18821 "ConditionWaitOpened",
18822 json!({
18823 "sequence": 7,
18824 "condition_wait_id": "condition:7",
18825 "condition_wait_occurrence_id": "rust:condition-wait:0",
18826 "condition_key": "update-approval",
18827 "condition_definition_fingerprint": "sha256:update-approval-v1",
18828 }),
18829 ),
18830 history_event(
18831 "UpdateApplied",
18832 json!({
18833 "sequence": 7,
18834 "update_id": "update-1",
18835 "update_name": "approve",
18836 "arguments": fixture_envelope(json!([true])),
18837 }),
18838 ),
18839 ];
18840 let ctx = workflow_context(update_history);
18841 let predicate_ctx = ctx.clone();
18842 let mut wait = Box::pin(ctx.wait_condition(
18843 ConditionWaitOptions::new("update-approval", "sha256:update-approval-v1"),
18844 move || {
18845 Ok(predicate_ctx
18846 .updates("approve")?
18847 .first()
18848 .and_then(|arguments| arguments.first())
18849 .and_then(Value::as_bool)
18850 == Some(true))
18851 },
18852 ));
18853 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18854 assert!(matches!(
18855 wait.as_mut().poll(&mut task_context),
18856 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18857 ));
18858 assert!(ctx.take_commands().expect("commands").is_empty());
18859 ctx.ensure_history_consumed().expect("condition consumed");
18860 }
18861
18862 #[test]
18863 fn condition_wait_preserves_open_satisfied_and_timed_out_replay_states() {
18864 let open_history = vec![
18865 history_event(
18866 "ConditionWaitOpened",
18867 json!({
18868 "sequence": 3,
18869 "condition_wait_id": "condition:3",
18870 "condition_wait_occurrence_id": "rust:condition-wait:0",
18871 "condition_key": "two-votes",
18872 "condition_definition_fingerprint": "sha256:two-votes-v1",
18873 "timeout_seconds": 120,
18874 }),
18875 ),
18876 history_event(
18877 "SignalReceived",
18878 json!({
18879 "workflow_sequence": 3,
18880 "signal_name": "vote",
18881 "arguments": fixture_envelope(json!(["first"])),
18882 }),
18883 ),
18884 ];
18885 for _worker_before_or_after_restart in 0..2 {
18886 let ctx = workflow_context(open_history.clone());
18887 let predicate_ctx = ctx.clone();
18888 let mut wait = Box::pin(
18889 ctx.wait_condition(
18890 ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1")
18891 .timeout(Duration::from_secs(120)),
18892 move || Ok(predicate_ctx.signals("vote")?.len() >= 2),
18893 ),
18894 );
18895 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18896 assert!(matches!(
18897 wait.as_mut().poll(&mut task_context),
18898 Poll::Pending
18899 ));
18900 assert_eq!(
18901 ctx.take_commands().expect("reopened condition"),
18902 vec![json!({
18903 "type": "open_condition_wait",
18904 "condition_wait_occurrence_id": "rust:condition-wait:0",
18905 "condition_key": "two-votes",
18906 "condition_definition_fingerprint": "sha256:two-votes-v1",
18907 "timeout_seconds": 120,
18908 })]
18909 );
18910 }
18911
18912 let satisfied_ctx = workflow_context(vec![
18913 history_event(
18914 "ConditionWaitOpened",
18915 json!({
18916 "sequence": 5,
18917 "condition_wait_id": "condition:5",
18918 "condition_wait_occurrence_id": "rust:condition-wait:0",
18919 "condition_key": "approval",
18920 "condition_definition_fingerprint": "sha256:approval-v1",
18921 }),
18922 ),
18923 history_event(
18924 "ConditionWaitSatisfied",
18925 json!({
18926 "sequence": 5,
18927 "condition_wait_id": "condition:5",
18928 "condition_wait_occurrence_id": "rust:condition-wait:0",
18929 "condition_key": "approval",
18930 "condition_definition_fingerprint": "sha256:approval-v1",
18931 }),
18932 ),
18933 ]);
18934 let mut satisfied = Box::pin(satisfied_ctx.wait_condition(
18935 ConditionWaitOptions::new("approval", "sha256:approval-v1"),
18936 || Ok(false),
18937 ));
18938 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18939 assert!(matches!(
18940 satisfied.as_mut().poll(&mut task_context),
18941 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18942 ));
18943
18944 let timed_out_ctx = workflow_context(vec![
18945 history_event(
18946 "ConditionWaitOpened",
18947 json!({
18948 "sequence": 8,
18949 "condition_wait_id": "condition:8",
18950 "condition_wait_occurrence_id": "rust:condition-wait:0",
18951 "condition_key": "approval-timeout",
18952 "condition_definition_fingerprint": "sha256:approval-timeout-v1",
18953 "timeout_seconds": 5,
18954 }),
18955 ),
18956 history_event(
18957 "TimerScheduled",
18958 json!({
18959 "sequence": 9,
18960 "timer_id": "condition-timer:9",
18961 "timer_kind": "condition_timeout",
18962 "condition_wait_id": "condition:8",
18963 "delay_seconds": 5,
18964 }),
18965 ),
18966 history_event(
18967 "TimerFired",
18968 json!({
18969 "sequence": 9,
18970 "timer_id": "condition-timer:9",
18971 "timer_kind": "condition_timeout",
18972 "condition_wait_id": "condition:8",
18973 "delay_seconds": 5,
18974 }),
18975 ),
18976 ]);
18977 let mut timed_out = Box::pin(
18978 timed_out_ctx.wait_condition(
18979 ConditionWaitOptions::new("approval-timeout", "sha256:approval-timeout-v1")
18980 .timeout(Duration::from_secs(5)),
18981 || Ok(true),
18982 ),
18983 );
18984 assert!(matches!(
18985 timed_out.as_mut().poll(&mut task_context),
18986 Poll::Ready(Ok(ConditionWaitResult::TimedOut))
18987 ));
18988 }
18989
18990 #[test]
18991 fn condition_wait_replays_repeated_physical_opens_as_one_logical_wait() {
18992 let history = vec![
18993 history_event(
18994 "ConditionWaitOpened",
18995 json!({
18996 "sequence": 3,
18997 "condition_wait_id": "condition:3",
18998 "condition_wait_occurrence_id": "rust:condition-wait:0",
18999 "condition_key": "two-votes",
19000 "condition_definition_fingerprint": "sha256:two-votes-v1",
19001 }),
19002 ),
19003 history_event(
19004 "SignalReceived",
19005 json!({
19006 "workflow_sequence": 3,
19007 "signal_name": "vote",
19008 "arguments": fixture_envelope(json!(["first"])),
19009 }),
19010 ),
19011 history_event(
19012 "ConditionWaitSatisfied",
19013 json!({
19014 "sequence": 3,
19015 "condition_wait_id": "condition:3",
19016 "condition_wait_occurrence_id": "rust:condition-wait:0",
19017 "condition_key": "two-votes",
19018 "condition_definition_fingerprint": "sha256:two-votes-v1",
19019 }),
19020 ),
19021 history_event(
19022 "ConditionWaitOpened",
19023 json!({
19024 "sequence": 5,
19025 "condition_wait_id": "condition:5",
19026 "condition_wait_occurrence_id": "rust:condition-wait:0",
19027 "condition_key": "two-votes",
19028 "condition_definition_fingerprint": "sha256:two-votes-v1",
19029 }),
19030 ),
19031 history_event(
19032 "SignalReceived",
19033 json!({
19034 "workflow_sequence": 5,
19035 "signal_name": "vote",
19036 "arguments": fixture_envelope(json!(["second"])),
19037 }),
19038 ),
19039 history_event(
19040 "ConditionWaitSatisfied",
19041 json!({
19042 "sequence": 5,
19043 "condition_wait_id": "condition:5",
19044 "condition_wait_occurrence_id": "rust:condition-wait:0",
19045 "condition_key": "two-votes",
19046 "condition_definition_fingerprint": "sha256:two-votes-v1",
19047 }),
19048 ),
19049 ];
19050 for _cold_worker_or_restart in 0..2 {
19051 let ctx = workflow_context(history.clone());
19052 let predicate_ctx = ctx.clone();
19053 let mut wait = Box::pin(ctx.wait_condition(
19054 ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1"),
19055 move || Ok(predicate_ctx.signals("vote")?.len() >= 2),
19056 ));
19057 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19058
19059 assert!(matches!(
19060 wait.as_mut().poll(&mut task_context),
19061 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19062 ));
19063 assert!(ctx.take_commands().expect("commands").is_empty());
19064 ctx.ensure_history_consumed()
19065 .expect("every physical wait-open is consumed");
19066 }
19067 }
19068
19069 #[test]
19070 fn condition_wait_replays_update_driven_physical_opens_as_one_occurrence() {
19071 let history = vec![
19072 history_event(
19073 "ConditionWaitOpened",
19074 json!({
19075 "sequence": 3,
19076 "condition_wait_id": "condition:3",
19077 "condition_wait_occurrence_id": "rust:condition-wait:0",
19078 "condition_key": "approved",
19079 "condition_definition_fingerprint": "sha256:approved-v1",
19080 }),
19081 ),
19082 history_event(
19083 "UpdateApplied",
19084 json!({
19085 "sequence": 3,
19086 "update_id": "update-1",
19087 "update_name": "approve",
19088 "arguments": fixture_envelope(json!([false])),
19089 }),
19090 ),
19091 history_event(
19092 "ConditionWaitOpened",
19093 json!({
19094 "sequence": 5,
19095 "condition_wait_id": "condition:5",
19096 "condition_wait_occurrence_id": "rust:condition-wait:0",
19097 "condition_key": "approved",
19098 "condition_definition_fingerprint": "sha256:approved-v1",
19099 }),
19100 ),
19101 history_event(
19102 "UpdateApplied",
19103 json!({
19104 "sequence": 5,
19105 "update_id": "update-2",
19106 "update_name": "approve",
19107 "arguments": fixture_envelope(json!([true])),
19108 }),
19109 ),
19110 ];
19111
19112 for _cold_worker_or_restart in 0..2 {
19113 let ctx = workflow_context(history.clone());
19114 let predicate_ctx = ctx.clone();
19115 let mut wait = Box::pin(ctx.wait_condition(
19116 ConditionWaitOptions::new("approved", "sha256:approved-v1"),
19117 move || {
19118 Ok(predicate_ctx
19119 .updates("approve")?
19120 .last()
19121 .and_then(|arguments| arguments.first())
19122 .and_then(Value::as_bool)
19123 == Some(true))
19124 },
19125 ));
19126 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19127
19128 assert!(matches!(
19129 wait.as_mut().poll(&mut task_context),
19130 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19131 ));
19132 assert!(ctx.take_commands().expect("commands").is_empty());
19133 ctx.ensure_history_consumed()
19134 .expect("every update-driven reopen is consumed");
19135 }
19136 }
19137
19138 #[test]
19139 fn condition_wait_replay_keeps_every_adjacent_authored_occurrence_distinct() {
19140 for (first_key, first_fingerprint, second_key, second_fingerprint) in [
19141 ("shared", "sha256:first", "shared", "sha256:second"),
19142 ("first", "sha256:shared", "second", "sha256:shared"),
19143 ("shared", "sha256:shared", "shared", "sha256:shared"),
19144 ("first", "sha256:first", "second", "sha256:second"),
19145 ] {
19146 let history = vec![
19147 history_event(
19148 "ConditionWaitOpened",
19149 json!({
19150 "sequence": 3,
19151 "condition_wait_id": "condition:3",
19152 "condition_wait_occurrence_id": "rust:condition-wait:0",
19153 "condition_key": first_key,
19154 "condition_definition_fingerprint": first_fingerprint,
19155 }),
19156 ),
19157 history_event(
19158 "ConditionWaitSatisfied",
19159 json!({
19160 "sequence": 3,
19161 "condition_wait_id": "condition:3",
19162 "condition_wait_occurrence_id": "rust:condition-wait:0",
19163 "condition_key": first_key,
19164 "condition_definition_fingerprint": first_fingerprint,
19165 }),
19166 ),
19167 history_event(
19168 "ConditionWaitOpened",
19169 json!({
19170 "sequence": 4,
19171 "condition_wait_id": "condition:4",
19172 "condition_wait_occurrence_id": "rust:condition-wait:1",
19173 "condition_key": second_key,
19174 "condition_definition_fingerprint": second_fingerprint,
19175 }),
19176 ),
19177 history_event(
19178 "ConditionWaitSatisfied",
19179 json!({
19180 "sequence": 4,
19181 "condition_wait_id": "condition:4",
19182 "condition_wait_occurrence_id": "rust:condition-wait:1",
19183 "condition_key": second_key,
19184 "condition_definition_fingerprint": second_fingerprint,
19185 }),
19186 ),
19187 ];
19188 for _cold_worker_or_restart in 0..2 {
19189 let ctx = workflow_context(history.clone());
19190 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19191 let mut first = Box::pin(ctx.wait_condition(
19192 ConditionWaitOptions::new(first_key, first_fingerprint),
19193 || Ok(false),
19194 ));
19195 assert!(matches!(
19196 first.as_mut().poll(&mut task_context),
19197 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19198 ));
19199
19200 let mut second = Box::pin(ctx.wait_condition(
19201 ConditionWaitOptions::new(second_key, second_fingerprint),
19202 || Ok(false),
19203 ));
19204 assert!(matches!(
19205 second.as_mut().poll(&mut task_context),
19206 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19207 ));
19208 assert!(ctx.take_commands().expect("commands").is_empty());
19209 ctx.ensure_history_consumed()
19210 .expect("each authored wait consumes one occurrence");
19211 }
19212 }
19213 }
19214
19215 #[test]
19216 fn cold_workers_replay_adjacent_condition_waits_from_one_loop_call_site() {
19217 fn worker() -> Worker {
19218 let client = Client::new("http://127.0.0.1:8080").expect("client");
19219 let mut worker = Worker::new(client, "rust-workers");
19220 worker.register_workflow("rust.condition-loop", |ctx, _input| async move {
19221 let mut outcomes = Vec::new();
19222 for _ in 0..2 {
19223 outcomes.push(
19224 ctx.wait_condition(
19225 ConditionWaitOptions::new("shared", "sha256:shared"),
19226 || Ok(false),
19227 )
19228 .await?,
19229 );
19230 }
19231 Ok(json!(outcomes))
19232 });
19233 worker
19234 }
19235
19236 let task = workflow_task(
19237 "rust.condition-loop",
19238 vec![
19239 history_event(
19240 "ConditionWaitOpened",
19241 json!({
19242 "sequence": 1,
19243 "condition_wait_id": "condition:1",
19244 "condition_wait_occurrence_id": "rust:condition-wait:0",
19245 "condition_key": "shared",
19246 "condition_definition_fingerprint": "sha256:shared",
19247 }),
19248 ),
19249 history_event(
19250 "ConditionWaitSatisfied",
19251 json!({
19252 "sequence": 1,
19253 "condition_wait_id": "condition:1",
19254 "condition_wait_occurrence_id": "rust:condition-wait:0",
19255 "condition_key": "shared",
19256 "condition_definition_fingerprint": "sha256:shared",
19257 }),
19258 ),
19259 history_event(
19260 "ConditionWaitOpened",
19261 json!({
19262 "sequence": 2,
19263 "condition_wait_id": "condition:2",
19264 "condition_wait_occurrence_id": "rust:condition-wait:1",
19265 "condition_key": "shared",
19266 "condition_definition_fingerprint": "sha256:shared",
19267 }),
19268 ),
19269 history_event(
19270 "ConditionWaitSatisfied",
19271 json!({
19272 "sequence": 2,
19273 "condition_wait_id": "condition:2",
19274 "condition_wait_occurrence_id": "rust:condition-wait:1",
19275 "condition_key": "shared",
19276 "condition_definition_fingerprint": "sha256:shared",
19277 }),
19278 ),
19279 ],
19280 DEFAULT_CODEC,
19281 );
19282
19283 for _cold_worker_or_restart in 0..2 {
19284 let commands = worker()
19285 .execute_workflow_task(task.clone())
19286 .expect("adjacent loop waits replay deterministically");
19287 assert_eq!(commands.len(), 1);
19288 assert_eq!(commands[0]["type"], "complete_workflow");
19289 assert_eq!(
19290 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
19291 json!(["satisfied", "satisfied"])
19292 );
19293 }
19294 }
19295
19296 #[test]
19297 fn condition_wait_replay_rejects_identity_predicate_and_timeout_changes() {
19298 let history = vec![history_event(
19299 "ConditionWaitOpened",
19300 json!({
19301 "sequence": 12,
19302 "condition_wait_id": "condition:12",
19303 "condition_wait_occurrence_id": "rust:condition-wait:0",
19304 "condition_key": "approval",
19305 "condition_definition_fingerprint": "sha256:approval-v1",
19306 "timeout_seconds": 30,
19307 }),
19308 )];
19309 for (options, expected_reason) in [
19310 (
19311 ConditionWaitOptions::new("changed", "sha256:approval-v1")
19312 .timeout(Duration::from_secs(30)),
19313 "condition_wait_key_mismatch",
19314 ),
19315 (
19316 ConditionWaitOptions::new("approval", "sha256:approval-v2")
19317 .timeout(Duration::from_secs(30)),
19318 "condition_wait_predicate_mismatch",
19319 ),
19320 (
19321 ConditionWaitOptions::new("approval", "sha256:approval-v1")
19322 .timeout(Duration::from_secs(29)),
19323 "condition_wait_timeout_mismatch",
19324 ),
19325 ] {
19326 let ctx = workflow_context(history.clone());
19327 let mut wait = Box::pin(ctx.wait_condition(options, || Ok(false)));
19328 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19329 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
19330 wait.as_mut().poll(&mut task_context)
19331 else {
19332 panic!("changed condition definition must fail replay");
19333 };
19334 assert_eq!(failure.reason, expected_reason);
19335 assert_eq!(failure.sequence, Some(12));
19336 }
19337 }
19338
19339 #[test]
19340 fn condition_wait_history_requires_the_canonical_predicate_fingerprint() {
19341 let error = WorkflowState::new(
19342 vec![history_event(
19343 "ConditionWaitOpened",
19344 json!({
19345 "sequence": 12,
19346 "condition_wait_id": "condition:12",
19347 "condition_wait_occurrence_id": "rust:condition-wait:0",
19348 "condition_key": "approval",
19349 }),
19350 )],
19351 "rust-workers".to_string(),
19352 DEFAULT_CODEC.to_string(),
19353 None,
19354 )
19355 .expect_err("condition history without a predicate fingerprint must fail");
19356
19357 assert!(matches!(
19358 error,
19359 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19360 if reason == "condition_wait_predicate_fingerprint_missing"
19361 ));
19362 }
19363
19364 #[test]
19365 fn condition_wait_history_requires_authored_occurrence_identity() {
19366 let error = WorkflowState::new(
19367 vec![history_event(
19368 "ConditionWaitOpened",
19369 json!({
19370 "sequence": 12,
19371 "condition_wait_id": "condition:12",
19372 "condition_key": "approval",
19373 "condition_definition_fingerprint": "sha256:approval-v1",
19374 }),
19375 )],
19376 "rust-workers".to_string(),
19377 DEFAULT_CODEC.to_string(),
19378 None,
19379 )
19380 .expect_err("condition history without occurrence identity must fail");
19381
19382 assert!(matches!(
19383 error,
19384 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19385 if reason == "condition_wait_occurrence_id_missing"
19386 ));
19387 }
19388
19389 #[test]
19390 fn typed_search_attribute_updates_validate_emit_and_replay() {
19391 let update = SearchAttributeUpdate::new()
19392 .keyword("OrderStatus", " waiting ")
19393 .expect("keyword")
19394 .int("Attempt", 3)
19395 .expect("int")
19396 .bool("Escalated", false)
19397 .expect("bool")
19398 .keyword_list("Regions", ["us-east", "eu-west"])
19399 .expect("list")
19400 .datetime("UpdatedAt", "2026-08-22T04:00:00Z")
19401 .expect("datetime")
19402 .delete("LegacyStatus")
19403 .expect("delete");
19404 let ctx = workflow_context(Vec::new());
19405 ctx.upsert_search_attributes(update.clone())
19406 .expect("typed update");
19407 assert_eq!(
19408 ctx.take_commands().expect("search-attribute command"),
19409 vec![json!({
19410 "type": "upsert_search_attributes",
19411 "attributes": {
19412 "Attempt": 3,
19413 "Escalated": false,
19414 "LegacyStatus": null,
19415 "OrderStatus": "waiting",
19416 "Regions": ["us-east", "eu-west"],
19417 "UpdatedAt": "2026-08-22T04:00:00Z",
19418 },
19419 "attribute_types": {
19420 "Attempt": "int",
19421 "Escalated": "bool",
19422 "OrderStatus": "keyword",
19423 "Regions": "keyword_list",
19424 "UpdatedAt": "datetime",
19425 },
19426 })]
19427 );
19428
19429 let replay = workflow_context(vec![history_event(
19430 "SearchAttributesUpserted",
19431 json!({
19432 "sequence": 6,
19433 "attributes": {
19434 "Attempt": 3,
19435 "Escalated": false,
19436 "LegacyStatus": null,
19437 "OrderStatus": "waiting",
19438 "Regions": ["us-east", "eu-west"],
19439 "UpdatedAt": "2026-08-22T04:00:00Z",
19440 },
19441 "attribute_types": {
19442 "Attempt": "int",
19443 "Escalated": "bool",
19444 "OrderStatus": "keyword",
19445 "Regions": "keyword_list",
19446 "UpdatedAt": "datetime",
19447 },
19448 "merged": {},
19449 }),
19450 )]);
19451 replay
19452 .upsert_search_attributes(update)
19453 .expect("matching update replays");
19454 assert!(replay.take_commands().expect("commands").is_empty());
19455 replay.ensure_history_consumed().expect("history consumed");
19456
19457 let type_drift = workflow_context(vec![history_event(
19458 "SearchAttributesUpserted",
19459 json!({
19460 "sequence": 7,
19461 "attributes": {"OrderStatus": "waiting"},
19462 "attribute_types": {"OrderStatus": "keyword"},
19463 "merged": {"OrderStatus": "waiting"},
19464 }),
19465 )]);
19466 let error = type_drift
19467 .upsert_search_attributes(
19468 SearchAttributeUpdate::new()
19469 .string("OrderStatus", "waiting")
19470 .expect("string update"),
19471 )
19472 .expect_err("same JSON value with a changed type must fail replay");
19473 assert!(matches!(
19474 error,
19475 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19476 if reason == "search_attribute_type_mismatch"
19477 ));
19478
19479 let malformed_types = WorkflowState::new(
19480 vec![history_event(
19481 "SearchAttributesUpserted",
19482 json!({
19483 "sequence": 8,
19484 "attributes": {"OrderStatus": "waiting"},
19485 "attribute_types": {"OrderStatus": "unsupported"},
19486 "merged": {"OrderStatus": "waiting"},
19487 }),
19488 )],
19489 "rust-workers".to_string(),
19490 DEFAULT_CODEC.to_string(),
19491 None,
19492 )
19493 .expect_err("unsupported search-attribute type metadata must fail");
19494 assert!(matches!(
19495 malformed_types,
19496 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19497 if reason == "search_attribute_types_malformed"
19498 ));
19499
19500 assert!(matches!(
19501 SearchAttributeUpdate::new().keyword("bad key", "value"),
19502 Err(SearchAttributeUpdateError::InvalidKey(_))
19503 ));
19504 assert!(matches!(
19505 SearchAttributeUpdate::new().float("Ratio", f64::NAN),
19506 Err(SearchAttributeUpdateError::NonFiniteFloat(_))
19507 ));
19508 assert!(matches!(
19509 SearchAttributeUpdate::new().keyword("UnicodeKeyword", "é".repeat(128)),
19510 Err(SearchAttributeUpdateError::ValueTooLong { .. })
19511 ));
19512 assert!(matches!(
19513 SearchAttributeUpdate::new().datetime("UpdatedAt", "2026-02-30T04:00:00Z"),
19514 Err(SearchAttributeUpdateError::InvalidDateTime(_))
19515 ));
19516 assert!(matches!(
19517 workflow_context(Vec::new()).upsert_search_attributes(SearchAttributeUpdate::new()),
19518 Err(Error::InvalidSearchAttributeUpdate(
19519 SearchAttributeUpdateError::Empty
19520 ))
19521 ));
19522 }
19523
19524 #[test]
19525 fn typed_search_attribute_text_uses_the_runtime_byte_limit() {
19526 let ascii = "a".repeat(MAX_SEARCH_ATTRIBUTE_STRING_LENGTH);
19527 let utf8 = "é".repeat(MAX_SEARCH_ATTRIBUTE_STRING_LENGTH / 2);
19528
19529 assert!(SearchAttributeUpdate::new()
19530 .string("AsciiDescription", ascii)
19531 .is_ok());
19532 assert!(SearchAttributeUpdate::new()
19533 .string("Utf8Description", utf8)
19534 .is_ok());
19535 assert!(matches!(
19536 SearchAttributeUpdate::new().string(
19537 "TooLongDescription",
19538 "é".repeat((MAX_SEARCH_ATTRIBUTE_STRING_LENGTH / 2) + 1),
19539 ),
19540 Err(SearchAttributeUpdateError::ValueTooLong {
19541 kind: "string",
19542 limit: MAX_SEARCH_ATTRIBUTE_STRING_LENGTH,
19543 ..
19544 })
19545 ));
19546 }
19547
19548 #[test]
19549 fn workflow_history_rejects_unpaired_or_mismatched_timer_events() {
19550 let lone_fire = WorkflowState::new(
19551 vec![history_event(
19552 "TimerFired",
19553 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19554 )],
19555 "rust-workers".to_string(),
19556 DEFAULT_CODEC.to_string(),
19557 None,
19558 )
19559 .expect_err("TimerFired requires TimerScheduled");
19560 assert!(matches!(
19561 lone_fire,
19562 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19563 if reason == "timer_schedule_missing_or_duplicate"
19564 ));
19565
19566 let wrong_identity = WorkflowState::new(
19567 vec![
19568 history_event(
19569 "TimerScheduled",
19570 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19571 ),
19572 history_event(
19573 "TimerFired",
19574 json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
19575 ),
19576 ],
19577 "rust-workers".to_string(),
19578 DEFAULT_CODEC.to_string(),
19579 None,
19580 )
19581 .expect_err("fire must match scheduled timer identity");
19582 assert!(matches!(
19583 wrong_identity,
19584 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19585 if reason == "timer_identity_mismatch"
19586 ));
19587
19588 let duplicate_fire = WorkflowState::new(
19589 vec![
19590 history_event(
19591 "TimerScheduled",
19592 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19593 ),
19594 history_event(
19595 "TimerFired",
19596 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19597 ),
19598 history_event(
19599 "TimerFired",
19600 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19601 ),
19602 ],
19603 "rust-workers".to_string(),
19604 DEFAULT_CODEC.to_string(),
19605 None,
19606 )
19607 .expect_err("a durable timer cannot fire twice");
19608 assert!(matches!(
19609 duplicate_fire,
19610 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19611 if reason == "duplicate_timer_fire"
19612 ));
19613
19614 let wrong_fired_delay = WorkflowState::new(
19615 vec![
19616 history_event(
19617 "TimerScheduled",
19618 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19619 ),
19620 history_event(
19621 "TimerFired",
19622 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 6}),
19623 ),
19624 ],
19625 "rust-workers".to_string(),
19626 DEFAULT_CODEC.to_string(),
19627 None,
19628 )
19629 .expect_err("timer schedule and fire delays must agree");
19630 assert!(matches!(
19631 wrong_fired_delay,
19632 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19633 if reason == "timer_history_delay_mismatch"
19634 ));
19635 }
19636
19637 #[test]
19638 fn replay_rejects_activity_moved_before_recorded_timer() {
19639 let ctx = workflow_context(vec![
19640 history_event(
19641 "TimerScheduled",
19642 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19643 ),
19644 history_event(
19645 "TimerFired",
19646 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19647 ),
19648 history_event(
19649 "ActivityCompleted",
19650 json!({
19651 "sequence": 2,
19652 "activity_type": "after-timer",
19653 "payload_codec": DEFAULT_CODEC,
19654 "result": fixture_envelope(json!("done")),
19655 }),
19656 ),
19657 ]);
19658 let mut activity = Box::pin(ctx.activity("after-timer", json!([])));
19659 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19660
19661 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
19662 activity.as_mut().poll(&mut task_context)
19663 else {
19664 panic!("reordered durable command must be rejected");
19665 };
19666 assert_eq!(failure.reason, "recorded_command_mismatch");
19667 assert_eq!(failure.sequence, Some(1));
19668 assert_eq!(failure.expected.as_deref(), Some("timer"));
19669 assert_eq!(failure.actual.as_deref(), Some("activity:after-timer"));
19670 }
19671
19672 #[test]
19673 fn workflow_context_emits_a_typed_named_signal_wait() {
19674 let ctx = workflow_context(Vec::new());
19675 let mut signal = Box::pin(ctx.wait_signal("finish"));
19676 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19677
19678 assert!(matches!(
19679 signal.as_mut().poll(&mut task_context),
19680 Poll::Pending
19681 ));
19682 assert_eq!(
19683 ctx.take_commands().expect("signal-wait command"),
19684 vec![json!({
19685 "type": "open_signal_wait",
19686 "signal_name": "finish",
19687 })]
19688 );
19689 }
19690
19691 #[test]
19692 fn runtime_message_stream_transport_cannot_be_opened_as_a_user_signal() {
19693 let ctx = workflow_context(Vec::new());
19694 let mut signal = Box::pin(ctx.wait_signal(MESSAGE_STREAM_SIGNAL));
19695 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19696
19697 let Poll::Ready(Err(Error::Codec(message))) = signal.as_mut().poll(&mut task_context)
19698 else {
19699 panic!("runtime-reserved signal should be rejected");
19700 };
19701 assert!(message.contains("reserved by the workflow runtime"));
19702 assert!(ctx.take_commands().expect("commands").is_empty());
19703 }
19704
19705 #[tokio::test]
19706 async fn runtime_message_stream_transport_cannot_be_sent_as_a_user_signal() {
19707 let client = Client::builder("http://127.0.0.1:9")
19708 .build()
19709 .expect("client");
19710 let error = client
19711 .signal_workflow("workflow-1", MESSAGE_STREAM_SIGNAL, json!(["forged"]))
19712 .await
19713 .expect_err("runtime-reserved signal should be rejected before transport");
19714
19715 assert!(
19716 matches!(error, Error::Codec(ref message) if message.contains("reserved by the workflow runtime"))
19717 );
19718 }
19719
19720 #[test]
19721 fn message_stream_worker_task_consumes_current_contiguous_bounded_batch() {
19722 fn delivery(message_id: &str, position: u64, value: &str) -> Value {
19723 let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
19724 value.to_string(),
19725 )]))
19726 .expect("message payload");
19727 json!({
19728 "schema": MESSAGE_STREAM_SCHEMA,
19729 "stream_name": "orders",
19730 "message_id": message_id,
19731 "position": position,
19732 "payload_envelope": payload,
19733 })
19734 }
19735
19736 fn opened(sequence: u64) -> HistoryEvent {
19737 history_event(
19738 "SignalWaitOpened",
19739 json!({
19740 "sequence": sequence,
19741 "signal_name": MESSAGE_STREAM_SIGNAL,
19742 }),
19743 )
19744 }
19745
19746 fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
19747 history_event(
19748 "SignalApplied",
19749 json!({
19750 "sequence": sequence,
19751 "signal_name": MESSAGE_STREAM_SIGNAL,
19752 "value": fixture_envelope(json!([delivery])),
19753 }),
19754 )
19755 }
19756
19757 fn received(delivery: Value) -> HistoryEvent {
19758 history_event(
19759 "SignalReceived",
19760 json!({
19761 "signal_name": MESSAGE_STREAM_SIGNAL,
19762 "arguments": fixture_envelope(json!([delivery])),
19763 "payload_codec": DEFAULT_CODEC,
19764 }),
19765 )
19766 }
19767
19768 let client = Client::new("http://127.0.0.1:8080").expect("client");
19769 let mut worker = Worker::new(client, "rust-workers");
19770 worker.register_workflow("rust.message-stream-batch", |ctx, _input| async move {
19771 let messages = ctx.message_stream("orders")?.receive(2).await?;
19772 Ok(json!(messages
19773 .into_iter()
19774 .map(|message| message.message_id)
19775 .collect::<Vec<_>>()))
19776 });
19777
19778 let first = delivery("message-1", 1, "one");
19779 let second = delivery("message-2", 2, "two");
19780 let batch = worker
19781 .execute_workflow_task_decision(workflow_task(
19782 "rust.message-stream-batch",
19783 vec![
19784 opened(1),
19785 received(first.clone()),
19786 applied(1, first.clone()),
19787 received(first.clone()),
19788 received(second),
19789 ],
19790 DEFAULT_CODEC,
19791 ))
19792 .expect("worker task consumes the available batch");
19793
19794 assert_eq!(batch.commands.len(), 1);
19795 assert_eq!(batch.commands[0]["type"], "complete_workflow");
19796 assert_eq!(
19797 decode_wire_value(&batch.commands[0]["result"], DEFAULT_CODEC)
19798 .expect("workflow result"),
19799 json!(["message-1", "message-2"])
19800 );
19801 assert_eq!(
19802 batch.message_stream_cursors,
19803 vec![json!({"stream_name": "orders", "through_position": 2})]
19804 );
19805 assert!(batch.message_stream_waits.is_empty());
19806
19807 let partial = worker
19808 .execute_workflow_task_decision(workflow_task(
19809 "rust.message-stream-batch",
19810 vec![opened(1), received(first.clone()), applied(1, first)],
19811 DEFAULT_CODEC,
19812 ))
19813 .expect("worker task returns without waiting for a missing second item");
19814 assert_eq!(partial.commands.len(), 1);
19815 assert_eq!(partial.commands[0]["type"], "complete_workflow");
19816 assert_eq!(
19817 decode_wire_value(&partial.commands[0]["result"], DEFAULT_CODEC)
19818 .expect("workflow result"),
19819 json!(["message-1"])
19820 );
19821 assert_eq!(
19822 partial.message_stream_cursors,
19823 vec![json!({"stream_name": "orders", "through_position": 1})]
19824 );
19825 assert!(partial.message_stream_waits.is_empty());
19826 }
19827
19828 #[test]
19829 fn message_stream_replay_preserves_partial_batch_boundary_before_later_wait() {
19830 fn delivery(message_id: &str, position: u64, value: &str) -> Value {
19831 let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
19832 value.to_string(),
19833 )]))
19834 .expect("message payload");
19835 json!({
19836 "schema": MESSAGE_STREAM_SCHEMA,
19837 "stream_name": "orders",
19838 "message_id": message_id,
19839 "position": position,
19840 "payload_envelope": payload,
19841 })
19842 }
19843
19844 fn opened(sequence: u64) -> HistoryEvent {
19845 history_event(
19846 "SignalWaitOpened",
19847 json!({
19848 "sequence": sequence,
19849 "signal_name": MESSAGE_STREAM_SIGNAL,
19850 }),
19851 )
19852 }
19853
19854 fn received(delivery: Value) -> HistoryEvent {
19855 history_event(
19856 "SignalReceived",
19857 json!({
19858 "signal_name": MESSAGE_STREAM_SIGNAL,
19859 "arguments": fixture_envelope(json!([delivery])),
19860 "payload_codec": DEFAULT_CODEC,
19861 }),
19862 )
19863 }
19864
19865 fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
19866 history_event(
19867 "SignalApplied",
19868 json!({
19869 "sequence": sequence,
19870 "signal_name": MESSAGE_STREAM_SIGNAL,
19871 "value": fixture_envelope(json!([delivery])),
19872 }),
19873 )
19874 }
19875
19876 let client = Client::new("http://127.0.0.1:8080").expect("client");
19877 let mut worker = Worker::new(client, "rust-workers");
19878 worker.register_workflow(
19879 "rust.message-stream-partial-batches",
19880 |ctx, _input| async move {
19881 let stream = ctx.message_stream("orders")?;
19882 let first = stream.receive(10).await?;
19883 let second = stream.receive(10).await?;
19884 Ok(json!([
19885 first
19886 .into_iter()
19887 .map(|message| message.message_id)
19888 .collect::<Vec<_>>(),
19889 second
19890 .into_iter()
19891 .map(|message| message.message_id)
19892 .collect::<Vec<_>>(),
19893 ]))
19894 },
19895 );
19896
19897 let first = delivery("message-1", 1, "one");
19898 let second = delivery("message-2", 2, "two");
19899 let decision = worker
19900 .execute_workflow_task_decision(workflow_task(
19901 "rust.message-stream-partial-batches",
19902 vec![
19903 opened(1),
19904 received(first.clone()),
19905 applied(1, first),
19906 opened(2),
19907 received(second.clone()),
19908 applied(2, second),
19909 ],
19910 DEFAULT_CODEC,
19911 ))
19912 .expect("cold replay preserves both authored receive boundaries");
19913
19914 assert_eq!(decision.commands.len(), 1);
19915 assert_eq!(decision.commands[0]["type"], "complete_workflow");
19916 assert_eq!(
19917 decode_wire_value(&decision.commands[0]["result"], DEFAULT_CODEC)
19918 .expect("workflow result"),
19919 json!([["message-1"], ["message-2"]])
19920 );
19921 assert_eq!(
19922 decision.message_stream_cursors,
19923 vec![json!({"stream_name": "orders", "through_position": 2})]
19924 );
19925 assert!(decision.message_stream_waits.is_empty());
19926 }
19927
19928 #[test]
19929 fn empty_message_stream_opens_internal_signal_wait_and_reports_position() {
19930 let ctx = workflow_context(Vec::new());
19931 let stream = ctx.message_stream("orders").expect("message stream");
19932 let mut receive = Box::pin(stream.receive(10));
19933 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19934
19935 assert!(matches!(
19936 receive.as_mut().poll(&mut task_context),
19937 Poll::Pending
19938 ));
19939 assert_eq!(
19940 ctx.take_commands().expect("message-stream wait command"),
19941 vec![json!({
19942 "type": "open_signal_wait",
19943 "signal_name": MESSAGE_STREAM_SIGNAL,
19944 })]
19945 );
19946 let (cursors, waits) = ctx.message_stream_metadata().expect("stream metadata");
19947 assert!(cursors.is_empty());
19948 assert_eq!(
19949 waits,
19950 vec![json!({"stream_name": "orders", "after_position": 0})]
19951 );
19952 }
19953
19954 #[test]
19955 fn continue_as_new_cursor_checkpoint_preserves_global_pending_position() {
19956 let ctx = workflow_context(vec![history_event(
19957 "SignalReceived",
19958 json!({
19959 "signal_name": MESSAGE_STREAM_SIGNAL,
19960 "arguments": fixture_envelope(json!([{
19961 "schema": MESSAGE_STREAM_CURSOR_SCHEMA,
19962 "stream_name": "orders",
19963 "through_position": 2,
19964 }])),
19965 "payload_codec": DEFAULT_CODEC,
19966 }),
19967 )]);
19968 let stream = ctx.message_stream("orders").expect("message stream");
19969 let mut receive = Box::pin(stream.receive(10));
19970 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19971
19972 assert!(matches!(
19973 receive.as_mut().poll(&mut task_context),
19974 Poll::Pending
19975 ));
19976 let (cursors, waits) = ctx.message_stream_metadata().expect("stream metadata");
19977 assert_eq!(
19978 cursors,
19979 vec![json!({"stream_name": "orders", "through_position": 2})]
19980 );
19981 assert_eq!(
19982 waits,
19983 vec![json!({"stream_name": "orders", "after_position": 2})]
19984 );
19985 }
19986
19987 #[test]
19988 fn message_stream_delivery_preserves_typed_avro_arguments_across_replay() {
19989 let mut empty_map = BTreeMap::new();
19990 let mut nested = BTreeMap::new();
19991 nested.insert(
19992 "value".to_string(),
19993 AvroValue::Array(vec![AvroValue::Bytes(b"nested".to_vec())]),
19994 );
19995 let values = vec![
19996 AvroValue::Bytes(vec![0, 255]),
19997 AvroValue::Long(1),
19998 AvroValue::Double(1.0),
19999 AvroValue::Array(Vec::new()),
20000 AvroValue::Map(std::mem::take(&mut empty_map)),
20001 AvroValue::Map(nested),
20002 ];
20003 let payload = encode_avro_value(&AvroValue::Array(values.clone())).expect("payload");
20004 let transport = vec![json!({
20005 "schema": MESSAGE_STREAM_SCHEMA,
20006 "stream_name": "orders",
20007 "message_id": "message-1",
20008 "position": 1,
20009 "payload_envelope": payload,
20010 })];
20011
20012 for _ in 0..2 {
20013 let Some(MessageStreamDelivery::Message(message)) =
20014 decode_message_stream_delivery(transport.clone()).expect("delivery")
20015 else {
20016 panic!("message delivery expected");
20017 };
20018 assert_eq!(message.arguments, values);
20019 assert!(matches!(message.arguments[1], AvroValue::Long(1)));
20020 assert!(matches!(message.arguments[2], AvroValue::Double(1.0)));
20021 }
20022 }
20023
20024 #[test]
20025 fn cold_worker_replacement_consumes_message_stream_wait_arrivals_once_in_order() {
20026 fn delivery(message_id: &str, position: u64, value: &str) -> Value {
20027 let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
20028 value.to_string(),
20029 )]))
20030 .expect("message payload");
20031 json!({
20032 "schema": MESSAGE_STREAM_SCHEMA,
20033 "stream_name": "orders",
20034 "message_id": message_id,
20035 "position": position,
20036 "payload_envelope": payload,
20037 })
20038 }
20039
20040 fn opened(sequence: u64) -> HistoryEvent {
20041 history_event(
20042 "SignalWaitOpened",
20043 json!({
20044 "sequence": sequence,
20045 "signal_name": MESSAGE_STREAM_SIGNAL,
20046 }),
20047 )
20048 }
20049
20050 fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
20051 history_event(
20052 "SignalApplied",
20053 json!({
20054 "sequence": sequence,
20055 "signal_name": MESSAGE_STREAM_SIGNAL,
20056 "value": fixture_envelope(json!([delivery])),
20057 }),
20058 )
20059 }
20060
20061 fn worker() -> Worker {
20062 let client = Client::new("http://127.0.0.1:8080").expect("client");
20063 let mut worker = Worker::new(client, "rust-workers");
20064 worker.register_workflow("rust.message-stream", |ctx, _input| async move {
20065 let stream = ctx.message_stream("orders")?;
20066 let first = stream.receive_one().await?;
20067 let second = stream.receive_one().await?;
20068 Ok(json!([first.message_id, second.message_id]))
20069 });
20070 worker
20071 }
20072
20073 fn task_with_resume(history: Vec<HistoryEvent>, delivery: Value) -> WorkflowTask {
20074 let mut task = workflow_task("rust.message-stream", history, DEFAULT_CODEC);
20075 task.signal_name = Some(MESSAGE_STREAM_SIGNAL.to_string());
20076 task.signal_arguments = Some(fixture_envelope(json!([delivery])));
20077 task
20078 }
20079
20080 let waiting = worker()
20081 .execute_workflow_task_decision(workflow_task(
20082 "rust.message-stream",
20083 Vec::new(),
20084 DEFAULT_CODEC,
20085 ))
20086 .expect("first worker opens the stream wait");
20087 assert_eq!(
20088 waiting.commands,
20089 vec![json!({
20090 "type": "open_signal_wait",
20091 "signal_name": MESSAGE_STREAM_SIGNAL,
20092 })]
20093 );
20094 assert!(waiting.message_stream_cursors.is_empty());
20095 assert_eq!(
20096 waiting.message_stream_waits,
20097 vec![json!({"stream_name": "orders", "after_position": 0})]
20098 );
20099
20100 let first_delivery = delivery("message-1", 1, "one");
20101 let first_arrival = worker()
20102 .execute_workflow_task_decision(task_with_resume(
20103 vec![opened(1)],
20104 first_delivery.clone(),
20105 ))
20106 .expect("replacement worker consumes the first arrival");
20107 assert_eq!(
20108 first_arrival.commands,
20109 vec![json!({
20110 "type": "open_signal_wait",
20111 "signal_name": MESSAGE_STREAM_SIGNAL,
20112 })]
20113 );
20114 assert_eq!(
20115 first_arrival.message_stream_cursors,
20116 vec![json!({"stream_name": "orders", "through_position": 1})]
20117 );
20118 assert_eq!(
20119 first_arrival.message_stream_waits,
20120 vec![json!({"stream_name": "orders", "after_position": 1})]
20121 );
20122
20123 let second_delivery = delivery("message-2", 2, "two");
20124 let first_applied = applied(1, first_delivery);
20125 let completed = worker()
20126 .execute_workflow_task_decision(task_with_resume(
20127 vec![opened(1), first_applied.clone(), opened(2)],
20128 second_delivery.clone(),
20129 ))
20130 .expect("next replacement worker consumes the second arrival");
20131 assert_eq!(completed.commands.len(), 1);
20132 assert_eq!(completed.commands[0]["type"], "complete_workflow");
20133 assert_eq!(
20134 decode_wire_value(&completed.commands[0]["result"], DEFAULT_CODEC)
20135 .expect("workflow result"),
20136 json!(["message-1", "message-2"])
20137 );
20138 assert_eq!(
20139 completed.message_stream_cursors,
20140 vec![json!({"stream_name": "orders", "through_position": 2})]
20141 );
20142 assert!(completed.message_stream_waits.is_empty());
20143
20144 let replay_history = vec![
20145 opened(1),
20146 first_applied,
20147 opened(2),
20148 applied(2, second_delivery),
20149 ];
20150 for _cold_worker_or_restart in 0..2 {
20151 let replayed = worker()
20152 .execute_workflow_task_decision(workflow_task(
20153 "rust.message-stream",
20154 replay_history.clone(),
20155 DEFAULT_CODEC,
20156 ))
20157 .expect("cold worker replays each logical message exactly once");
20158 assert_eq!(replayed.commands.len(), 1);
20159 assert_eq!(
20160 decode_wire_value(&replayed.commands[0]["result"], DEFAULT_CODEC)
20161 .expect("replayed workflow result"),
20162 json!(["message-1", "message-2"])
20163 );
20164 assert_eq!(
20165 replayed.message_stream_cursors,
20166 vec![json!({"stream_name": "orders", "through_position": 2})]
20167 );
20168 assert!(replayed.message_stream_waits.is_empty());
20169 }
20170 }
20171
20172 #[test]
20173 fn message_stream_capability_and_completion_require_protocol_one_fifteen() {
20174 assert!(!worker_protocol_supports_message_streams("1.14"));
20175 assert!(worker_protocol_supports_message_streams("1.15"));
20176 assert!(worker_protocol_supports_message_streams("1.16"));
20177 assert!(worker_protocol_supports_message_streams(
20178 WORKER_PROTOCOL_VERSION
20179 ));
20180 assert_eq!(MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION, "1.15");
20181 }
20182
20183 #[test]
20184 fn condition_wait_history_cannot_be_consumed_as_a_typed_signal_wait() {
20185 let ctx = workflow_context(vec![
20186 history_event(
20187 "ConditionWaitOpened",
20188 json!({
20189 "sequence": 1,
20190 "condition_wait_id": "condition:1",
20191 "condition_wait_occurrence_id": "rust:condition-wait:0",
20192 "condition_key": "signal:finish",
20193 "condition_definition_fingerprint": "sha256:signal-finish-v1",
20194 }),
20195 ),
20196 history_event(
20197 "ConditionWaitSatisfied",
20198 json!({
20199 "sequence": 1,
20200 "condition_wait_id": "condition:1",
20201 "condition_wait_occurrence_id": "rust:condition-wait:0",
20202 "condition_key": "signal:finish",
20203 "condition_definition_fingerprint": "sha256:signal-finish-v1",
20204 }),
20205 ),
20206 history_event(
20207 "SignalReceived",
20208 json!({"signal_name": "finish", "arguments": []}),
20209 ),
20210 ]);
20211 let mut signal = Box::pin(ctx.wait_signal("finish"));
20212 let mut task_context = TaskContext::from_waker(noop_waker_ref());
20213
20214 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
20215 signal.as_mut().poll(&mut task_context)
20216 else {
20217 panic!("condition history must not resolve as a typed signal wait");
20218 };
20219 assert_eq!(failure.reason, "recorded_command_mismatch");
20220 assert_eq!(failure.expected.as_deref(), Some("condition wait"));
20221 }
20222
20223 #[test]
20224 fn replay_orders_signal_waits_and_timers_in_one_command_stream() {
20225 let signal_then_timer = vec![
20226 history_event(
20227 "SignalWaitOpened",
20228 json!({"sequence": 1, "signal_name": "go"}),
20229 ),
20230 history_event(
20231 "SignalApplied",
20232 json!({
20233 "sequence": 1,
20234 "signal_name": "go",
20235 "value": fixture_envelope(json!(["now"])),
20236 }),
20237 ),
20238 history_event(
20239 "TimerScheduled",
20240 json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
20241 ),
20242 history_event(
20243 "TimerFired",
20244 json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
20245 ),
20246 ];
20247
20248 let ctx = workflow_context(signal_then_timer.clone());
20249 let mut signal = Box::pin(ctx.wait_signal("go"));
20250 let mut task_context = TaskContext::from_waker(noop_waker_ref());
20251 assert!(matches!(
20252 signal.as_mut().poll(&mut task_context),
20253 Poll::Ready(Ok(arguments)) if arguments == vec![json!("now")]
20254 ));
20255 let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5)));
20256 assert!(matches!(
20257 timer.as_mut().poll(&mut task_context),
20258 Poll::Ready(Ok(()))
20259 ));
20260 ctx.ensure_history_consumed()
20261 .expect("signal and timer history consumed in order");
20262
20263 let reordered = workflow_context(signal_then_timer);
20264 let mut timer_first = Box::pin(reordered.sleep(Duration::from_secs(5)));
20265 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
20266 timer_first.as_mut().poll(&mut task_context)
20267 else {
20268 panic!("timer cannot consume signal-wait-first history");
20269 };
20270 assert_eq!(failure.reason, "recorded_command_mismatch");
20271 assert_eq!(failure.sequence, Some(1));
20272 assert_eq!(failure.expected.as_deref(), Some("signal wait"));
20273
20274 let timer_then_signal = vec![
20275 history_event(
20276 "TimerScheduled",
20277 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20278 ),
20279 history_event(
20280 "TimerFired",
20281 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20282 ),
20283 history_event(
20284 "SignalWaitOpened",
20285 json!({"sequence": 2, "signal_name": "go"}),
20286 ),
20287 history_event(
20288 "SignalApplied",
20289 json!({
20290 "sequence": 2,
20291 "signal_name": "go",
20292 "value": fixture_envelope(json!([])),
20293 }),
20294 ),
20295 ];
20296 let reordered = workflow_context(timer_then_signal);
20297 let mut signal_first = Box::pin(reordered.wait_signal("go"));
20298 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
20299 signal_first.as_mut().poll(&mut task_context)
20300 else {
20301 panic!("signal wait cannot consume timer-first history");
20302 };
20303 assert_eq!(failure.reason, "recorded_command_mismatch");
20304 assert_eq!(failure.sequence, Some(1));
20305 assert_eq!(failure.expected.as_deref(), Some("timer"));
20306 }
20307
20308 #[test]
20309 fn workflow_history_rejects_duplicate_or_colliding_command_sequences() {
20310 let duplicate_timer = WorkflowState::new(
20311 vec![
20312 history_event(
20313 "TimerScheduled",
20314 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20315 ),
20316 history_event(
20317 "TimerScheduled",
20318 json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
20319 ),
20320 ],
20321 "rust-workers".to_string(),
20322 DEFAULT_CODEC.to_string(),
20323 None,
20324 )
20325 .expect_err("one workflow sequence cannot schedule two timers");
20326 assert!(matches!(
20327 duplicate_timer,
20328 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20329 if reason == "timer_schedule_missing_or_duplicate"
20330 ));
20331
20332 let colliding_kinds = WorkflowState::new(
20333 vec![
20334 history_event(
20335 "TimerScheduled",
20336 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20337 ),
20338 history_event(
20339 "ActivityCompleted",
20340 json!({"sequence": 1, "activity_type": "same-sequence"}),
20341 ),
20342 ],
20343 "rust-workers".to_string(),
20344 DEFAULT_CODEC.to_string(),
20345 None,
20346 )
20347 .expect_err("one workflow sequence cannot identify two command kinds");
20348 assert!(matches!(
20349 colliding_kinds,
20350 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20351 if reason == "durable_command_sequence_collision"
20352 ));
20353
20354 let duplicate_signal_wait = WorkflowState::new(
20355 vec![
20356 history_event(
20357 "SignalWaitOpened",
20358 json!({"sequence": 1, "signal_name": "go"}),
20359 ),
20360 history_event(
20361 "SignalWaitOpened",
20362 json!({"sequence": 1, "signal_name": "go"}),
20363 ),
20364 ],
20365 "rust-workers".to_string(),
20366 DEFAULT_CODEC.to_string(),
20367 None,
20368 )
20369 .expect_err("one workflow sequence cannot open two signal waits");
20370 assert!(matches!(
20371 duplicate_signal_wait,
20372 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20373 if reason == "signal_wait_open_missing_or_duplicate"
20374 ));
20375 }
20376
20377 #[test]
20378 fn workflow_history_accepts_a_first_command_after_global_sequence_gaps() {
20379 let result = encode_value_envelope(&json!({"captured": true}), DEFAULT_CODEC)
20380 .expect("side-effect result");
20381 let ctx = workflow_context(vec![history_event(
20382 "SideEffectRecorded",
20383 json!({"sequence": 99, "result": result}),
20384 )]);
20385
20386 let replayed: Value = ctx
20387 .side_effect(|| panic!("recorded side effect must not run"))
20388 .expect("positive global workflow sequence is valid");
20389 assert_eq!(replayed, json!({"captured": true}));
20390 ctx.ensure_history_consumed().expect("history consumed");
20391 }
20392
20393 #[test]
20394 fn workflow_history_rejects_zero_and_descending_command_sequences() {
20395 let result =
20396 encode_value_envelope(&json!("captured"), DEFAULT_CODEC).expect("side-effect result");
20397 let zero = WorkflowState::new(
20398 vec![history_event(
20399 "SideEffectRecorded",
20400 json!({"sequence": 0, "result": result.clone()}),
20401 )],
20402 "rust-workers".to_string(),
20403 DEFAULT_CODEC.to_string(),
20404 None,
20405 )
20406 .expect_err("durable command sequences must be positive");
20407 assert!(matches!(
20408 zero,
20409 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20410 if reason == "durable_command_sequence_invalid"
20411 ));
20412
20413 let descending = WorkflowState::new(
20414 vec![
20415 history_event(
20416 "SideEffectRecorded",
20417 json!({"sequence": 3, "result": result}),
20418 ),
20419 history_event(
20420 "VersionMarkerRecorded",
20421 json!({
20422 "sequence": 2,
20423 "change_id": "descending-marker",
20424 "version": 1,
20425 "min_supported": 1,
20426 "max_supported": 1,
20427 }),
20428 ),
20429 ],
20430 "rust-workers".to_string(),
20431 DEFAULT_CODEC.to_string(),
20432 None,
20433 )
20434 .expect_err("new durable commands must remain strictly ordered");
20435 let Error::NonDeterministicReplay(failure) = descending else {
20436 panic!("expected typed replay failure");
20437 };
20438 assert_eq!(failure.reason, "durable_command_sequence_mismatch");
20439 assert_eq!(failure.sequence, Some(2));
20440 assert_eq!(
20441 failure.expected.as_deref(),
20442 Some("workflow sequence greater than 3")
20443 );
20444 assert_eq!(failure.actual.as_deref(), Some("2"));
20445 }
20446
20447 #[test]
20448 fn workflow_task_replay_completes_after_signals_create_sequence_gaps() {
20449 fn worker() -> Worker {
20450 let client = Client::new("http://127.0.0.1:8080").expect("client");
20451 let mut worker = Worker::new(client, "rust-workers");
20452 worker.register_workflow("rust.finish-after-gaps", |ctx, _input| async move {
20453 ctx.wait_signal("finish").await?;
20454 let marker: String =
20455 ctx.side_effect(|| panic!("recorded side effect must not run"))?;
20456 assert_eq!(marker, "after-finish");
20457 Ok(json!("finished"))
20458 });
20459 worker
20460 }
20461
20462 let marker = encode_value_envelope(&json!("after-finish"), DEFAULT_CODEC)
20463 .expect("side-effect result");
20464 let task = workflow_task(
20465 "rust.finish-after-gaps",
20466 vec![
20467 history_event(
20468 "SignalWaitOpened",
20469 json!({"sequence": 1, "signal_name": "finish"}),
20470 ),
20471 history_event(
20472 "SignalReceived",
20473 json!({
20474 "signal_id": "increment-3",
20475 "signal_name": "increment",
20476 "workflow_sequence": 2,
20477 "payload_codec": DEFAULT_CODEC,
20478 "arguments": fixture_envelope(json!([3])),
20479 }),
20480 ),
20481 history_event(
20482 "SignalReceived",
20483 json!({
20484 "signal_id": "increment-5",
20485 "signal_name": "increment",
20486 "workflow_sequence": 3,
20487 "payload_codec": DEFAULT_CODEC,
20488 "arguments": fixture_envelope(json!([5])),
20489 }),
20490 ),
20491 history_event(
20492 "SignalReceived",
20493 json!({
20494 "signal_id": "finish",
20495 "signal_name": "finish",
20496 "workflow_sequence": 4,
20497 "payload_codec": DEFAULT_CODEC,
20498 "arguments": fixture_envelope(json!([])),
20499 }),
20500 ),
20501 history_event(
20502 "SignalApplied",
20503 json!({
20504 "sequence": 1,
20505 "signal_id": "finish",
20506 "signal_name": "finish",
20507 "payload_codec": DEFAULT_CODEC,
20508 "value": fixture_envelope(json!([])),
20509 }),
20510 ),
20511 history_event(
20512 "SideEffectRecorded",
20513 json!({"sequence": 5, "result": marker}),
20514 ),
20515 ],
20516 DEFAULT_CODEC,
20517 );
20518
20519 for _original_or_cold_worker in 0..2 {
20520 let commands = worker()
20521 .execute_workflow_task(task.clone())
20522 .expect("signal gaps preserve deterministic replay");
20523 assert_eq!(commands.len(), 1, "replay emits only terminal completion");
20524 assert_eq!(commands[0]["type"], "complete_workflow");
20525 assert_eq!(
20526 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
20527 json!("finished")
20528 );
20529 }
20530 }
20531
20532 #[test]
20533 fn workflow_sleep_rejects_unrepresentable_rounded_duration() {
20534 let ctx = workflow_context(Vec::new());
20535 let mut sleep = Box::pin(ctx.start_timer(Duration::new(u64::MAX, 1)));
20536 let mut task_context = TaskContext::from_waker(noop_waker_ref());
20537 assert!(matches!(
20538 sleep.as_mut().poll(&mut task_context),
20539 Poll::Ready(Err(Error::TimerDurationOverflow))
20540 ));
20541 assert!(ctx.take_commands().expect("commands").is_empty());
20542 }
20543
20544 #[test]
20545 fn workflow_memo_update_emits_canonical_command_and_replays_once() {
20546 let entries = AvroValue::Map(BTreeMap::from([
20547 ("text".to_string(), AvroValue::String("same".to_string())),
20548 (
20549 "nested".to_string(),
20550 AvroValue::Map(BTreeMap::from([
20551 ("beta".to_string(), AvroValue::Long(2)),
20552 ("alpha".to_string(), AvroValue::Long(1)),
20553 ])),
20554 ),
20555 ("long".to_string(), AvroValue::Long(7)),
20556 ("double".to_string(), AvroValue::Double(7.0)),
20557 ("binary".to_string(), AvroValue::Bytes(b"same".to_vec())),
20558 ]));
20559 let ctx = workflow_context(Vec::new());
20560 ctx.upsert_memo(entries.clone()).expect("valid memo update");
20561 let commands = ctx.take_commands().expect("commands");
20562
20563 assert_eq!(commands.len(), 1);
20564 assert_eq!(commands[0]["type"], "upsert_memo");
20565 let server_entries = json!({
20566 "codec": "avro",
20567 "blob": "wwHioz3/VYAiNw4KDGJpbmFyeQgIc2FtZQxkb3VibGUGAAAAAAAAHEAIbG9uZwQODG5lc3RlZA4ECmFscGhhBAIIYmV0YQQEAAh0ZXh0CghzYW1lAA==",
20568 });
20569 assert_eq!(
20570 commands[0]["entries"]
20571 .as_object()
20572 .expect("entries envelope")
20573 .keys()
20574 .collect::<Vec<_>>(),
20575 vec!["blob", "codec"]
20576 );
20577 assert_eq!(commands[0]["entries"], server_entries);
20578 let wire_entries =
20579 decode_wire_avro_value(&commands[0]["entries"], DEFAULT_CODEC).expect("memo entries");
20580 assert_eq!(wire_entries, entries);
20581
20582 let history = vec![history_event(
20583 "MemoUpserted",
20584 json!({
20585 "sequence": 1,
20586 "entries": server_entries.clone(),
20587 "merged": server_entries,
20588 }),
20589 )];
20590 let replay = workflow_context(history.clone());
20591 replay
20592 .upsert_memo(entries.clone())
20593 .expect("matching replay identity");
20594 assert!(replay.take_commands().expect("replay commands").is_empty());
20595
20596 let changed_types = AvroValue::Map(BTreeMap::from([
20597 ("text".to_string(), AvroValue::Bytes(b"same".to_vec())),
20598 (
20599 "nested".to_string(),
20600 AvroValue::Map(BTreeMap::from([
20601 ("alpha".to_string(), AvroValue::Long(1)),
20602 ("beta".to_string(), AvroValue::Long(2)),
20603 ])),
20604 ),
20605 ("long".to_string(), AvroValue::Double(7.0)),
20606 ("double".to_string(), AvroValue::Long(7)),
20607 ("binary".to_string(), AvroValue::String("same".to_string())),
20608 ]));
20609 let error = workflow_context(history)
20610 .upsert_memo(changed_types)
20611 .expect_err("memo replay identity must preserve Avro value types");
20612 assert!(matches!(
20613 error,
20614 Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20615 ));
20616 }
20617
20618 #[test]
20619 fn workflow_memo_update_rejects_changed_replay_identity_and_invalid_keys() {
20620 let original = encode_value_envelope(&json!({"stage": "original"}), DEFAULT_CODEC)
20621 .expect("memo envelope");
20622 let replay = workflow_context(vec![history_event(
20623 "MemoUpserted",
20624 json!({
20625 "sequence": 1,
20626 "entries": original.clone(),
20627 "merged": original
20628 }),
20629 )]);
20630 let error = replay
20631 .upsert_memo(json!({"stage": "changed"}))
20632 .expect_err("changed memo update must fail replay");
20633 assert!(matches!(
20634 error,
20635 Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20636 ));
20637
20638 let invalid = workflow_context(Vec::new())
20639 .upsert_memo(
20640 json!({"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx": true}),
20641 )
20642 .expect_err("oversized key");
20643 assert!(matches!(invalid, Error::InvalidMemoUpdate(_)));
20644 }
20645
20646 #[test]
20647 fn workflow_memo_replay_distinguishes_signed_zero_identity() {
20648 let negative_zero = AvroValue::Map(BTreeMap::from([(
20649 "reading".to_string(),
20650 AvroValue::Double(-0.0),
20651 )]));
20652 let negative_zero_envelope =
20653 encode_typed_envelope(&negative_zero, DEFAULT_CODEC).expect("negative zero envelope");
20654 let history = vec![history_event(
20655 "MemoUpserted",
20656 json!({
20657 "sequence": 1,
20658 "entries": negative_zero_envelope.clone(),
20659 "merged": negative_zero_envelope,
20660 }),
20661 )];
20662
20663 workflow_context(history.clone())
20664 .upsert_memo(negative_zero)
20665 .expect("matching negative-zero history identity");
20666
20667 let error = workflow_context(history)
20668 .upsert_memo(AvroValue::Map(BTreeMap::from([(
20669 "reading".to_string(),
20670 AvroValue::Double(0.0),
20671 )])))
20672 .expect_err("positive zero must not consume negative-zero memo history");
20673 assert!(matches!(
20674 error,
20675 Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20676 ));
20677 }
20678
20679 #[test]
20680 fn workflow_memo_capability_requires_flag_and_command_advertisement() {
20681 let supported = json!({
20682 "workflow_memo_updates": {"supported": true, "minimum_protocol_version": "1.14"},
20683 "supported_workflow_task_commands": ["complete_workflow", "upsert_memo"]
20684 });
20685 assert!(runtime_supports_workflow_memo_updates(Some(&supported)));
20686 assert!(!runtime_supports_workflow_memo_updates(Some(&json!({
20687 "workflow_memo_updates": {"supported": false},
20688 "supported_workflow_task_commands": ["upsert_memo"]
20689 }))));
20690 assert!(commands_use_workflow_memo_updates(&[json!({
20691 "type": "upsert_memo",
20692 "entries": {"stage": "processing"}
20693 })]));
20694 }
20695
20696 #[test]
20697 fn workflow_task_replay_completes_without_rescheduling_recorded_commands() {
20698 let client = Client::new("http://127.0.0.1:8080").expect("client");
20699 let mut worker = Worker::new(client, "rust-workers");
20700 worker.register_workflow("rust.timer", |ctx, _input| async move {
20701 ctx.sleep(Duration::from_secs(5)).await?;
20702 ctx.activity("after-timer", json!([])).await
20703 });
20704
20705 let task = |history_events| WorkflowTask {
20706 task_id: "wft-rust-timer-1".to_string(),
20707 workflow_command_id: None,
20708 workflow_id: Some("wf-rust-timer".to_string()),
20709 run_id: Some("run-rust-timer".to_string()),
20710 workflow_type: "rust.timer".to_string(),
20711 cancel_requested: false,
20712 payload_codec: DEFAULT_CODEC.to_string(),
20713 arguments: Some(
20714 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
20715 ),
20716 history_events,
20717 total_history_events: None,
20718 history_size_bytes: None,
20719 continue_as_new_recommended: None,
20720 history_budget_pressure: None,
20721 next_history_page_token: None,
20722 workflow_task_attempt: 1,
20723 workflow_signal_id: None,
20724 signal_name: None,
20725 signal_arguments: None,
20726 workflow_update_id: None,
20727 update_name: None,
20728 lease_owner: Some("rust-worker".to_string()),
20729 };
20730
20731 let initial = worker
20732 .execute_workflow_task(task(Vec::new()))
20733 .expect("initial timer task");
20734 assert_eq!(
20735 initial,
20736 vec![json!({"type": "start_timer", "delay_seconds": 5})]
20737 );
20738
20739 let activity_result =
20740 encode_value_envelope(&json!("done"), DEFAULT_CODEC).expect("activity result");
20741 let replayed = worker
20742 .execute_workflow_task(task(vec![
20743 history_event(
20744 "TimerScheduled",
20745 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20746 ),
20747 history_event(
20748 "TimerFired",
20749 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20750 ),
20751 history_event(
20752 "ActivityCompleted",
20753 json!({
20754 "sequence": 2,
20755 "activity_type": "after-timer",
20756 "payload_codec": DEFAULT_CODEC,
20757 "result": activity_result,
20758 }),
20759 ),
20760 ]))
20761 .expect("replayed workflow task");
20762 assert_eq!(replayed.len(), 1);
20763 assert_eq!(replayed[0]["type"], "complete_workflow");
20764 assert_eq!(
20765 decode_wire_value(&replayed[0]["result"], DEFAULT_CODEC).expect("result"),
20766 json!("done")
20767 );
20768 }
20769
20770 #[test]
20771 fn workflow_continue_as_new_emits_arguments_type_and_queue_once() {
20772 let client = Client::new("http://127.0.0.1:8080").expect("client");
20773 let mut worker = Worker::new(client, "rust-workers");
20774 worker.register_workflow("rust.continue", |ctx, _input| async move {
20775 ctx.continue_as_new_with_options(
20776 ContinueAsNewOptions::new()
20777 .workflow_type("rust.next")
20778 .task_queue("next-workers"),
20779 json!([2, {"cursor": "next"}]),
20780 )
20781 });
20782
20783 let commands = worker
20784 .execute_workflow_task(workflow_task("rust.continue", Vec::new(), DEFAULT_CODEC))
20785 .expect("continue-as-new command");
20786
20787 assert_eq!(commands.len(), 1);
20788 assert_eq!(commands[0]["type"], "continue_as_new");
20789 assert_eq!(commands[0]["workflow_type"], "rust.next");
20790 assert_eq!(commands[0]["queue"], "next-workers");
20791 assert_eq!(
20792 decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC)
20793 .expect("continue-as-new arguments"),
20794 json!([2, {"cursor": "next"}])
20795 );
20796 }
20797
20798 #[test]
20799 fn continue_as_new_preserves_typed_arguments() {
20800 let client = Client::new("http://127.0.0.1:8080").expect("client");
20801 let mut worker = Worker::new(client, "rust-workers");
20802 worker.register_workflow_avro_value("rust.typed-continue", |ctx, _input| async move {
20803 ctx.continue_as_new(AvroValue::Array(vec![typed_fidelity_probe()]))?;
20804 unreachable!("continue-as-new returns a control-flow error")
20805 });
20806
20807 let commands = worker
20808 .execute_workflow_task(workflow_task(
20809 "rust.typed-continue",
20810 Vec::new(),
20811 DEFAULT_CODEC,
20812 ))
20813 .expect("typed continue-as-new command");
20814
20815 assert_eq!(commands[0]["type"], "continue_as_new");
20816 assert_eq!(
20817 decode_wire_avro_value(&commands[0]["arguments"], DEFAULT_CODEC)
20818 .expect("typed continue arguments"),
20819 AvroValue::Array(vec![typed_fidelity_probe()])
20820 );
20821 }
20822
20823 #[test]
20824 fn recorded_continue_as_new_is_consumed_without_duplicate_successor_command() {
20825 let client = Client::new("http://127.0.0.1:8080").expect("client");
20826 let mut worker = Worker::new(client, "rust-workers");
20827 worker.register_workflow("rust.continue", |ctx, _input| async move {
20828 ctx.continue_as_new(json!([2]))
20829 });
20830 let task = workflow_task(
20831 "rust.continue",
20832 vec![history_event(
20833 "WorkflowContinuedAsNew",
20834 json!({"sequence": 1, "continued_to_run_id": "run-next"}),
20835 )],
20836 DEFAULT_CODEC,
20837 );
20838
20839 for _worker_restart_or_redelivery in 0..2 {
20840 let commands = worker
20841 .execute_workflow_task(task.clone())
20842 .expect("recorded transition replays");
20843 assert!(
20844 commands.is_empty(),
20845 "replay must not emit another successor"
20846 );
20847 }
20848 }
20849
20850 #[test]
20851 fn continue_as_new_rejects_invalid_overrides_before_emitting_a_command() {
20852 let ctx = workflow_context(Vec::new());
20853 let error = ctx
20854 .continue_as_new_with_options(ContinueAsNewOptions::new().task_queue(" "), json!([1]))
20855 .expect_err("blank queue must be rejected");
20856
20857 let Error::InvalidContinueAsNewOptions(error) = error else {
20858 panic!("expected typed continue-as-new validation error");
20859 };
20860 assert_eq!(error.field, "task_queue");
20861 assert!(ctx.take_commands().expect("commands").is_empty());
20862 }
20863
20864 #[test]
20865 fn workflow_context_exposes_server_history_budget() {
20866 let client = Client::new("http://127.0.0.1:8080").expect("client");
20867 let mut worker = Worker::new(client, "rust-workers");
20868 worker.register_workflow("rust.history-budget", |ctx, _input| async move {
20869 let budget = ctx.history_budget()?;
20870 Ok(json!({
20871 "events": budget.event_count,
20872 "bytes": budget.size_bytes,
20873 "recommended": budget.continue_as_new_recommended,
20874 "pressure": budget.pressure,
20875 }))
20876 });
20877 let task: WorkflowTask = serde_json::from_value(json!({
20878 "task_id": "task-history-budget",
20879 "workflow_type": "rust.history-budget",
20880 "payload_codec": DEFAULT_CODEC,
20881 "history_events": [],
20882 "total_history_events": 480,
20883 "history_size_bytes": 1_048_576,
20884 "continue_as_new_recommended": true,
20885 "history_budget_pressure": "continue_as_new_recommended",
20886 }))
20887 .expect("published workflow task");
20888
20889 let commands = worker
20890 .execute_workflow_task(task)
20891 .expect("history-budget workflow");
20892 let result = decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("result");
20893 assert_eq!(result["events"], 480);
20894 assert_eq!(result["bytes"], 1_048_576);
20895 assert_eq!(result["recommended"], true);
20896 assert_eq!(result["pressure"], "continue_as_new_recommended");
20897 }
20898
20899 #[test]
20900 fn uncaught_workflow_handler_error_emits_terminal_failure_command() {
20901 let client = Client::new("http://127.0.0.1:8080").expect("client");
20902 let mut worker = Worker::new(client, "rust-workers");
20903 worker.register_workflow("rust.failing", |_ctx, _input| async move {
20904 Err(Error::Codec("rust_conformance_failure".to_string()))
20905 });
20906 let task = WorkflowTask {
20907 task_id: "wft-rust-failing-1".to_string(),
20908 workflow_command_id: None,
20909 workflow_id: Some("wf-rust-failing".to_string()),
20910 run_id: Some("run-rust-failing".to_string()),
20911 workflow_type: "rust.failing".to_string(),
20912 cancel_requested: false,
20913 payload_codec: DEFAULT_CODEC.to_string(),
20914 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
20915 history_events: Vec::new(),
20916 total_history_events: Some(0),
20917 history_size_bytes: None,
20918 continue_as_new_recommended: None,
20919 history_budget_pressure: None,
20920 next_history_page_token: None,
20921 workflow_task_attempt: 1,
20922 workflow_signal_id: None,
20923 signal_name: None,
20924 signal_arguments: None,
20925 workflow_update_id: None,
20926 update_name: None,
20927 lease_owner: Some("rust-worker".to_string()),
20928 };
20929
20930 let commands = worker
20931 .execute_workflow_task(task)
20932 .expect("handler failure becomes a workflow command");
20933
20934 assert_eq!(commands.len(), 1);
20935 assert_eq!(commands[0]["type"], "fail_workflow");
20936 assert_eq!(commands[0]["exception_type"], "RustWorkflowError");
20937 assert_eq!(commands[0]["exception_class"], "durable_workflow::Error");
20938 assert_eq!(commands[0]["non_retryable"], false);
20939 assert_eq!(
20940 commands[0]["message"],
20941 "codec error: rust_conformance_failure"
20942 );
20943 assert_eq!(
20944 commands[0]["exception"]["message"],
20945 "codec error: rust_conformance_failure"
20946 );
20947 }
20948
20949 #[test]
20950 fn ordinary_handler_error_preserves_commands_queued_in_the_same_decision() {
20951 let client = Client::new("http://127.0.0.1:8080").expect("client");
20952 let mut worker = Worker::new(client, "rust-workers");
20953 worker.register_workflow("rust.failing-after-side-effect", |ctx, _input| async move {
20954 let _: String = ctx.side_effect(|| "captured".to_string())?;
20955 Err(Error::WorkerLoop("application failure".to_string()))
20956 });
20957
20958 let commands = worker
20959 .execute_workflow_task(workflow_task(
20960 "rust.failing-after-side-effect",
20961 Vec::new(),
20962 DEFAULT_CODEC,
20963 ))
20964 .expect("ordinary failure remains a workflow decision");
20965
20966 assert_eq!(commands.len(), 2);
20967 assert_eq!(commands[0]["type"], "record_side_effect");
20968 assert_eq!(commands[1]["type"], "fail_workflow");
20969 }
20970
20971 #[test]
20972 fn handler_error_cannot_hide_an_unconsumed_committed_side_effect() {
20973 let client = Client::new("http://127.0.0.1:8080").expect("client");
20974 let mut worker = Worker::new(client, "rust-workers");
20975 worker.register_workflow("rust.removed-side-effect", |_ctx, _input| async move {
20976 Err(Error::WorkerLoop("application failure".to_string()))
20977 });
20978 let result =
20979 encode_value_envelope(&json!("committed"), DEFAULT_CODEC).expect("side-effect result");
20980
20981 let error = worker
20982 .execute_workflow_task(workflow_task(
20983 "rust.removed-side-effect",
20984 vec![history_event(
20985 "SideEffectRecorded",
20986 json!({"sequence": 1, "result": result}),
20987 )],
20988 DEFAULT_CODEC,
20989 ))
20990 .expect_err("removed committed history must not become fail_workflow");
20991
20992 let Error::NonDeterministicReplay(failure) = error else {
20993 panic!("expected typed replay failure");
20994 };
20995 assert_eq!(failure.reason, "recorded_commands_unconsumed");
20996 assert_eq!(failure.sequence, Some(1));
20997 assert_eq!(failure.expected.as_deref(), Some("side effect"));
20998 }
20999
21000 #[test]
21001 fn replay_error_discards_side_effect_queued_before_incompatible_marker_check() {
21002 let client = Client::new("http://127.0.0.1:8080").expect("client");
21003 let mut worker = Worker::new(client, "rust-workers");
21004 worker.register_workflow(
21005 "rust.side-effect-before-marker-error",
21006 |ctx, _input| async move {
21007 assert_eq!(ctx.get_version("restart-safe", 1, 1)?, 1);
21008 let _: String = ctx.side_effect(|| "must-not-commit".to_string())?;
21009 ctx.get_version("restart-safe", 2, 2)?;
21010 Ok(Value::Null)
21011 },
21012 );
21013
21014 let error = worker
21015 .execute_workflow_task(workflow_task(
21016 "rust.side-effect-before-marker-error",
21017 vec![history_event(
21018 "VersionMarkerRecorded",
21019 json!({
21020 "sequence": 1,
21021 "change_id": "restart-safe",
21022 "version": 1,
21023 "min_supported": 1,
21024 "max_supported": 1,
21025 }),
21026 )],
21027 DEFAULT_CODEC,
21028 ))
21029 .expect_err("replay error must return no queued workflow commands");
21030
21031 let Error::NonDeterministicReplay(failure) = error else {
21032 panic!("expected typed replay failure");
21033 };
21034 assert_eq!(failure.reason, "version_marker_incompatible_range");
21035 assert_eq!(failure.sequence, Some(1));
21036 }
21037
21038 #[test]
21039 fn workflow_task_replay_keeps_recorded_unfired_timer_pending_without_rescheduling() {
21040 let client = Client::new("http://127.0.0.1:8080").expect("client");
21041 let mut worker = Worker::new(client, "rust-workers");
21042 worker.register_workflow("rust.timer.pending", |ctx, _input| async move {
21043 ctx.sleep(Duration::from_secs(5)).await?;
21044 Ok(json!({"status": "timer fired"}))
21045 });
21046
21047 let task = WorkflowTask {
21048 task_id: "wft-rust-timer-pending".to_string(),
21049 workflow_command_id: None,
21050 workflow_id: Some("wf-rust-timer".to_string()),
21051 run_id: Some("run-rust-timer".to_string()),
21052 workflow_type: "rust.timer.pending".to_string(),
21053 cancel_requested: false,
21054 payload_codec: DEFAULT_CODEC.to_string(),
21055 arguments: Some(
21056 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
21057 ),
21058 history_events: vec![history_event(
21059 "TimerScheduled",
21060 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
21061 )],
21062 total_history_events: Some(1),
21063 history_size_bytes: None,
21064 continue_as_new_recommended: None,
21065 history_budget_pressure: None,
21066 next_history_page_token: None,
21067 workflow_task_attempt: 1,
21068 workflow_signal_id: None,
21069 signal_name: None,
21070 signal_arguments: None,
21071 workflow_update_id: None,
21072 update_name: None,
21073 lease_owner: Some("rust-worker".to_string()),
21074 };
21075
21076 for _redelivery_or_restart in 0..2 {
21077 let commands = worker
21078 .execute_workflow_task(task.clone())
21079 .expect("recorded timer remains pending");
21080 assert!(
21081 commands.is_empty(),
21082 "recorded timer must not be rescheduled"
21083 );
21084 }
21085 }
21086
21087 #[test]
21088 fn workflow_task_rejects_recorded_command_removed_from_workflow_code() {
21089 let client = Client::new("http://127.0.0.1:8080").expect("client");
21090 let mut worker = Worker::new(client, "rust-workers");
21091 worker.register_workflow("rust.timer.removed", |_ctx, _input| async move {
21092 Ok(json!({"status": "completed"}))
21093 });
21094 let task = WorkflowTask {
21095 task_id: "wft-rust-timer-removed".to_string(),
21096 workflow_command_id: None,
21097 workflow_id: Some("wf-rust-timer".to_string()),
21098 run_id: Some("run-rust-timer".to_string()),
21099 workflow_type: "rust.timer.removed".to_string(),
21100 cancel_requested: false,
21101 payload_codec: DEFAULT_CODEC.to_string(),
21102 arguments: Some(
21103 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
21104 ),
21105 history_events: vec![
21106 history_event(
21107 "TimerScheduled",
21108 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
21109 ),
21110 history_event(
21111 "TimerFired",
21112 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
21113 ),
21114 ],
21115 total_history_events: Some(2),
21116 history_size_bytes: None,
21117 continue_as_new_recommended: None,
21118 history_budget_pressure: None,
21119 next_history_page_token: None,
21120 workflow_task_attempt: 1,
21121 workflow_signal_id: None,
21122 signal_name: None,
21123 signal_arguments: None,
21124 workflow_update_id: None,
21125 update_name: None,
21126 lease_owner: Some("rust-worker".to_string()),
21127 };
21128
21129 let Error::NonDeterministicReplay(failure) = worker
21130 .execute_workflow_task(task)
21131 .expect_err("removed timer must fail replay")
21132 else {
21133 panic!("expected typed replay failure");
21134 };
21135 assert_eq!(failure.reason, "recorded_commands_unconsumed");
21136 assert_eq!(failure.sequence, Some(1));
21137 }
21138
21139 #[test]
21140 fn workflow_context_emits_explicit_child_workflow_contract() {
21141 let ctx = WorkflowContext {
21142 state: Arc::new(Mutex::new(
21143 WorkflowState::new_with_identity(
21144 Vec::new(),
21145 Some("wf-parent".to_string()),
21146 Some("run-parent".to_string()),
21147 "parent-workers".to_string(),
21148 DEFAULT_CODEC.to_string(),
21149 None,
21150 )
21151 .expect("workflow state"),
21152 )),
21153 };
21154 let options = ChildWorkflowOptions::new("python-workers")
21155 .parent_close_policy(ParentClosePolicy::RequestCancel)
21156 .retry_policy(ChildWorkflowRetryPolicy {
21157 max_attempts: Some(3),
21158 backoff_seconds: vec![1, 5],
21159 non_retryable_error_types: vec!["ValidationError".to_string()],
21160 })
21161 .execution_timeout_seconds(600)
21162 .run_timeout_seconds(120);
21163 let mut call = Box::pin(ctx.start_child_workflow(
21164 "python.fulfil-order",
21165 options,
21166 json!([{"order_id": "order-42"}]),
21167 ));
21168 let mut task_context = TaskContext::from_waker(noop_waker_ref());
21169
21170 assert!(matches!(
21171 call.as_mut().poll(&mut task_context),
21172 Poll::Pending
21173 ));
21174 let commands = ctx.take_commands().expect("commands");
21175 assert_eq!(commands.len(), 1);
21176 let command = &commands[0];
21177 assert_eq!(command["type"], "start_child_workflow");
21178 assert_eq!(command["workflow_type"], "python.fulfil-order");
21179 assert_eq!(command["queue"], "python-workers");
21180 assert_eq!(command["parent_close_policy"], "request_cancel");
21181 assert_eq!(command["retry_policy"]["max_attempts"], 3);
21182 assert_eq!(command["execution_timeout_seconds"], 600);
21183 assert_eq!(command["run_timeout_seconds"], 120);
21184 assert_eq!(
21185 decode_wire_value(&command["arguments"], DEFAULT_CODEC).expect("child args"),
21186 json!([{"order_id": "order-42"}])
21187 );
21188 }
21189
21190 fn child_parent_worker() -> Worker {
21191 let client = Client::new("http://127.0.0.1:8080").expect("client");
21192 let mut worker = Worker::new(client, "rust-parent-workers");
21193 worker.register_workflow("rust.parent", |ctx, _input| async move {
21194 let child = ctx
21195 .start_child_workflow(
21196 "python.child",
21197 ChildWorkflowOptions::new("python-child-workers")
21198 .parent_close_policy(ParentClosePolicy::Terminate),
21199 json!([{"codec_probe": [1, true, "rust"]}]),
21200 )
21201 .await?;
21202 Ok(json!({
21203 "parent_workflow_id": child.parent.workflow_id,
21204 "parent_run_id": child.parent.run_id,
21205 "child_workflow_id": child.child.workflow_id,
21206 "child_run_id": child.child.run_id,
21207 "child_workflow_type": child.child_workflow_type,
21208 "result": child.result,
21209 }))
21210 });
21211 worker
21212 }
21213
21214 fn child_parent_task(event_type: &str, payload: Value) -> WorkflowTask {
21215 WorkflowTask {
21216 task_id: "wft-child-parent".to_string(),
21217 workflow_command_id: None,
21218 workflow_id: Some("wf-parent".to_string()),
21219 run_id: Some("run-parent".to_string()),
21220 workflow_type: "rust.parent".to_string(),
21221 cancel_requested: false,
21222 payload_codec: DEFAULT_CODEC.to_string(),
21223 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21224 history_events: vec![
21225 HistoryEvent {
21226 event_type: "ChildWorkflowScheduled".to_string(),
21227 payload: json!({
21228 "sequence": 1,
21229 "child_call_id": "call-child",
21230 "child_workflow_instance_id": "wf-child",
21231 "child_workflow_run_id": "run-child",
21232 "child_workflow_type": "python.child",
21233 }),
21234 raw: HashMap::new(),
21235 },
21236 HistoryEvent {
21237 event_type: event_type.to_string(),
21238 payload,
21239 raw: HashMap::new(),
21240 },
21241 ],
21242 total_history_events: Some(2),
21243 history_size_bytes: None,
21244 continue_as_new_recommended: None,
21245 history_budget_pressure: None,
21246 next_history_page_token: None,
21247 workflow_task_attempt: 1,
21248 workflow_signal_id: None,
21249 signal_name: None,
21250 signal_arguments: None,
21251 workflow_update_id: None,
21252 update_name: None,
21253 lease_owner: Some("rust-worker".to_string()),
21254 }
21255 }
21256
21257 #[test]
21258 fn committed_child_result_replays_without_starting_a_duplicate() {
21259 let worker = child_parent_worker();
21260 let task = child_parent_task(
21261 "ChildRunCompleted",
21262 json!({
21263 "sequence": 1,
21264 "child_call_id": "call-child",
21265 "child_workflow_instance_id": "wf-child",
21266 "child_workflow_run_id": "run-child",
21267 "child_workflow_type": "python.child",
21268 "payload_codec": DEFAULT_CODEC,
21269 "result": fixture_envelope(json!({"from":"python","ok":true})),
21270 }),
21271 );
21272
21273 for _restart in 0..2 {
21274 let commands = worker
21275 .execute_workflow_task(task.clone())
21276 .expect("replayed parent task");
21277 assert_eq!(commands.len(), 1);
21278 assert_eq!(commands[0]["type"], "complete_workflow");
21279 assert!(!commands
21280 .iter()
21281 .any(|command| command["type"] == "start_child_workflow"));
21282 let output =
21283 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
21284 assert_eq!(output["parent_workflow_id"], "wf-parent");
21285 assert_eq!(output["parent_run_id"], "run-parent");
21286 assert_eq!(output["child_workflow_id"], "wf-child");
21287 assert_eq!(output["child_run_id"], "run-child");
21288 assert_eq!(output["result"], json!({"from": "python", "ok": true}));
21289 }
21290 }
21291
21292 #[test]
21293 fn typed_child_arguments_and_results_survive_replay() {
21294 let client = Client::new("http://127.0.0.1:8080").expect("client");
21295 let mut worker = Worker::new(client, "rust-parent-workers");
21296 worker.register_workflow_avro_value("rust.typed-parent", |ctx, _input| async move {
21297 let child = ctx
21298 .start_child_workflow_avro_value(
21299 "python.typed-child",
21300 ChildWorkflowOptions::new("python-workers"),
21301 AvroValue::Array(vec![typed_fidelity_probe()]),
21302 )
21303 .await?;
21304 Ok(child.result)
21305 });
21306
21307 let initial = worker
21308 .execute_workflow_task(workflow_task(
21309 "rust.typed-parent",
21310 Vec::new(),
21311 DEFAULT_CODEC,
21312 ))
21313 .expect("typed child start");
21314 assert_eq!(initial[0]["type"], "start_child_workflow");
21315 assert_eq!(
21316 decode_wire_avro_value(&initial[0]["arguments"], DEFAULT_CODEC)
21317 .expect("typed child arguments"),
21318 AvroValue::Array(vec![typed_fidelity_probe()])
21319 );
21320
21321 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
21322 .expect("typed child result");
21323 let task = workflow_task(
21324 "rust.typed-parent",
21325 vec![
21326 history_event(
21327 "ChildWorkflowScheduled",
21328 json!({
21329 "sequence": 1,
21330 "child_call_id": "call-typed",
21331 "child_workflow_instance_id": "wf-child",
21332 "child_workflow_run_id": "run-child",
21333 "child_workflow_type": "python.typed-child",
21334 }),
21335 ),
21336 history_event(
21337 "ChildRunCompleted",
21338 json!({
21339 "sequence": 1,
21340 "child_call_id": "call-typed",
21341 "child_workflow_instance_id": "wf-child",
21342 "child_workflow_run_id": "run-child",
21343 "child_workflow_type": "python.typed-child",
21344 "payload_codec": DEFAULT_CODEC,
21345 "result": result,
21346 }),
21347 ),
21348 ],
21349 DEFAULT_CODEC,
21350 );
21351
21352 let commands = worker
21353 .execute_workflow_task(task)
21354 .expect("typed child replay");
21355 assert_eq!(commands[0]["type"], "complete_workflow");
21356 assert_eq!(
21357 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
21358 .expect("typed parent result"),
21359 typed_fidelity_probe()
21360 );
21361 }
21362
21363 #[test]
21364 fn pending_child_replays_after_restart_without_starting_a_duplicate() {
21365 let worker = child_parent_worker();
21366 let mut task = child_parent_task("unused", Value::Null);
21367 task.history_events.truncate(1);
21368 task.total_history_events = Some(1);
21369
21370 for _redelivery_or_restart in 0..2 {
21371 let commands = worker
21372 .execute_workflow_task(task.clone())
21373 .expect("recorded child remains pending");
21374 assert!(
21375 commands.is_empty(),
21376 "recorded pending child must not be started again"
21377 );
21378 }
21379 }
21380
21381 #[test]
21382 fn child_cancellation_becomes_stable_parent_failure_command() {
21383 let worker = child_parent_worker();
21384 let task = child_parent_task(
21385 "ChildRunCancelled",
21386 json!({
21387 "sequence": 1,
21388 "child_workflow_instance_id": "wf-child",
21389 "child_workflow_run_id": "run-child",
21390 "child_workflow_type": "python.child",
21391 "failure_id": "failure-child",
21392 "failure_category": "cancelled",
21393 "message": "cancelled by parent-close policy",
21394 }),
21395 );
21396
21397 let commands = worker
21398 .execute_workflow_task(task)
21399 .expect("parent settlement");
21400 assert_eq!(commands.len(), 1);
21401 assert_eq!(commands[0]["type"], "fail_workflow");
21402 assert_eq!(commands[0]["exception_type"], "ChildWorkflowCancelled");
21403 assert_eq!(
21404 commands[0]["exception"]["properties"]["reason"],
21405 "cancelled"
21406 );
21407 assert_eq!(
21408 commands[0]["exception"]["properties"]["child_workflow_run_id"],
21409 "run-child"
21410 );
21411 }
21412
21413 #[test]
21414 fn workflow_can_handle_typed_child_failure() {
21415 let client = Client::new("http://127.0.0.1:8080").expect("client");
21416 let mut worker = Worker::new(client, "rust-parent-workers");
21417 worker.register_workflow("rust.handled-parent", |ctx, _input| async move {
21418 match ctx
21419 .start_child_workflow(
21420 "python.child",
21421 ChildWorkflowOptions::new("python-child-workers"),
21422 json!([]),
21423 )
21424 .await
21425 {
21426 Err(Error::ChildWorkflowFailed(failure)) => Ok(json!({
21427 "reason": failure.reason,
21428 "failure_id": failure.failure_id,
21429 "exception_class": failure.exception_class,
21430 "child_run_id": failure.child_workflow_run_id,
21431 })),
21432 Err(error) => Err(error),
21433 Ok(_) => Err(Error::WorkerLoop(
21434 "child unexpectedly succeeded".to_string(),
21435 )),
21436 }
21437 });
21438 let mut task = child_parent_task(
21439 "ChildRunFailed",
21440 json!({
21441 "sequence": 1,
21442 "child_workflow_instance_id": "wf-child",
21443 "child_workflow_run_id": "run-child",
21444 "child_workflow_type": "python.child",
21445 "failure_id": "failure-child",
21446 "failure_category": "child_workflow",
21447 "message": "payment rejected",
21448 "exception": {
21449 "type": "PaymentRejected",
21450 "class": "payments.PaymentRejected",
21451 "message": "payment rejected"
21452 }
21453 }),
21454 );
21455 task.workflow_type = "rust.handled-parent".to_string();
21456
21457 let commands = worker.execute_workflow_task(task).expect("handled failure");
21458 assert_eq!(commands[0]["type"], "complete_workflow");
21459 let output =
21460 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
21461 assert_eq!(output["reason"], "child_workflow");
21462 assert_eq!(output["failure_id"], "failure-child");
21463 assert_eq!(output["exception_class"], "payments.PaymentRejected");
21464 assert_eq!(output["child_run_id"], "run-child");
21465 }
21466
21467 #[test]
21468 fn rust_hello_world_uses_signal_arguments_from_resume_payload() {
21469 let client = Client::new("http://127.0.0.1:8080").expect("client");
21470 let mut worker = Worker::new(client, "rust-workers");
21471
21472 worker.register_workflow("rust.hello_workflow", |ctx, _input| async move {
21473 let signal = ctx.wait_signal("start").await?;
21474 let name = signal
21475 .first()
21476 .and_then(|value| value.as_str())
21477 .unwrap_or("world");
21478 let greeting = ctx.activity("rust.hello_activity", json!([name])).await?;
21479 Ok(json!({
21480 "greeting": greeting,
21481 "language": "rust"
21482 }))
21483 });
21484
21485 let signal_arguments =
21486 encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC).expect("signal arguments");
21487 let task = WorkflowTask {
21488 task_id: "wft-rust-signal-1".to_string(),
21489 workflow_command_id: None,
21490 workflow_id: Some("wf-rust-hello".to_string()),
21491 run_id: Some("run-rust-hello".to_string()),
21492 workflow_type: "rust.hello_workflow".to_string(),
21493 cancel_requested: false,
21494 payload_codec: DEFAULT_CODEC.to_string(),
21495 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21496 history_events: vec![HistoryEvent {
21497 event_type: "SignalReceived".to_string(),
21498 payload: json!({
21499 "signal_id": "sig-rust-1",
21500 "signal_name": "start"
21501 }),
21502 raw: HashMap::new(),
21503 }],
21504 total_history_events: Some(1),
21505 history_size_bytes: None,
21506 continue_as_new_recommended: None,
21507 history_budget_pressure: None,
21508 next_history_page_token: None,
21509 workflow_task_attempt: 1,
21510 workflow_signal_id: Some("sig-rust-1".to_string()),
21511 signal_name: Some("start".to_string()),
21512 signal_arguments: Some(signal_arguments),
21513 workflow_update_id: None,
21514 update_name: None,
21515 lease_owner: Some("rust-worker".to_string()),
21516 };
21517
21518 let commands = worker.execute_workflow_task(task).expect("workflow task");
21519
21520 assert_eq!(commands.len(), 1);
21521 assert_eq!(commands[0]["type"], "schedule_activity");
21522 assert_eq!(commands[0]["activity_type"], "rust.hello_activity");
21523 assert_eq!(
21524 decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC).expect("activity args"),
21525 json!(["Rust"])
21526 );
21527 }
21528
21529 #[test]
21530 fn workflow_task_appends_paginated_history_events() {
21531 let mut task = WorkflowTask {
21532 task_id: "wft-rust-pages-1".to_string(),
21533 workflow_command_id: None,
21534 workflow_id: Some("wf-rust-pages".to_string()),
21535 run_id: Some("run-rust-pages".to_string()),
21536 workflow_type: "rust.hello_workflow".to_string(),
21537 cancel_requested: false,
21538 payload_codec: DEFAULT_CODEC.to_string(),
21539 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21540 history_events: vec![HistoryEvent {
21541 event_type: "WorkflowStarted".to_string(),
21542 payload: json!({}),
21543 raw: HashMap::new(),
21544 }],
21545 total_history_events: Some(3),
21546 history_size_bytes: None,
21547 continue_as_new_recommended: None,
21548 history_budget_pressure: None,
21549 next_history_page_token: Some("MQ==".to_string()),
21550 workflow_task_attempt: 1,
21551 workflow_signal_id: None,
21552 signal_name: None,
21553 signal_arguments: None,
21554 workflow_update_id: None,
21555 update_name: None,
21556 lease_owner: Some("rust-worker".to_string()),
21557 };
21558
21559 task.append_history_page(WorkflowTaskHistoryPage {
21560 history_events: vec![
21561 HistoryEvent {
21562 event_type: "SignalReceived".to_string(),
21563 payload: json!({
21564 "signal_id": "sig-rust-1",
21565 "signal_name": "start",
21566 "arguments": encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC)
21567 .expect("signal arguments")
21568 }),
21569 raw: HashMap::new(),
21570 },
21571 HistoryEvent {
21572 event_type: "MarkerRecorded".to_string(),
21573 payload: json!({"sequence": 3}),
21574 raw: HashMap::new(),
21575 },
21576 ],
21577 total_history_events: Some(3),
21578 next_history_page_token: None,
21579 });
21580
21581 assert_eq!(task.history_events.len(), 3);
21582 assert_eq!(task.total_history_events, Some(3));
21583 assert_eq!(task.next_history_page_token, None);
21584
21585 let signal = task
21586 .history_events
21587 .iter()
21588 .find(|event| event.event_type == "SignalReceived")
21589 .expect("signal event");
21590 assert_eq!(
21591 decode_signal_event_arguments(signal, DEFAULT_CODEC).expect("signal arguments"),
21592 vec![AvroValue::String("Rust".to_string())]
21593 );
21594 }
21595
21596 #[tokio::test]
21597 async fn query_handler_reads_ordered_cross_codec_signals_without_commands() {
21598 let client = Client::new("http://127.0.0.1:8080").expect("client");
21599 let mut worker = Worker::new(client, "rust-workers");
21600 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21601 worker.register_query("counter", "current", |ctx, _args| async move {
21602 let mut count = 0_i64;
21603 for signal in ctx.signal_events() {
21604 let value = signal
21605 .arguments
21606 .first()
21607 .and_then(Value::as_i64)
21608 .unwrap_or_default();
21609 match signal.name.as_str() {
21610 "increment" => count += value,
21611 "set" => count = value,
21612 _ => {}
21613 }
21614 }
21615 Ok(json!(count))
21616 });
21617
21618 let task = QueryTask {
21619 query_task_id: "query-rust-counter".to_string(),
21620 query_task_attempt: 1,
21621 lease_owner: Some("rust-worker".to_string()),
21622 workflow_id: Some("counter-1".to_string()),
21623 run_id: Some("run-counter-1".to_string()),
21624 workflow_type: "counter".to_string(),
21625 query_name: "current".to_string(),
21626 payload_codec: DEFAULT_CODEC.to_string(),
21627 workflow_arguments: Some(
21628 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
21629 ),
21630 query_arguments: Some(
21631 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("query arguments"),
21632 ),
21633 history_events: vec![
21634 HistoryEvent {
21635 event_type: "SignalReceived".to_string(),
21636 payload: json!({
21637 "signal_id": "php-signal-1",
21638 "signal_name": "increment",
21639 "workflow_sequence": 1,
21640 "payload_codec": DEFAULT_CODEC,
21641 "arguments": encode_value_envelope(&json!([3]), DEFAULT_CODEC).expect("php avro signal")
21642 }),
21643 raw: HashMap::new(),
21644 },
21645 HistoryEvent {
21646 event_type: "SignalReceived".to_string(),
21647 payload: json!({
21648 "signal_id": "python-signal-2",
21649 "signal_name": "increment",
21650 "workflow_sequence": 2,
21651 "payload_codec": DEFAULT_CODEC,
21652 "arguments": encode_value_envelope(&json!([5]), DEFAULT_CODEC).expect("python avro signal")
21653 }),
21654 raw: HashMap::new(),
21655 },
21656 HistoryEvent {
21657 event_type: "SignalReceived".to_string(),
21658 payload: json!({
21659 "signal_id": "rust-signal-3",
21660 "signal_name": "set",
21661 "workflow_sequence": 3,
21662 "payload_codec": DEFAULT_CODEC,
21663 "arguments": encode_value_envelope(&json!([0]), DEFAULT_CODEC).expect("rust avro signal")
21664 }),
21665 raw: HashMap::new(),
21666 },
21667 ],
21668 history_export: None,
21669 run_status: Some("completed".to_string()),
21670 };
21671
21672 let result = worker.execute_query_task(task).await.expect("query result");
21673 assert_eq!(result.into_json().expect("query projection"), json!(0));
21674 }
21675
21676 #[tokio::test]
21677 async fn replayed_queries_read_running_completed_and_cold_restarted_instance_state() {
21678 let worker = replay_counter_worker();
21679 let running_history = json!([
21680 {
21681 "type": "ActivityCompleted",
21682 "payload": {
21683 "sequence": 1,
21684 "activity_type": "load-counter",
21685 "payload_codec": DEFAULT_CODEC,
21686 "result": fixture_envelope(json!("loaded"))
21687 }
21688 },
21689 {
21690 "type": "SignalWaitOpened",
21691 "payload": {
21692 "sequence": 3,
21693 "signal_name": "increment"
21694 }
21695 },
21696 {
21697 "type": "SignalReceived",
21698 "payload": {
21699 "signal_id": "signal-3",
21700 "signal_name": "increment",
21701 "workflow_sequence": 2,
21702 "payload_codec": DEFAULT_CODEC,
21703 "arguments": fixture_envelope(json!([3]))
21704 }
21705 },
21706 {
21707 "type": "SignalApplied",
21708 "payload": {
21709 "sequence": 3,
21710 "signal_id": "signal-3",
21711 "signal_name": "increment",
21712 "payload_codec": DEFAULT_CODEC,
21713 "value": fixture_envelope(json!([3]))
21714 }
21715 }
21716 ]);
21717
21718 let running = worker
21719 .execute_query_task(replay_counter_query(
21720 "current",
21721 running_history.clone(),
21722 "running",
21723 ))
21724 .await
21725 .expect("running replay query");
21726 assert_eq!(
21727 running.clone().into_json().expect("query projection"),
21728 json!({"loaded": "loaded", "count": 3, "finished": false})
21729 );
21730
21731 let detached = worker
21732 .execute_query_task(replay_counter_query(
21733 "detached-mutation",
21734 running_history.clone(),
21735 "running",
21736 ))
21737 .await
21738 .expect("query mutates only its detached state clone");
21739 assert_eq!(detached.into_json().expect("query projection"), json!(999));
21740 let failed = worker
21741 .execute_query_task(replay_counter_query(
21742 "failed-mutation",
21743 running_history.clone(),
21744 "running",
21745 ))
21746 .await
21747 .expect_err("failed query");
21748 assert_eq!(failed.reason, "query_rejected");
21749 let unchanged = worker
21750 .execute_query_task(replay_counter_query("current", running_history, "running"))
21751 .await
21752 .expect("later query reconstructs unchanged state");
21753 assert_eq!(unchanged, running);
21754
21755 let restarted_worker = replay_counter_worker();
21756 let empty_arguments = fixture_envelope(json!([]));
21757 let loaded_result = fixture_envelope(json!("loaded"));
21758 let signal_three = fixture_blob(json!([3]));
21759 let signal_five = fixture_blob(json!([5]));
21760 let restarted_task: QueryTask = serde_json::from_value(json!({
21761 "query_task_id": "query-after-restart",
21762 "workflow_id": "counter-1",
21763 "run_id": "run-counter-1",
21764 "workflow_type": "replay-counter",
21765 "query_name": "current",
21766 "payload_codec": DEFAULT_CODEC,
21767 "workflow_arguments": empty_arguments.clone(),
21768 "query_arguments": empty_arguments,
21769 "history_events": [],
21770 "history_export": {
21771 "payloads": {"codec": DEFAULT_CODEC},
21772 "history_events": [
21773 {
21774 "type": "ActivityCompleted",
21775 "payload": {
21776 "sequence": 1,
21777 "activity_type": "load-counter",
21778 "payload_codec": DEFAULT_CODEC,
21779 "result": null
21780 }
21781 },
21782 {
21783 "type": "SignalWaitOpened",
21784 "payload": {
21785 "sequence": 3,
21786 "signal_name": "increment"
21787 }
21788 },
21789 {
21790 "type": "SignalReceived",
21791 "payload": {
21792 "signal_id": "signal-3",
21793 "signal_name": "increment",
21794 "workflow_sequence": 2
21795 }
21796 },
21797 {
21798 "type": "SignalApplied",
21799 "payload": {
21800 "sequence": 3,
21801 "signal_id": "signal-3",
21802 "signal_name": "increment"
21803 }
21804 },
21805 {
21806 "type": "SignalWaitOpened",
21807 "payload": {
21808 "sequence": 5,
21809 "signal_name": "increment"
21810 }
21811 },
21812 {
21813 "type": "SignalReceived",
21814 "payload": {
21815 "signal_id": "signal-5",
21816 "signal_name": "increment",
21817 "workflow_sequence": 4
21818 }
21819 },
21820 {
21821 "type": "SignalApplied",
21822 "payload": {
21823 "sequence": 5,
21824 "signal_id": "signal-5",
21825 "signal_name": "increment"
21826 }
21827 }
21828 ],
21829 "activities": [{
21830 "sequence": 1,
21831 "activity_type": "load-counter",
21832 "payload_codec": DEFAULT_CODEC,
21833 "result": loaded_result
21834 }],
21835 "signals": [
21836 {
21837 "id": "signal-3",
21838 "name": "increment",
21839 "workflow_sequence": 2,
21840 "payload_codec": DEFAULT_CODEC,
21841 "arguments": signal_three
21842 },
21843 {
21844 "id": "signal-5",
21845 "name": "increment",
21846 "workflow_sequence": 4,
21847 "payload_codec": DEFAULT_CODEC,
21848 "arguments": signal_five
21849 }
21850 ]
21851 },
21852 "run_status": "completed"
21853 }))
21854 .expect("cold replay query task");
21855 let completed = restarted_worker
21856 .execute_query_task(restarted_task)
21857 .await
21858 .expect("completed cold replay query");
21859 assert_eq!(
21860 completed.into_json().expect("query projection"),
21861 json!({"loaded": "loaded", "count": 8, "finished": true})
21862 );
21863 }
21864
21865 #[tokio::test]
21866 async fn replayed_query_replay_failures_are_machine_readable() {
21867 let worker = replay_counter_worker();
21868 let task = replay_counter_query(
21869 "current",
21870 json!([{
21871 "type": "ActivityCompleted",
21872 "payload": {
21873 "sequence": 1,
21874 "payload_codec": DEFAULT_CODEC,
21875 "result": {"codec": DEFAULT_CODEC, "blob": "{"}
21876 }
21877 }]),
21878 "running",
21879 );
21880 let failure = worker
21881 .execute_query_task(task)
21882 .await
21883 .expect_err("invalid replay history payload");
21884 assert_eq!(failure.reason, "query_payload_decode_failed");
21885 assert_eq!(failure.failure_type, "QueryPayloadDecodeFailed");
21886 assert!(failure.message.contains("invalid_payload_framing"));
21887 }
21888
21889 #[tokio::test]
21890 async fn query_task_restores_compact_history_from_export() {
21891 let client = Client::new("http://127.0.0.1:8080").expect("client");
21892 let mut worker = Worker::new(client, "rust-workers");
21893 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21894 worker.register_query("counter", "current", |ctx, _args| async move {
21895 Ok(json!(ctx.signals("increment")[0][0]))
21896 });
21897 let empty_arguments = fixture_envelope(json!([]));
21898 let exported_signal = fixture_blob(json!([9]));
21899 let task: QueryTask = serde_json::from_value(json!({
21900 "query_task_id": "query-export",
21901 "workflow_type": "counter",
21902 "query_name": "current",
21903 "payload_codec": DEFAULT_CODEC,
21904 "workflow_arguments": empty_arguments.clone(),
21905 "query_arguments": empty_arguments,
21906 "history_events": [],
21907 "history_export": {
21908 "payloads": {"codec": DEFAULT_CODEC},
21909 "history_events": [{
21910 "type": "SignalReceived",
21911 "payload": {"signal_id": "signal-export", "signal_name": "increment"}
21912 }],
21913 "signals": [{
21914 "id": "signal-export",
21915 "name": "increment",
21916 "status": "applied",
21917 "workflow_sequence": 1,
21918 "payload_codec": DEFAULT_CODEC,
21919 "arguments": exported_signal
21920 }]
21921 }
21922 }))
21923 .expect("query task");
21924
21925 let result = worker.execute_query_task(task).await.expect("query result");
21926 assert_eq!(result.into_json().expect("query projection"), json!(9));
21927 }
21928
21929 #[tokio::test]
21930 async fn query_task_failures_have_stable_reasons() {
21931 let client = Client::new("http://127.0.0.1:8080").expect("client");
21932 let mut worker = Worker::new(client, "rust-workers");
21933 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21934 worker.register_query(
21935 "counter",
21936 "current",
21937 |_ctx, _args| async move { Ok(json!(0)) },
21938 );
21939
21940 let base_task = QueryTask {
21941 query_task_id: "query-errors".to_string(),
21942 query_task_attempt: 1,
21943 lease_owner: None,
21944 workflow_id: Some("counter-errors".to_string()),
21945 run_id: Some("run-errors".to_string()),
21946 workflow_type: "counter".to_string(),
21947 query_name: "missing".to_string(),
21948 payload_codec: DEFAULT_CODEC.to_string(),
21949 workflow_arguments: Some(fixture_envelope(json!([]))),
21950 query_arguments: Some(fixture_envelope(json!([]))),
21951 history_events: Vec::new(),
21952 history_export: None,
21953 run_status: Some("running".to_string()),
21954 };
21955
21956 let unknown = worker
21957 .execute_query_task(base_task.clone())
21958 .await
21959 .expect_err("unknown query");
21960 assert_eq!(unknown.reason, "rejected_unknown_query");
21961
21962 let mut malformed = base_task;
21963 malformed.query_name = "current".to_string();
21964 malformed.query_arguments = Some(json!({"codec": DEFAULT_CODEC, "blob": "{"}));
21965 let malformed = worker
21966 .execute_query_task(malformed)
21967 .await
21968 .expect_err("malformed payload");
21969 assert_eq!(malformed.reason, "query_payload_decode_failed");
21970
21971 let client = Client::new("http://127.0.0.1:8080").expect("client");
21972 let mut unavailable_worker = Worker::new(client, "rust-workers");
21973 unavailable_worker
21974 .register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21975 let empty_arguments = fixture_envelope(json!([]));
21976 let unavailable_task: QueryTask = serde_json::from_value(json!({
21977 "query_task_id": "query-unavailable",
21978 "workflow_type": "counter",
21979 "query_name": "current",
21980 "payload_codec": DEFAULT_CODEC,
21981 "workflow_arguments": empty_arguments.clone(),
21982 "query_arguments": empty_arguments
21983 }))
21984 .expect("query task");
21985 let unavailable = unavailable_worker
21986 .execute_query_task(unavailable_task)
21987 .await
21988 .expect_err("query handler unavailable");
21989 assert_eq!(unavailable.reason, "query_handler_unavailable");
21990 }
21991
21992 #[tokio::test]
21993 async fn client_query_decodes_result_and_typed_failure() {
21994 let server = MockWorkerServer::start();
21995 let client = Client::builder(server.base_url())
21996 .timeout(Duration::from_secs(2))
21997 .build()
21998 .expect("client");
21999
22000 let result = client
22001 .query_workflow("counter-1", "current", json!([]))
22002 .await
22003 .expect("query result");
22004 assert_eq!(result, json!({"count": 8}));
22005
22006 let error = client
22007 .query_workflow("counter-1", "missing", json!([]))
22008 .await
22009 .expect_err("unknown query");
22010 let Error::QueryFailed(failure) = error else {
22011 panic!("expected typed query failure");
22012 };
22013 assert_eq!(failure.status, 404);
22014 assert_eq!(failure.reason, "rejected_unknown_query");
22015 }
22016
22017 #[tokio::test]
22018 async fn public_client_surfaces_send_and_receive_lossless_avro_values() {
22019 let server = MockWorkerServer::start();
22020 let client = Client::builder(server.base_url())
22021 .timeout(Duration::from_secs(2))
22022 .build()
22023 .expect("client");
22024 let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
22025
22026 client
22027 .start_workflow(
22028 "typed.echo",
22029 "rust-workers",
22030 "typed-start",
22031 arguments.clone(),
22032 )
22033 .await
22034 .expect("typed workflow start");
22035 assert_eq!(
22036 decode_wire_avro_value(
22037 &server.request_body("/api/workflows")["input"],
22038 DEFAULT_CODEC,
22039 )
22040 .expect("typed start input"),
22041 arguments
22042 );
22043
22044 client
22045 .signal_workflow("typed-1", "changed", arguments.clone())
22046 .await
22047 .expect("typed signal");
22048 assert_eq!(
22049 decode_wire_avro_value(
22050 &server.request_body("/api/workflows/typed-1/signal/changed")["input"],
22051 DEFAULT_CODEC,
22052 )
22053 .expect("typed signal input"),
22054 arguments
22055 );
22056
22057 assert_eq!(
22058 client
22059 .query_workflow_avro_value("typed-1", "inspect", arguments.clone())
22060 .await
22061 .expect("typed query"),
22062 typed_fidelity_probe()
22063 );
22064 assert_eq!(
22065 decode_wire_avro_value(
22066 &server.request_body("/api/workflows/typed-1/query/inspect")["input"],
22067 DEFAULT_CODEC,
22068 )
22069 .expect("typed query input"),
22070 arguments
22071 );
22072
22073 assert_eq!(
22074 client
22075 .update_workflow_avro_value(
22076 "typed-1",
22077 "replace",
22078 arguments.clone(),
22079 Some("typed-request"),
22080 )
22081 .await
22082 .expect("typed update"),
22083 typed_fidelity_probe()
22084 );
22085 let update = server.request_body("/api/workflows/typed-1/update/replace");
22086 assert_eq!(update["request_id"], "typed-request");
22087 assert_eq!(
22088 decode_wire_avro_value(&update["input"], DEFAULT_CODEC).expect("typed update input"),
22089 arguments
22090 );
22091
22092 let handle = WorkflowHandle {
22093 client: client.clone(),
22094 workflow_id: "typed-1".to_string(),
22095 run_id: Some("run-typed-1".to_string()),
22096 workflow_type: "typed.echo".to_string(),
22097 };
22098 assert_eq!(
22099 handle
22100 .result_avro_value(WorkflowResultOptions::default())
22101 .await
22102 .expect("typed workflow result"),
22103 typed_fidelity_probe()
22104 );
22105
22106 client
22107 .complete_activity_task(
22108 "activity-typed",
22109 "attempt-typed",
22110 "rust-worker",
22111 typed_fidelity_probe(),
22112 DEFAULT_CODEC,
22113 )
22114 .await
22115 .expect("typed activity completion");
22116 assert_eq!(
22117 decode_wire_avro_value(
22118 &server.request_body("/api/worker/activity-tasks/activity-typed/complete")
22119 ["result"],
22120 DEFAULT_CODEC,
22121 )
22122 .expect("typed activity result"),
22123 typed_fidelity_probe()
22124 );
22125 client
22126 .fail_activity_task(
22127 "activity-typed",
22128 "attempt-typed",
22129 "rust-worker",
22130 "typed failure",
22131 true,
22132 )
22133 .await
22134 .expect("activity failure");
22135 }
22136
22137 #[tokio::test]
22138 async fn lifecycle_commands_support_instance_and_selected_run_targets() {
22139 let server = MockWorkerServer::start();
22140 let client = Client::builder(server.base_url())
22141 .timeout(Duration::from_secs(2))
22142 .build()
22143 .expect("client");
22144
22145 let options = WorkflowCommandOptions::new()
22146 .reason("cleanup requested")
22147 .request_id("cancel-17");
22148 let cancelled = client
22149 .cancel_workflow("wf-lifecycle", options)
22150 .await
22151 .expect("instance cancellation");
22152 assert_eq!(cancelled.command, WorkflowCommandKind::Cancel);
22153 assert_eq!(cancelled.run_id.as_deref(), Some("run-current"));
22154 assert_eq!(cancelled.outcome.as_deref(), Some("cancelled"));
22155 assert_eq!(
22156 server.request_body("/api/workflows/wf-lifecycle/cancel"),
22157 json!({"reason":"cleanup requested","request_id":"cancel-17"})
22158 );
22159
22160 let terminated = client
22161 .terminate_workflow(
22162 "wf-lifecycle",
22163 WorkflowCommandOptions::new().reason("forced stop"),
22164 )
22165 .await
22166 .expect("instance termination");
22167 assert_eq!(terminated.command, WorkflowCommandKind::Terminate);
22168 assert_eq!(terminated.outcome.as_deref(), Some("terminated"));
22169
22170 client
22171 .cancel_workflow_run(
22172 "wf-lifecycle",
22173 "run-current",
22174 WorkflowCommandOptions::default(),
22175 )
22176 .await
22177 .expect("selected run cancellation");
22178 client
22179 .terminate_workflow_run(
22180 "wf-lifecycle",
22181 "run-current",
22182 WorkflowCommandOptions::default(),
22183 )
22184 .await
22185 .expect("selected run termination");
22186
22187 for (command, error) in [
22188 (
22189 WorkflowCommandKind::Cancel,
22190 client
22191 .cancel_workflow_run(
22192 "wf-lifecycle",
22193 "run-stale",
22194 WorkflowCommandOptions::default(),
22195 )
22196 .await
22197 .expect_err("stale cancellation must be rejected"),
22198 ),
22199 (
22200 WorkflowCommandKind::Terminate,
22201 client
22202 .terminate_workflow_run(
22203 "wf-lifecycle",
22204 "run-stale",
22205 WorkflowCommandOptions::default(),
22206 )
22207 .await
22208 .expect_err("stale termination must be rejected"),
22209 ),
22210 ] {
22211 let Error::WorkflowCommandRejected(rejection) = error else {
22212 panic!("expected typed command rejection");
22213 };
22214 assert_eq!(rejection.command, command);
22215 assert_eq!(rejection.status, 409);
22216 assert_eq!(rejection.reason, "historical_run_command_rejected");
22217 assert_eq!(rejection.run_id.as_deref(), Some("run-stale"));
22218 assert_eq!(rejection.target_scope.as_deref(), Some("run"));
22219 }
22220 }
22221
22222 #[tokio::test]
22223 async fn workflow_start_options_send_server_enforced_deadlines() {
22224 let server = MockWorkerServer::start();
22225 let client = Client::builder(server.base_url())
22226 .timeout(Duration::from_secs(2))
22227 .build()
22228 .expect("client");
22229
22230 let handle = client
22231 .start_workflow_with_options(
22232 "rust.timeout",
22233 "rust-timeouts",
22234 "wf-start-options",
22235 WorkflowStartOptions::new()
22236 .execution_timeout_seconds(30)
22237 .run_timeout_seconds(1),
22238 json!([]),
22239 )
22240 .await
22241 .expect("workflow start");
22242
22243 assert_eq!(handle.run_id.as_deref(), Some("run-start-options"));
22244 let body = server.request_body("/api/workflows");
22245 assert_eq!(body["execution_timeout_seconds"], 30);
22246 assert_eq!(body["run_timeout_seconds"], 1);
22247
22248 let invalid = client
22249 .start_workflow_with_options(
22250 "rust.timeout",
22251 "rust-timeouts",
22252 "wf-invalid-options",
22253 WorkflowStartOptions::new()
22254 .execution_timeout_seconds(1)
22255 .run_timeout_seconds(2),
22256 json!([]),
22257 )
22258 .await
22259 .expect_err("invalid deadline ordering");
22260 assert!(invalid
22261 .to_string()
22262 .contains("run_timeout_seconds cannot exceed execution_timeout_seconds"));
22263 }
22264
22265 #[tokio::test]
22266 async fn workflow_result_returns_each_typed_terminal_outcome() {
22267 let server = MockWorkerServer::start();
22268 let client = Client::builder(server.base_url())
22269 .timeout(Duration::from_secs(2))
22270 .build()
22271 .expect("client");
22272 let options = WorkflowResultOptions {
22273 poll_interval: Duration::ZERO,
22274 timeout: Duration::from_secs(1),
22275 };
22276
22277 let failed = WorkflowHandle {
22278 client: client.clone(),
22279 workflow_id: "wf-failed".to_string(),
22280 run_id: Some("run-failed".to_string()),
22281 workflow_type: "failure".to_string(),
22282 }
22283 .result(options)
22284 .await
22285 .expect_err("failed outcome");
22286 let Error::WorkflowFailed(failure) = failed else {
22287 panic!("expected WorkflowFailed");
22288 };
22289 assert_eq!(failure.workflow_id, "wf-failed");
22290 assert_eq!(failure.run_id.as_deref(), Some("run-failed"));
22291 assert_eq!(failure.failure_id.as_deref(), Some("failure-17"));
22292 assert_eq!(failure.failure_category.as_deref(), Some("application"));
22293 assert_eq!(failure.exception_type.as_deref(), Some("PaymentError"));
22294 assert_eq!(
22295 failure.exception_class.as_deref(),
22296 Some("billing::PaymentError")
22297 );
22298 assert_eq!(failure.non_retryable, Some(true));
22299
22300 for (workflow_id, expected_kind, expected_reason) in [
22301 (
22302 "wf-cancelled",
22303 WorkflowTerminalKind::Cancelled,
22304 "cleanup requested",
22305 ),
22306 (
22307 "wf-terminated",
22308 WorkflowTerminalKind::Terminated,
22309 "forced stop",
22310 ),
22311 (
22312 "wf-timed-out",
22313 WorkflowTerminalKind::TimedOut,
22314 "run_timeout",
22315 ),
22316 ] {
22317 let error = WorkflowHandle {
22318 client: client.clone(),
22319 workflow_id: workflow_id.to_string(),
22320 run_id: None,
22321 workflow_type: "terminal".to_string(),
22322 }
22323 .result(options)
22324 .await
22325 .expect_err("typed terminal outcome");
22326 let outcome = match error {
22327 Error::WorkflowCancelled(outcome) => outcome,
22328 Error::WorkflowTerminated(outcome) => outcome,
22329 Error::WorkflowTimedOut(outcome) => outcome,
22330 other => panic!("unexpected terminal error: {other}"),
22331 };
22332 assert_eq!(outcome.kind, expected_kind);
22333 assert_eq!(outcome.workflow_id, workflow_id);
22334 assert_eq!(outcome.reason, expected_reason);
22335 }
22336
22337 let wait_timeout = WorkflowHandle {
22338 client,
22339 workflow_id: "wf-waiting".to_string(),
22340 run_id: Some("run-waiting".to_string()),
22341 workflow_type: "waiting".to_string(),
22342 }
22343 .result(WorkflowResultOptions {
22344 poll_interval: Duration::ZERO,
22345 timeout: Duration::ZERO,
22346 })
22347 .await
22348 .expect_err("client wait timeout");
22349 let Error::WorkflowTimedOut(timeout) = wait_timeout else {
22350 panic!("expected typed client timeout");
22351 };
22352 assert_eq!(timeout.reason, "result_wait_timeout");
22353 assert_eq!(timeout.failure_category.as_deref(), Some("client_timeout"));
22354 assert_eq!(timeout.run_id.as_deref(), Some("run-waiting"));
22355 }
22356
22357 #[tokio::test]
22358 async fn workflow_result_follows_chain_and_selected_result_preserves_history() {
22359 let server = MockWorkerServer::start();
22360 let client = Client::builder(server.base_url())
22361 .timeout(Duration::from_secs(2))
22362 .build()
22363 .expect("client");
22364
22365 let handle = WorkflowHandle {
22366 client,
22367 workflow_id: "wf-selected".to_string(),
22368 run_id: Some("run-selected".to_string()),
22369 workflow_type: "selected".to_string(),
22370 };
22371 let options = WorkflowResultOptions {
22372 poll_interval: Duration::ZERO,
22373 timeout: Duration::from_secs(1),
22374 };
22375
22376 let current = handle
22377 .result(options)
22378 .await
22379 .expect("instance result follows the current run");
22380 assert_eq!(current, json!("current run output"));
22381
22382 let error = handle
22383 .result_selected_run(options)
22384 .await
22385 .expect_err("the selected run is cancelled even though the current run completed");
22386
22387 let Error::WorkflowCancelled(outcome) = error else {
22388 panic!("expected selected run cancellation");
22389 };
22390 assert_eq!(outcome.run_id.as_deref(), Some("run-selected"));
22391 assert_eq!(outcome.reason, "selected run cancelled");
22392 assert_eq!(
22393 server.request_count("/api/workflows/wf-selected/runs/run-selected"),
22394 1
22395 );
22396 assert_eq!(server.request_count("/api/workflows/wf-selected"), 1);
22397 }
22398
22399 #[tokio::test]
22400 async fn poll_responses_decode_http_conflict_drain_as_a_stable_stop() {
22401 let server = MockWorkerServer::draining_polls();
22402 let client = Client::builder(server.base_url())
22403 .timeout(Duration::from_secs(2))
22404 .build()
22405 .expect("client");
22406
22407 let workflow = client
22408 .poll_workflow_task_response("draining-worker", "rust-workers", Duration::ZERO)
22409 .await
22410 .expect("workflow drain response");
22411 let activity = client
22412 .poll_activity_task_response("draining-worker", "rust-workers", Duration::ZERO)
22413 .await
22414 .expect("activity drain response");
22415 let query = client
22416 .poll_query_task_response("draining-worker", "rust-workers", Duration::ZERO)
22417 .await
22418 .expect("query drain response");
22419
22420 for outcome in [workflow.outcome(), activity.outcome(), query.outcome()] {
22421 assert_eq!(
22422 outcome,
22423 WorkerPollOutcome::Stop {
22424 poll_status: Some("draining".to_string()),
22425 reason: Some("worker_draining".to_string()),
22426 }
22427 );
22428 }
22429
22430 assert!(client
22431 .poll_workflow_task("draining-worker", "rust-workers", Duration::ZERO)
22432 .await
22433 .expect("compatibility poll")
22434 .is_none());
22435 }
22436
22437 #[tokio::test]
22438 async fn managed_worker_honors_drain_stop_for_every_task_family() {
22439 let server = MockWorkerServer::draining_polls();
22440 let client = Client::builder(server.base_url())
22441 .timeout(Duration::from_secs(2))
22442 .build()
22443 .expect("client");
22444
22445 let mut workflow_worker = Worker::new(client.clone(), "rust-workers")
22446 .worker_id("draining-workflow-worker")
22447 .poll_timeout(Duration::ZERO);
22448 workflow_worker.register_workflow("counter", |_ctx, _args| async { Ok(Value::Null) });
22449 workflow_worker
22450 .run()
22451 .await
22452 .expect("workflow drain is a clean stop");
22453
22454 let mut activity_worker = Worker::new(client.clone(), "rust-workers")
22455 .worker_id("draining-activity-worker")
22456 .poll_timeout(Duration::ZERO);
22457 activity_worker.register_activity("write", |_ctx, _args| async { Ok(Value::Null) });
22458 activity_worker
22459 .run()
22460 .await
22461 .expect("activity drain is a clean stop");
22462
22463 let mut query_worker = Worker::new(client, "rust-workers")
22464 .worker_id("draining-query-worker")
22465 .poll_timeout(Duration::ZERO);
22466 query_worker.register_query("counter", "current", |_ctx, _args| async {
22467 Ok(Value::Null)
22468 });
22469 query_worker
22470 .run()
22471 .await
22472 .expect("query drain is a clean stop");
22473 }
22474
22475 #[tokio::test]
22476 async fn activity_cancellation_and_late_completion_remain_machine_readable() {
22477 let server = MockWorkerServer::start();
22478 let client = Client::builder(server.base_url())
22479 .timeout(Duration::from_secs(2))
22480 .build()
22481 .expect("client");
22482
22483 let heartbeat = client
22484 .heartbeat_activity_task(
22485 "activity-cancel",
22486 "attempt-cancel",
22487 "rust-worker",
22488 typed_fidelity_probe(),
22489 )
22490 .await
22491 .expect("cancellation heartbeat");
22492 assert!(heartbeat.cancel_requested);
22493 assert!(heartbeat.should_stop());
22494 assert_eq!(heartbeat.reason.as_deref(), Some("run_cancelled"));
22495 assert_eq!(heartbeat.run_closed_reason.as_deref(), Some("cancelled"));
22496 let heartbeat_body =
22497 server.request_body("/api/worker/activity-tasks/activity-cancel/heartbeat");
22498 assert_eq!(heartbeat_body["details"]["codec"], DEFAULT_CODEC);
22499 assert_eq!(
22500 decode_wire_avro_value(&heartbeat_body["details"], DEFAULT_CODEC)
22501 .expect("typed heartbeat details"),
22502 typed_fidelity_probe()
22503 );
22504
22505 let error = client
22506 .complete_activity_task(
22507 "activity-cancel",
22508 "attempt-cancel",
22509 "rust-worker",
22510 json!({"late":true}),
22511 DEFAULT_CODEC,
22512 )
22513 .await
22514 .expect_err("late completion must be refused");
22515 assert!(activity_task_rejection_is_final(&error));
22516 let Error::ActivityTaskRejected(rejection) = error else {
22517 panic!("expected typed activity rejection");
22518 };
22519 assert_eq!(rejection.status, 409);
22520 assert_eq!(rejection.reason, "run_cancelled");
22521 assert!(rejection.cancel_requested);
22522 assert_eq!(rejection.can_continue, Some(false));
22523 }
22524
22525 #[tokio::test]
22526 async fn managed_worker_survives_late_completion_and_restart_during_cancellation() {
22527 let server = MockWorkerServer::cancelled_activity();
22528 let client = Client::builder(server.base_url())
22529 .timeout(Duration::from_secs(2))
22530 .build()
22531 .expect("client");
22532 let cancellation_observed = Arc::new(AtomicBool::new(false));
22533 let observed = Arc::clone(&cancellation_observed);
22534 let mut worker = Worker::new(client.clone(), "rust-workers")
22535 .worker_id("rust-cancel-worker")
22536 .poll_timeout(Duration::from_millis(10));
22537 worker.register_activity("cancel-aware", move |ctx, _args| {
22538 let observed = Arc::clone(&observed);
22539 async move {
22540 let heartbeat = ctx.heartbeat(json!({"stage":"running"})).await?;
22541 observed.store(heartbeat.should_stop(), Ordering::SeqCst);
22542 Ok(json!({"late":"completion"}))
22543 }
22544 });
22545
22546 assert_eq!(
22547 worker.run_once().await.expect("cancelled attempt handled"),
22548 1
22549 );
22550 assert!(cancellation_observed.load(Ordering::SeqCst));
22551 assert_eq!(
22552 server.request_count("/api/worker/activity-tasks/activity-cancel/complete"),
22553 1
22554 );
22555
22556 let mut restarted = Worker::new(client, "rust-workers")
22557 .worker_id("rust-cancel-worker-restarted")
22558 .poll_timeout(Duration::from_millis(10));
22559 restarted.register_activity("cancel-aware", |_ctx, _args| async move { Ok(Value::Null) });
22560 assert_eq!(
22561 restarted
22562 .run_once()
22563 .await
22564 .expect("replacement worker continues polling"),
22565 0
22566 );
22567 }
22568
22569 #[tokio::test]
22570 async fn managed_worker_absorbs_selected_run_terminal_timeout_completion_race() {
22571 let response = r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"outcome":"completed","recorded":false,"run_id":"run-selected-timeout","run_status":"failed","created_task_ids":[],"reason":"run_timed_out"}"#;
22572 let server = MockWorkerServer::workflow_completion("409 Conflict", response);
22573 let client = Client::builder(server.base_url())
22574 .timeout(Duration::from_secs(2))
22575 .build()
22576 .expect("client");
22577
22578 let direct_error = client
22579 .complete_workflow_task(
22580 "workflow-timeout-task",
22581 "timeout-worker",
22582 3,
22583 vec![json!({
22584 "type": "complete_workflow",
22585 "result": fixture_envelope(Value::Null)
22586 })],
22587 )
22588 .await
22589 .expect_err("the low-level client preserves the completion rejection");
22590 let Error::Http { status, body } = direct_error else {
22591 panic!("expected the original HTTP completion rejection");
22592 };
22593 assert_eq!(status, reqwest::StatusCode::CONFLICT);
22594 assert_eq!(
22595 serde_json::from_str::<Value>(&body).expect("response body")["reason"],
22596 "run_timed_out"
22597 );
22598
22599 let mut worker = Worker::new(client, "rust-workers")
22600 .worker_id("timeout-worker")
22601 .poll_timeout(Duration::from_millis(10));
22602 worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
22603 Ok(json!({"late": "result"}))
22604 });
22605
22606 assert_eq!(
22607 worker
22608 .run_once()
22609 .await
22610 .expect("authoritative selected-run timeout settles the tick"),
22611 1
22612 );
22613 assert_eq!(
22614 server.request_count("/api/worker/workflow-tasks/workflow-timeout-task/complete"),
22615 2,
22616 "both the direct client proof and managed worker must see the rejection"
22617 );
22618 }
22619
22620 #[tokio::test]
22621 async fn managed_worker_does_not_swallow_nearby_completion_errors() {
22622 for (name, status, response) in [
22623 ("bare conflict", "409 Conflict", r#"{"message":"conflict"}"#),
22624 (
22625 "command was recorded",
22626 "409 Conflict",
22627 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":true,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22628 ),
22629 (
22630 "lease conflict",
22631 "409 Conflict",
22632 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"lease_expired"}"#,
22633 ),
22634 (
22635 "nonterminal run",
22636 "409 Conflict",
22637 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"waiting","reason":"run_timed_out"}"#,
22638 ),
22639 (
22640 "different selected run",
22641 "409 Conflict",
22642 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-reused-workflow-current","run_status":"failed","reason":"run_timed_out"}"#,
22643 ),
22644 (
22645 "different task attempt",
22646 "409 Conflict",
22647 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":4,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22648 ),
22649 (
22650 "authentication failure",
22651 "401 Unauthorized",
22652 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22653 ),
22654 (
22655 "authorization failure",
22656 "403 Forbidden",
22657 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22658 ),
22659 (
22660 "protocol failure",
22661 "400 Bad Request",
22662 r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.2","requested_version":"1.3"}"#,
22663 ),
22664 (
22665 "malformed command",
22666 "422 Unprocessable Entity",
22667 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22668 ),
22669 (
22670 "transient server failure",
22671 "503 Service Unavailable",
22672 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22673 ),
22674 ] {
22675 let server = MockWorkerServer::workflow_completion(status, response);
22676 let client = Client::builder(server.base_url())
22677 .timeout(Duration::from_secs(2))
22678 .build()
22679 .expect("client");
22680 let mut worker = Worker::new(client, "rust-workers")
22681 .worker_id("timeout-worker")
22682 .poll_timeout(Duration::from_millis(10));
22683 worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
22684 Ok(json!({"late": "result"}))
22685 });
22686
22687 let error = worker
22688 .run_once()
22689 .await
22690 .expect_err(&format!("{name} must remain an error"));
22691 assert!(
22692 matches!(error, Error::Http { .. } | Error::Protocol(_)),
22693 "{name} returned an unexpected error variant: {error}"
22694 );
22695 }
22696 }
22697
22698 #[tokio::test]
22699 async fn worker_deregistration_uses_worker_plane_method_path_headers_and_result() {
22700 let server = MockWorkerServer::start();
22701 let client = Client::builder(server.base_url())
22702 .worker_token(Some("worker-secret".to_string()))
22703 .namespace("orders")
22704 .timeout(Duration::from_secs(2))
22705 .build()
22706 .expect("client");
22707 let path = "/api/worker/registrations/worker%2F%CE%B1%20space";
22708
22709 let result = client
22710 .deregister_worker_registration("worker/α space")
22711 .await
22712 .expect("deregister worker registration");
22713
22714 assert_eq!(server.method_for(path).as_deref(), Some("DELETE"));
22715 assert_eq!(
22716 server.worker_protocol_for(path).as_deref(),
22717 Some(WORKER_PROTOCOL_VERSION)
22718 );
22719 assert_eq!(server.control_protocol_for(path), None);
22720 assert_eq!(server.namespace_for(path).as_deref(), Some("orders"));
22721 assert_eq!(
22722 server.authorization_for(path).as_deref(),
22723 Some("Bearer worker-secret")
22724 );
22725 assert_eq!(
22726 result,
22727 WorkerDeregistrationEnvelope {
22728 worker_id: "deregistered-worker".to_string(),
22729 outcome: "deregistered".to_string(),
22730 recovered_workflow_task_count: 2,
22731 }
22732 );
22733 }
22734
22735 #[tokio::test]
22736 async fn low_level_registration_rejects_update_validators_before_transport() {
22737 let server = MockWorkerServer::start();
22738 let client = Client::builder(server.base_url())
22739 .timeout(Duration::from_secs(2))
22740 .build()
22741 .expect("client");
22742
22743 for update_validators in [json!(["approve"]), json!("approve")] {
22744 let error = client
22745 .register_worker_with_command_contracts(
22746 "validator-claiming-worker",
22747 "rust-workers",
22748 vec!["orders".to_string()],
22749 vec![],
22750 1,
22751 1,
22752 vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
22753 json!({
22754 "orders": {
22755 "queries": ["current"],
22756 "updates": ["approve"],
22757 "update_validators": update_validators,
22758 },
22759 }),
22760 )
22761 .await
22762 .expect_err("unsupported validator claims must fail before registration");
22763
22764 let Error::UnsupportedUpdateValidators { workflow_type } = error else {
22765 panic!("expected typed unsupported-validator failure");
22766 };
22767 assert_eq!(workflow_type, "orders");
22768 }
22769 assert_eq!(server.request_count("/api/worker/register"), 0);
22770 }
22771
22772 #[tokio::test]
22773 async fn low_level_registration_preserves_query_and_update_contracts() {
22774 let server = MockWorkerServer::start();
22775 let client = Client::builder(server.base_url())
22776 .timeout(Duration::from_secs(2))
22777 .build()
22778 .expect("client");
22779 let contracts = json!({
22780 "orders": {
22781 "queries": ["current"],
22782 "updates": ["approve"],
22783 "update_validators": [],
22784 },
22785 "payments": {
22786 "queries": ["status"],
22787 "updates": ["capture"],
22788 },
22789 });
22790
22791 client
22792 .register_worker_with_command_contracts(
22793 "command-worker",
22794 "rust-workers",
22795 vec!["orders".to_string(), "payments".to_string()],
22796 vec![],
22797 1,
22798 1,
22799 vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
22800 contracts.clone(),
22801 )
22802 .await
22803 .expect("query and update contracts must remain supported");
22804
22805 assert_eq!(
22806 server.request_body("/api/worker/register")["workflow_command_contracts"],
22807 contracts
22808 );
22809 }
22810
22811 #[tokio::test]
22812 async fn role_scoped_tokens_are_never_used_for_the_opposite_plane() {
22813 let server = MockWorkerServer::start();
22814 let control_only = Client::builder(server.base_url())
22815 .control_token(Some("control-secret".to_string()))
22816 .build()
22817 .expect("control client");
22818
22819 let error = control_only
22820 .register_worker("worker", "queue", vec![], vec![], 1, 1)
22821 .await
22822 .expect_err("control token must not authorize a worker request");
22823 assert!(matches!(
22824 error,
22825 Error::MissingRoleCredentials { role: "worker", .. }
22826 ));
22827 assert_eq!(server.request_count("/api/worker/register"), 0);
22828
22829 let worker_only = Client::builder(server.base_url())
22830 .worker_token(Some("worker-secret".to_string()))
22831 .build()
22832 .expect("worker client");
22833 let error = worker_only
22834 .health()
22835 .await
22836 .expect_err("worker token must not authorize a control request");
22837 assert!(matches!(
22838 error,
22839 Error::MissingRoleCredentials {
22840 role: "control",
22841 ..
22842 }
22843 ));
22844 assert_eq!(server.request_count("/api/health"), 0);
22845 }
22846
22847 #[tokio::test]
22848 async fn shared_token_supports_worker_and_control_planes() {
22849 let server = MockWorkerServer::start();
22850 let client = Client::builder(server.base_url())
22851 .token(Some("shared-secret".to_string()))
22852 .build()
22853 .expect("client");
22854
22855 client.health().await.expect("control request");
22856 client
22857 .register_worker("worker", "queue", vec![], vec![], 1, 1)
22858 .await
22859 .expect("worker request");
22860
22861 assert_eq!(
22862 server.authorization_for("/api/health").as_deref(),
22863 Some("Bearer shared-secret")
22864 );
22865 assert_eq!(
22866 server.control_protocol_for("/api/health").as_deref(),
22867 Some(CONTROL_PLANE_VERSION)
22868 );
22869 assert_eq!(
22870 server.authorization_for("/api/worker/register").as_deref(),
22871 Some("Bearer shared-secret")
22872 );
22873 assert_eq!(
22874 server
22875 .worker_protocol_for("/api/worker/register")
22876 .as_deref(),
22877 Some(WORKER_PROTOCOL_VERSION)
22878 );
22879 }
22880
22881 #[tokio::test]
22882 async fn baseline_worker_endpoints_send_the_baseline_protocol() {
22883 let server = MockWorkerServer::start();
22884 let client = Client::builder(server.base_url())
22885 .timeout(Duration::from_secs(2))
22886 .build()
22887 .expect("client");
22888
22889 client
22890 .register_worker("capture-worker", "capture", vec![], vec![], 1, 1)
22891 .await
22892 .expect("register");
22893 client
22894 .heartbeat_worker("capture-worker", 1, 1)
22895 .await
22896 .expect("heartbeat");
22897 client
22898 .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
22899 .await
22900 .expect("workflow poll");
22901 client
22902 .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
22903 .await
22904 .expect("activity poll");
22905
22906 for path in [
22907 "/api/worker/register",
22908 "/api/worker/heartbeat",
22909 "/api/worker/workflow-tasks/poll",
22910 "/api/worker/activity-tasks/poll",
22911 ] {
22912 assert_eq!(
22913 server.worker_protocol_for(path).as_deref(),
22914 Some(WORKER_PROTOCOL_VERSION),
22915 "unexpected protocol for {path}"
22916 );
22917 }
22918
22919 assert_eq!(
22920 server.request_body("/api/worker/workflow-tasks/poll")["timeout_seconds"],
22921 1
22922 );
22923 assert_eq!(
22924 server.request_body("/api/worker/activity-tasks/poll")["timeout_seconds"],
22925 1
22926 );
22927 assert!(
22928 server.request_body("/api/worker/workflow-tasks/poll")["poll_request_id"]
22929 .as_str()
22930 .is_some_and(|id| id.starts_with("rust-workflow-poll-"))
22931 );
22932 assert!(
22933 server.request_body("/api/worker/activity-tasks/poll")["poll_request_id"]
22934 .as_str()
22935 .is_some_and(|id| id.starts_with("rust-activity-poll-"))
22936 );
22937 }
22938
22939 #[tokio::test]
22940 async fn query_task_endpoints_send_the_query_feature_protocol() {
22941 let server = MockWorkerServer::start();
22942 let client = Client::builder(server.base_url())
22943 .timeout(Duration::from_secs(2))
22944 .build()
22945 .expect("client");
22946
22947 client
22948 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
22949 .await
22950 .expect("query poll");
22951 client
22952 .complete_query_task(
22953 "query-capture",
22954 "capture-worker",
22955 1,
22956 json!(8),
22957 DEFAULT_CODEC,
22958 )
22959 .await
22960 .expect("query complete");
22961 client
22962 .fail_query_task(
22963 "query-capture",
22964 "capture-worker",
22965 1,
22966 "failed",
22967 "query_rejected",
22968 "QueryFailed",
22969 )
22970 .await
22971 .expect("query fail");
22972
22973 for path in [
22974 "/api/worker/query-tasks/poll",
22975 "/api/worker/query-tasks/query-capture/complete",
22976 "/api/worker/query-tasks/query-capture/fail",
22977 ] {
22978 assert_eq!(
22979 server.worker_protocol_for(path).as_deref(),
22980 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
22981 "unexpected protocol for {path}"
22982 );
22983 }
22984
22985 assert_eq!(
22986 server.request_body("/api/worker/query-tasks/poll")["timeout_seconds"],
22987 1
22988 );
22989 assert!(
22990 server.request_body("/api/worker/query-tasks/poll")["poll_request_id"]
22991 .as_str()
22992 .is_some_and(|id| id.starts_with("rust-query-poll-"))
22993 );
22994 }
22995
22996 #[tokio::test]
22997 async fn disconnected_client_polls_retry_once_with_the_same_request_id() {
22998 let server = MockWorkerServer::transient_worker_failures();
22999 let client = Client::builder(server.base_url())
23000 .timeout(Duration::from_secs(2))
23001 .build()
23002 .expect("client");
23003
23004 client
23005 .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
23006 .await
23007 .expect("workflow poll retry");
23008 client
23009 .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
23010 .await
23011 .expect("activity poll retry");
23012 client
23013 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
23014 .await
23015 .expect("query poll retry");
23016
23017 for path in [
23018 "/api/worker/workflow-tasks/poll",
23019 "/api/worker/activity-tasks/poll",
23020 "/api/worker/query-tasks/poll",
23021 ] {
23022 let bodies = server.request_bodies(path);
23023 assert_eq!(bodies.len(), 2, "{path} must be retried once");
23024 assert_eq!(
23025 bodies[0]["poll_request_id"], bodies[1]["poll_request_id"],
23026 "{path} must preserve the request binding across retry"
23027 );
23028 }
23029 }
23030
23031 #[tokio::test]
23032 async fn worker_poll_retries_preserve_request_id_across_consecutive_failures() {
23033 let server = MockWorkerServer::consecutive_poll_failures(2);
23034 let client = Client::builder(server.base_url())
23035 .timeout(Duration::from_secs(2))
23036 .build()
23037 .expect("client");
23038 let mut worker = Worker::new(client, "capture")
23039 .worker_id("capture-worker")
23040 .poll_timeout(Duration::from_millis(10))
23041 .retry_policy(WorkerRetryPolicy {
23042 max_retries: 2,
23043 initial_backoff: Duration::from_millis(1),
23044 max_backoff: Duration::from_millis(1),
23045 });
23046 worker.register_workflow(
23047 "capture.workflow",
23048 |_ctx, _input| async move { Ok(Value::Null) },
23049 );
23050 worker.register_activity(
23051 "capture.activity",
23052 |_ctx, _input| async move { Ok(Value::Null) },
23053 );
23054 worker.register_query("capture.workflow", "current", |_ctx, _args| async move {
23055 Ok(Value::Null)
23056 });
23057
23058 assert_eq!(worker.run_once().await.expect("poll retries"), 0);
23059
23060 for path in [
23061 "/api/worker/workflow-tasks/poll",
23062 "/api/worker/activity-tasks/poll",
23063 "/api/worker/query-tasks/poll",
23064 ] {
23065 let bodies = server.request_bodies(path);
23066 assert_eq!(bodies.len(), 3, "{path} must use exactly two retries");
23067 assert!(
23068 bodies
23069 .iter()
23070 .all(|body| body["poll_request_id"] == bodies[0]["poll_request_id"]),
23071 "{path} must preserve one request binding across every retry"
23072 );
23073 }
23074 }
23075
23076 fn storage_refusal(poll_id: Option<&str>, unavailable: bool, mid_poll: bool) -> Value {
23077 let reason = if unavailable {
23078 "storage_admission_unavailable"
23079 } else {
23080 "storage_pressure"
23081 };
23082 let mut body = json!({
23083 "reason": reason,
23084 "storage_state": if unavailable { "fenced" } else { "draining" },
23085 "retryable": true,
23086 "retry_after_seconds": 1,
23087 });
23088 if !mid_poll {
23089 body["request_admitted"] = json!(false);
23090 }
23091 if let Some(id) = poll_id {
23092 body["task"] = Value::Null;
23093 body["poll_status"] = json!(reason);
23094 body["poll_request_id"] = json!(id);
23095 body["retry_same_poll_request_id"] = json!(true);
23096 body["claim_admitted"] = json!(false);
23097 }
23098 body
23099 }
23100
23101 fn storage_worker(server: &MockWorkerServer) -> Worker {
23102 Worker::new(Client::new(server.base_url()).expect("client"), "storage")
23103 .worker_id("storage-worker")
23104 .retry_policy(WorkerRetryPolicy {
23105 max_retries: 1,
23106 initial_backoff: Duration::from_millis(1),
23107 max_backoff: Duration::from_millis(1),
23108 })
23109 }
23110
23111 fn assert_identical_requests(server: &MockWorkerServer, path: &str, count: usize) {
23112 let requests = server.requests.lock().expect("requests");
23113 let bodies: Vec<_> = requests
23114 .iter()
23115 .filter(|request| request.path == path)
23116 .map(|request| &request.body)
23117 .collect();
23118 assert_eq!(bodies.len(), count, "{path}");
23119 assert!(bodies.iter().all(|body| body == &bodies[0]), "{path}");
23120 }
23121
23122 #[test]
23123 fn storage_admission_requires_an_explicit_identity_preserving_contract() {
23124 for unavailable in [false, true] {
23125 for mid_poll in [false, true] {
23126 let body = storage_refusal(Some("same-poll"), unavailable, mid_poll);
23127 let error = Error::Http {
23128 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23129 body: body.to_string(),
23130 };
23131 assert_eq!(
23132 worker_storage_admission_retry_after(&error, Some("same-poll")),
23133 Some(Duration::from_secs(1))
23134 );
23135 assert!(
23136 !worker_operation_is_retryable(&error),
23137 "storage is not a bounded generic retry"
23138 );
23139 for (field, value) in [
23140 ("poll_request_id", json!("wrong-poll")),
23141 ("task", json!({"task_id":"claimed"})),
23142 ("retryable", json!(false)),
23143 ("retry_after_seconds", json!(0)),
23144 ("retry_after_seconds", json!(true)),
23145 ("retry_after_seconds", json!(1.0)),
23146 ("storage_state", json!("normal")),
23147 ("poll_status", json!("empty")),
23148 ("claim_admitted", json!(true)),
23149 ("retry_same_poll_request_id", json!(false)),
23150 ("request_admitted", json!(true)),
23151 ] {
23152 let mut invalid = body.clone();
23153 invalid[field] = value;
23154 let error = Error::Http {
23155 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23156 body: invalid.to_string(),
23157 };
23158 assert!(
23159 worker_storage_admission_retry_after(&error, Some("same-poll")).is_none(),
23160 "{field}"
23161 );
23162 }
23163 }
23164 }
23165 let body = storage_refusal(None, false, false);
23166 let error = Error::Http {
23167 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23168 body: body.to_string(),
23169 };
23170 assert!(worker_storage_admission_retry_after(&error, None).is_some());
23171 assert!(worker_storage_admission_retry_after(&error, Some("")).is_none());
23172 let error = Error::Http {
23173 status: reqwest::StatusCode::FORBIDDEN,
23174 body: body.to_string(),
23175 };
23176 assert!(worker_storage_admission_retry_after(&error, None).is_none());
23177 let body = storage_refusal(None, false, true);
23178 let error = Error::Http {
23179 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23180 body: body.to_string(),
23181 };
23182 assert!(worker_storage_admission_retry_after(&error, None).is_none());
23183 }
23184
23185 #[tokio::test]
23186 async fn storage_poll_recovery_preserves_ambiguous_claim_identity() {
23187 for unavailable in [false, true] {
23188 for mid_poll in [false, true] {
23189 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23190 poll_failures_per_path: 1,
23191 storage_refusals: 7,
23192 storage_path: Some("/poll"),
23193 storage_unavailable: unavailable,
23194 storage_mid_poll: mid_poll,
23195 ..MockWorkerBehavior::default()
23196 });
23197 let mut worker = storage_worker(&server);
23198 worker.register_query("unused", "state", |_, _| async { Ok(Value::Null) });
23199 assert_eq!(worker.run_once().await.expect("storage recovery"), 0);
23200 for path in [
23201 "/api/worker/workflow-tasks/poll",
23202 "/api/worker/activity-tasks/poll",
23203 "/api/worker/query-tasks/poll",
23204 ] {
23205 assert_identical_requests(&server, path, 9);
23206 }
23207 }
23208 }
23209 }
23210
23211 #[tokio::test]
23212 async fn storage_refused_mutations_do_not_reserialize_or_change_client_scope() {
23213 struct CountedBody(Arc<AtomicUsize>);
23214 impl Serialize for CountedBody {
23215 fn serialize<S: Serializer>(
23216 &self,
23217 serializer: S,
23218 ) -> std::result::Result<S::Ok, S::Error> {
23219 let count = self.0.fetch_add(1, Ordering::SeqCst);
23220 json!({"serialization":count,"lease_owner":"worker","attempt":7})
23221 .serialize(serializer)
23222 }
23223 }
23224 for path in [
23225 "/api/worker/register",
23226 "/api/worker/heartbeat",
23227 "/api/worker/workflow-tasks/storage-task/complete",
23228 "/api/worker/workflow-tasks/storage-task/fail",
23229 "/api/worker/activity-tasks/storage-task/complete",
23230 "/api/worker/activity-tasks/storage-task/fail",
23231 "/api/worker/activity-tasks/storage-task/heartbeat",
23232 "/api/worker/query-tasks/storage-task/complete",
23233 "/api/worker/query-tasks/storage-task/fail",
23234 ] {
23235 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23236 storage_refusals: 7,
23237 storage_path: Some(path),
23238 ..MockWorkerBehavior::default()
23239 });
23240 let worker =
23241 storage_worker(&server).with_storage_admission(Arc::new(AtomicBool::new(false)));
23242 let calls = Arc::new(AtomicUsize::new(0));
23243 let _: Value = worker
23244 .client
23245 .request_json(
23246 reqwest::Method::POST,
23247 &path[4..],
23248 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
23249 Some(&CountedBody(Arc::clone(&calls))),
23250 )
23251 .await
23252 .expect("prepared request recovery");
23253 assert_eq!(calls.load(Ordering::SeqCst), 1);
23254 assert_identical_requests(&server, path, 8);
23255 }
23256 for worker_scope in [false, true] {
23257 let path = if worker_scope {
23258 "/api/health"
23259 } else {
23260 "/api/worker/register"
23261 };
23262 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23263 storage_refusals: usize::MAX,
23264 storage_path: Some(path),
23265 ..MockWorkerBehavior::default()
23266 });
23267 let worker = storage_worker(&server);
23268 let client = worker.client.clone();
23269 let worker = worker.with_storage_admission(Arc::new(AtomicBool::new(false)));
23270 let error = if worker_scope {
23271 worker
23272 .client
23273 .health()
23274 .await
23275 .expect_err("control plane is not retried")
23276 } else {
23277 client
23278 .request_json::<Value, Value>(
23279 reqwest::Method::POST,
23280 "/worker/register",
23281 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
23282 Some(&json!({})),
23283 )
23284 .await
23285 .expect_err("direct client is not retried")
23286 };
23287 assert!(worker_storage_admission_body(&error).is_some());
23288 assert_eq!(server.request_count(path), 1);
23289 }
23290 }
23291
23292 #[tokio::test]
23293 async fn storage_activity_outcome_is_retained_without_reexecuting_handler() {
23294 for fail in [false, true] {
23295 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23296 storage_activity: true,
23297 storage_refusals: 7,
23298 storage_path: Some("/storage-activity/"),
23299 ..MockWorkerBehavior::default()
23300 });
23301 let mut worker = storage_worker(&server);
23302 let calls = Arc::new(AtomicUsize::new(0));
23303 let observed = Arc::clone(&calls);
23304 worker.register_activity("storage.activity", move |ctx, _| {
23305 let calls = Arc::clone(&observed);
23306 async move {
23307 calls.fetch_add(1, Ordering::SeqCst);
23308 ctx.heartbeat(json!({"step":2})).await?;
23309 if fail {
23310 Err(Error::WorkerLoop("intentional handler failure".to_string()))
23311 } else {
23312 Ok(json!({"receipt":true}))
23313 }
23314 }
23315 });
23316 assert_eq!(worker.run_once().await.expect("activity settled"), 1);
23317 assert_eq!(calls.load(Ordering::SeqCst), 1);
23318 assert_identical_requests(
23319 &server,
23320 "/api/worker/activity-tasks/storage-activity/heartbeat",
23321 8,
23322 );
23323 let suffix = if fail { "fail" } else { "complete" };
23324 assert_identical_requests(
23325 &server,
23326 &format!("/api/worker/activity-tasks/storage-activity/{suffix}"),
23327 8,
23328 );
23329 let other = if fail { "complete" } else { "fail" };
23330 assert_eq!(
23331 server.request_count(&format!(
23332 "/api/worker/activity-tasks/storage-activity/{other}"
23333 )),
23334 0
23335 );
23336 }
23337 }
23338
23339 #[tokio::test]
23340 async fn storage_waits_are_interruptible_without_false_activity_failure() {
23341 for path in [
23342 "/api/worker/register",
23343 "/api/worker/heartbeat",
23344 "/api/worker/activity-tasks/poll",
23345 "/api/worker/activity-tasks/storage-activity/heartbeat",
23346 "/api/worker/activity-tasks/storage-activity/complete",
23347 ] {
23348 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23349 storage_activity: true,
23350 storage_refusals: usize::MAX,
23351 storage_path: Some(path),
23352 ..MockWorkerBehavior::default()
23353 });
23354 let mut worker = storage_worker(&server).retry_policy(WorkerRetryPolicy::default());
23355 worker.register_activity("storage.activity", |ctx, _| async move {
23356 ctx.heartbeat(json!({"step":2})).await?;
23357 Ok(json!({"receipt":true}))
23358 });
23359 let shutdown = async {
23360 while server.request_count(path) == 0 {
23361 tokio::time::sleep(Duration::from_millis(1)).await;
23362 }
23363 };
23364 let result = tokio::time::timeout(Duration::from_secs(2), worker.run_until(shutdown))
23365 .await
23366 .expect("shutdown interrupts admission");
23367 assert!(
23368 result.is_err(),
23369 "a refused operation must not appear acknowledged: {path}, {result:?}"
23370 );
23371 assert_eq!(server.request_count(path), 1);
23372 assert_eq!(
23373 server.request_count("/api/worker/activity-tasks/storage-activity/fail"),
23374 0
23375 );
23376 assert_eq!(
23377 server.request_count("/api/worker/registrations/mock-worker"),
23378 usize::from(!path.ends_with("/register"))
23379 );
23380 }
23381 }
23382
23383 #[tokio::test]
23384 async fn storage_query_outcome_is_retained_without_reexecuting_handler() {
23385 for fail in [false, true] {
23386 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23387 storage_query: true,
23388 storage_refusals: 7,
23389 storage_path: Some("/storage-query/"),
23390 ..MockWorkerBehavior::default()
23391 });
23392 let mut worker = storage_worker(&server);
23393 worker.register_workflow("storage.workflow", |_, _| async { Ok(Value::Null) });
23394 let calls = Arc::new(AtomicUsize::new(0));
23395 let observed = Arc::clone(&calls);
23396 worker.register_query("storage.workflow", "state", move |_, _| {
23397 let calls = Arc::clone(&observed);
23398 async move {
23399 calls.fetch_add(1, Ordering::SeqCst);
23400 if fail {
23401 Err(Error::WorkerLoop("intentional query failure".to_string()))
23402 } else {
23403 Ok(json!({"state":"waiting"}))
23404 }
23405 }
23406 });
23407 assert_eq!(worker.run_once().await.expect("query settled"), 1);
23408 assert_eq!(calls.load(Ordering::SeqCst), 1);
23409 let suffix = if fail { "fail" } else { "complete" };
23410 assert_identical_requests(
23411 &server,
23412 &format!("/api/worker/query-tasks/storage-query/{suffix}"),
23413 8,
23414 );
23415 let other = if fail { "complete" } else { "fail" };
23416 assert_eq!(
23417 server.request_count(&format!("/api/worker/query-tasks/storage-query/{other}")),
23418 0
23419 );
23420 }
23421 }
23422
23423 #[tokio::test]
23424 async fn storage_recovery_does_not_override_auth_lease_or_invalid_contract() {
23425 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23426 storage_refusals: 7,
23427 storage_path: Some("/poll"),
23428 unauthorized_polls: true,
23429 ..MockWorkerBehavior::default()
23430 });
23431 let error = storage_worker(&server)
23432 .run_once()
23433 .await
23434 .expect_err("auth remains terminal");
23435 assert!(matches!(
23436 error,
23437 Error::Http {
23438 status: reqwest::StatusCode::UNAUTHORIZED,
23439 ..
23440 }
23441 ));
23442 assert_identical_requests(&server, "/api/worker/workflow-tasks/poll", 8);
23443
23444 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23445 storage_refusals: 7,
23446 storage_path: Some("/activity-cancel/complete"),
23447 ..MockWorkerBehavior::default()
23448 });
23449 let worker =
23450 storage_worker(&server).with_storage_admission(Arc::new(AtomicBool::new(false)));
23451 let error = worker
23452 .client
23453 .complete_activity_task(
23454 "activity-cancel",
23455 "attempt-cancel",
23456 "worker",
23457 json!({}),
23458 DEFAULT_CODEC,
23459 )
23460 .await
23461 .expect_err("cancellation remains terminal");
23462 assert!(activity_task_rejection_is_final(&error));
23463 assert_identical_requests(
23464 &server,
23465 "/api/worker/activity-tasks/activity-cancel/complete",
23466 8,
23467 );
23468
23469 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23470 storage_refusals: usize::MAX,
23471 storage_path: Some("/poll"),
23472 storage_wrong_poll_id: true,
23473 ..MockWorkerBehavior::default()
23474 });
23475 assert!(storage_worker(&server).run_once().await.is_err());
23476 assert_eq!(server.request_count("/api/worker/workflow-tasks/poll"), 1);
23477 }
23478
23479 #[tokio::test]
23480 async fn storage_pollers_stop_when_the_run_future_is_aborted() {
23481 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23482 storage_refusals: usize::MAX,
23483 storage_path: Some("/poll"),
23484 ..MockWorkerBehavior::default()
23485 });
23486 let mut worker = storage_worker(&server).retry_policy(WorkerRetryPolicy::default());
23487 worker.register_activity("unused", |_, _| async { Ok(Value::Null) });
23488 let run = tokio::spawn(async move { worker.run().await });
23489 tokio::time::timeout(Duration::from_secs(2), async {
23490 while server.request_count("/api/worker/activity-tasks/poll") == 0 {
23491 tokio::time::sleep(Duration::from_millis(1)).await;
23492 }
23493 })
23494 .await
23495 .expect("poll started");
23496 run.abort();
23497 assert!(run.await.expect_err("cancelled run").is_cancelled());
23498 tokio::time::sleep(Duration::from_millis(250)).await;
23499 assert_eq!(server.request_count("/api/worker/activity-tasks/poll"), 1);
23500 }
23501
23502 #[tokio::test]
23503 async fn query_protocol_rejection_from_older_server_is_typed() {
23504 let server = MockWorkerServer::reject_query_protocol();
23505 let client = Client::builder(server.base_url())
23506 .timeout(Duration::from_secs(2))
23507 .build()
23508 .expect("client");
23509
23510 let error = client
23511 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
23512 .await
23513 .expect_err("server below query protocol floor must reject");
23514 let Error::Protocol(failure) = error else {
23515 panic!("expected typed protocol failure");
23516 };
23517
23518 assert_eq!(failure.status, 400);
23519 assert_eq!(failure.reason, "unsupported_protocol_version");
23520 assert_eq!(failure.supported_version.as_deref(), Some("1.7"));
23521 assert_eq!(
23522 failure.requested_version.as_deref(),
23523 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
23524 );
23525 assert_eq!(
23526 server
23527 .worker_protocol_for("/api/worker/query-tasks/poll")
23528 .as_deref(),
23529 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
23530 );
23531 }
23532
23533 #[tokio::test]
23534 async fn run_once_without_query_handlers_keeps_pre_query_server_compatibility() {
23535 let server = MockWorkerServer::reject_query_protocol();
23536 let client = Client::builder(server.base_url())
23537 .timeout(Duration::from_secs(2))
23538 .build()
23539 .expect("client");
23540 let mut worker = Worker::new(client, "rust-workers")
23541 .worker_id("baseline-worker")
23542 .poll_timeout(Duration::from_millis(10));
23543
23544 worker.register_workflow("baseline.workflow", |_ctx, _input| async move {
23545 Ok(Value::Null)
23546 });
23547
23548 assert_eq!(worker.run_once().await.expect("baseline run once"), 0);
23549 assert_eq!(
23550 server
23551 .worker_protocol_for("/api/worker/workflow-tasks/poll")
23552 .as_deref(),
23553 Some(WORKER_PROTOCOL_VERSION)
23554 );
23555 assert_eq!(
23556 server.worker_protocol_for("/api/worker/query-tasks/poll"),
23557 None,
23558 "a worker without query handlers must not use the query-task endpoint"
23559 );
23560 }
23561
23562 #[tokio::test]
23563 async fn completion_time_query_rejection_is_typed_without_stopping_worker() {
23564 let server = MockWorkerServer::reject_query_completion();
23565 let client = Client::builder(server.base_url())
23566 .timeout(Duration::from_secs(2))
23567 .build()
23568 .expect("client");
23569
23570 let error = client
23571 .complete_query_task("query-late", "late-worker", 1, json!(8), DEFAULT_CODEC)
23572 .await
23573 .expect_err("expired completion must be rejected");
23574 let Error::QueryFailed(failure) = error else {
23575 panic!("expected typed query failure");
23576 };
23577 assert_eq!(failure.status, 409);
23578 assert_eq!(failure.reason, "query_task_timed_out");
23579
23580 let mut worker = Worker::new(client, "rust-workers")
23581 .worker_id("late-worker")
23582 .poll_timeout(Duration::from_millis(10));
23583 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
23584 worker.register_query(
23585 "counter",
23586 "current",
23587 |_ctx, _args| async move { Ok(json!(8)) },
23588 );
23589
23590 assert_eq!(worker.run_once().await.expect("late task is handled"), 1);
23591 assert_eq!(
23592 worker
23593 .run_once()
23594 .await
23595 .expect("worker continues after late completion"),
23596 0
23597 );
23598 assert_eq!(
23599 server.request_count("/api/worker/query-tasks/query-late/complete"),
23600 2
23601 );
23602 assert_eq!(
23603 server.request_count("/api/worker/query-tasks/query-late/fail"),
23604 0,
23605 "a server completion rejection must not be reported as an encoding failure"
23606 );
23607 }
23608
23609 #[tokio::test]
23610 async fn normal_shutdown_joins_pollers_and_deregisters_once() {
23611 let server = MockWorkerServer::start();
23612 let client = Client::builder(server.base_url())
23613 .timeout(Duration::from_secs(2))
23614 .build()
23615 .expect("client");
23616 let mut worker = Worker::new(client, "rust-workers")
23617 .worker_id("joined-worker")
23618 .poll_timeout(Duration::from_millis(10));
23619 worker.register_workflow(
23620 "joined.workflow",
23621 |_ctx, _input| async move { Ok(Value::Null) },
23622 );
23623 worker.register_activity(
23624 "joined.activity",
23625 |_ctx, _input| async move { Ok(Value::Null) },
23626 );
23627 worker.register_query("joined.workflow", "state", |_ctx, _input| async move {
23628 Ok(Value::Null)
23629 });
23630
23631 worker
23632 .run_until(tokio::time::sleep(Duration::from_millis(20)))
23633 .await
23634 .expect("normal shutdown");
23635
23636 let deregistration_path = "/api/worker/registrations/mock-worker";
23637 assert_eq!(server.request_count(deregistration_path), 1);
23638 for poll_path in [
23639 "/api/worker/workflow-tasks/poll",
23640 "/api/worker/activity-tasks/poll",
23641 "/api/worker/query-tasks/poll",
23642 ] {
23643 assert!(server.request_count(poll_path) > 0, "missing {poll_path}");
23644 }
23645 assert_eq!(
23646 server.captured_paths().last().map(String::as_str),
23647 Some(deregistration_path),
23648 "deregistration must start only after every poller has joined"
23649 );
23650 }
23651
23652 #[tokio::test]
23653 async fn registration_failure_does_not_deregister() {
23654 let server = MockWorkerServer::rejected_registration();
23655 let client = Client::builder(server.base_url())
23656 .timeout(Duration::from_secs(2))
23657 .build()
23658 .expect("client");
23659 let worker = Worker::new(client, "rust-workers").worker_id("never-registered");
23660
23661 let error = worker
23662 .run_until(async {})
23663 .await
23664 .expect_err("registration must fail");
23665 assert!(matches!(
23666 error,
23667 Error::Http {
23668 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23669 ..
23670 }
23671 ));
23672 assert!(server
23673 .captured_paths()
23674 .iter()
23675 .all(|path| !path.starts_with("/api/worker/registrations/")));
23676 }
23677
23678 #[tokio::test]
23679 async fn protocol_116_server_rejects_occurrence_identity_worker_registration() {
23680 let server = MockWorkerServer::rejected_registration_protocol();
23681 let client = Client::builder(server.base_url())
23682 .timeout(Duration::from_secs(2))
23683 .build()
23684 .expect("client");
23685 let worker = Worker::new(client, "rust-workers").worker_id("protocol-117-worker");
23686
23687 let error = worker
23688 .run_until(async {})
23689 .await
23690 .expect_err("a protocol 1.16 server must reject this worker");
23691 let Error::Protocol(failure) = error else {
23692 panic!("expected typed protocol rejection");
23693 };
23694 assert_eq!(failure.reason, "unsupported_protocol_version");
23695 assert_eq!(failure.supported_version.as_deref(), Some("1.16"));
23696 assert_eq!(failure.requested_version.as_deref(), Some("1.17"));
23697 assert_eq!(
23698 server
23699 .worker_protocol_for("/api/worker/register")
23700 .as_deref(),
23701 Some(WORKER_PROTOCOL_VERSION)
23702 );
23703 }
23704
23705 #[tokio::test]
23706 async fn declined_registration_does_not_deregister() {
23707 let server = MockWorkerServer::declined_registration();
23708 let client = Client::builder(server.base_url())
23709 .timeout(Duration::from_secs(2))
23710 .build()
23711 .expect("client");
23712 let worker = Worker::new(client, "rust-workers").worker_id("declined-worker");
23713
23714 let error = worker
23715 .run_until(async {})
23716 .await
23717 .expect_err("declined registration must fail");
23718 assert!(matches!(error, Error::WorkerLoop(_)));
23719 assert!(error.to_string().contains("was not accepted"));
23720 assert!(server
23721 .captured_paths()
23722 .iter()
23723 .all(|path| !path.starts_with("/api/worker/registrations/")));
23724 }
23725
23726 #[tokio::test]
23727 async fn deregistration_http_failure_is_returned_after_normal_shutdown() {
23728 let server = MockWorkerServer::rejected_deregistration();
23729 let client = Client::builder(server.base_url())
23730 .timeout(Duration::from_secs(2))
23731 .build()
23732 .expect("client");
23733 let worker = Worker::new(client, "rust-workers").worker_id("forbidden-cleanup");
23734
23735 let error = worker
23736 .run_until(async {})
23737 .await
23738 .expect_err("deregistration must fail");
23739 assert!(matches!(
23740 error,
23741 Error::Http {
23742 status: reqwest::StatusCode::FORBIDDEN,
23743 ..
23744 }
23745 ));
23746 assert_eq!(
23747 server.request_count("/api/worker/registrations/mock-worker"),
23748 1
23749 );
23750 }
23751
23752 #[tokio::test]
23753 async fn deregistration_protocol_failure_is_returned_after_normal_shutdown() {
23754 let server = MockWorkerServer::rejected_deregistration_protocol();
23755 let client = Client::builder(server.base_url())
23756 .timeout(Duration::from_secs(2))
23757 .build()
23758 .expect("client");
23759 let worker = Worker::new(client, "rust-workers").worker_id("protocol-cleanup");
23760
23761 let error = worker
23762 .run_until(async {})
23763 .await
23764 .expect_err("protocol rejection must fail shutdown");
23765 let Error::Protocol(failure) = error else {
23766 panic!("expected typed protocol failure");
23767 };
23768 assert_eq!(failure.reason, "unsupported_protocol_version");
23769 assert_eq!(
23770 failure.requested_version.as_deref(),
23771 Some(WORKER_PROTOCOL_VERSION)
23772 );
23773 assert_eq!(
23774 server.request_count("/api/worker/registrations/mock-worker"),
23775 1
23776 );
23777 }
23778
23779 #[tokio::test]
23780 async fn primary_poller_error_retains_deregistration_failure_context() {
23781 let server = MockWorkerServer::unauthorized_polls_and_rejected_deregistration();
23782 let client = Client::builder(server.base_url())
23783 .timeout(Duration::from_secs(2))
23784 .build()
23785 .expect("client");
23786 let mut worker = Worker::new(client, "rust-workers")
23787 .worker_id("combined-failure")
23788 .poll_timeout(Duration::from_millis(10));
23789 worker.register_workflow("combined.workflow", |_ctx, _input| async move {
23790 Ok(Value::Null)
23791 });
23792
23793 let error = worker
23794 .run()
23795 .await
23796 .expect_err("worker and cleanup must fail");
23797 let summary = error.to_string();
23798 assert!(summary.contains("authentication_failed"));
23799 assert!(summary.contains("worker cannot deregister"));
23800 let Error::WorkerShutdown {
23801 primary,
23802 deregistration,
23803 } = error
23804 else {
23805 panic!("expected combined worker shutdown error");
23806 };
23807 assert!(matches!(
23808 *primary,
23809 Error::Http {
23810 status: reqwest::StatusCode::UNAUTHORIZED,
23811 ..
23812 }
23813 ));
23814 assert!(matches!(
23815 *deregistration,
23816 Error::Http {
23817 status: reqwest::StatusCode::FORBIDDEN,
23818 ..
23819 }
23820 ));
23821 assert_eq!(
23822 server.request_count("/api/worker/registrations/mock-worker"),
23823 1
23824 );
23825 }
23826
23827 #[tokio::test]
23828 async fn activity_only_worker_can_shutdown_without_workflow_poller() {
23829 let server = MockWorkerServer::start();
23830 let client = Client::builder(server.base_url())
23831 .timeout(Duration::from_secs(2))
23832 .build()
23833 .expect("client");
23834 let mut worker = Worker::new(client, "rust-workers")
23835 .worker_id("activity-only-worker")
23836 .poll_timeout(Duration::from_millis(10));
23837
23838 worker.register_activity(
23839 "activity.only",
23840 |_ctx, _args| async move { Ok(Value::Null) },
23841 );
23842
23843 worker.run_until(async {}).await.expect("run worker");
23844 }
23845
23846 #[tokio::test]
23847 async fn workflow_only_worker_can_shutdown_without_activity_poller() {
23848 let server = MockWorkerServer::start();
23849 let client = Client::builder(server.base_url())
23850 .timeout(Duration::from_secs(2))
23851 .build()
23852 .expect("client");
23853 let mut worker = Worker::new(client, "rust-workers")
23854 .worker_id("workflow-only-worker")
23855 .poll_timeout(Duration::from_millis(10));
23856
23857 worker.register_workflow(
23858 "workflow.only",
23859 |_ctx, _input| async move { Ok(Value::Null) },
23860 );
23861
23862 worker.run_until(async {}).await.expect("run worker");
23863 }
23864
23865 #[tokio::test]
23866 async fn worker_heartbeat_observer_receives_server_acknowledgements() {
23867 let server = MockWorkerServer::start();
23868 let client = Client::builder(server.base_url())
23869 .timeout(Duration::from_secs(2))
23870 .build()
23871 .expect("client");
23872 let observations = Arc::new(Mutex::new(Vec::new()));
23873 let observed = Arc::clone(&observations);
23874 let mut worker = Worker::new(client, "rust-workers")
23875 .worker_id("observed-heartbeat-worker")
23876 .poll_timeout(Duration::from_millis(10))
23877 .on_worker_heartbeat(move |observation| {
23878 observed
23879 .lock()
23880 .expect("heartbeat observations")
23881 .push(observation.clone());
23882 });
23883
23884 worker.register_workflow("workflow.observed", |_ctx, _input| async move {
23885 Ok(Value::Null)
23886 });
23887 let acknowledged = Arc::clone(&observations);
23888 worker
23889 .run_until(async move {
23890 tokio::time::timeout(Duration::from_secs(2), async move {
23891 loop {
23892 if !acknowledged
23893 .lock()
23894 .expect("heartbeat observations")
23895 .is_empty()
23896 {
23897 break;
23898 }
23899 tokio::time::sleep(Duration::from_millis(1)).await;
23900 }
23901 })
23902 .await
23903 .expect("heartbeat acknowledgement within timeout");
23904 })
23905 .await
23906 .expect("run worker");
23907
23908 let observations = observations.lock().expect("heartbeat observations");
23909 let first = observations.first().expect("heartbeat acknowledgement");
23910 assert_eq!(first.worker_id, "observed-heartbeat-worker");
23911 assert_eq!(first.task_queue, "rust-workers");
23912 assert!(first.acknowledged_at_unix_millis > 0);
23913 assert_eq!(first.acknowledgement, json!({}));
23914 }
23915
23916 #[tokio::test]
23917 async fn delayed_worker_heartbeat_keeps_cadence_and_pollers_live() {
23918 let server = MockWorkerServer::delayed_heartbeat_worker();
23919 let client = Client::builder(server.base_url())
23920 .timeout(Duration::from_secs(3))
23921 .build()
23922 .expect("client");
23923 let observations = Arc::new(Mutex::new(Vec::new()));
23924 let observed = Arc::clone(&observations);
23925 let mut worker = Worker::new(client, "rust-snapshot-workers")
23926 .worker_id("rust-snapshot-worker")
23927 .poll_timeout(Duration::from_millis(10))
23928 .on_worker_heartbeat(move |observation| {
23929 observed
23930 .lock()
23931 .expect("heartbeat observations")
23932 .push(observation.clone());
23933 });
23934
23935 worker.register_workflow("snapshot", |ctx, _input| async move {
23936 ctx.wait_signal("finish").await?;
23937 Ok(json!({"status": "finished"}))
23938 });
23939 worker.register_query("snapshot", "current", |ctx, _args| async move {
23940 Ok(json!(ctx
23941 .signals("increment")
23942 .iter()
23943 .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
23944 .sum::<i64>()))
23945 });
23946 worker.register_activity("cancel-aware", |_ctx, _args| async move {
23947 Ok(json!({"late": "completion"}))
23948 });
23949
23950 worker
23951 .run_until(tokio::time::sleep(Duration::from_millis(3_800)))
23952 .await
23953 .expect("delayed heartbeat must allow a clean worker shutdown");
23954
23955 let observations = observations.lock().expect("heartbeat observations");
23956 assert!(
23957 observations.len() >= 3,
23958 "the immediate heartbeat, delayed acknowledgement, and next cadence heartbeat must complete"
23959 );
23960 assert!(
23961 observations.windows(2).all(|pair| {
23962 pair[1].acknowledged_at_unix_millis
23963 .saturating_sub(pair[0].acknowledged_at_unix_millis)
23964 >= 850
23965 }),
23966 "successful acknowledgements must not catch up faster than the advertised one-second cadence: {observations:?}"
23967 );
23968 drop(observations);
23969
23970 let heartbeat_times = server.request_times("/api/worker/heartbeat");
23971 let delayed_request_at = *heartbeat_times
23972 .get(1)
23973 .expect("intentionally delayed heartbeat request");
23974 let delay_window_start = delayed_request_at + Duration::from_millis(100);
23975 let delay_window_end = delayed_request_at + Duration::from_millis(1_400);
23976 for path in [
23977 "/api/worker/workflow-tasks/poll",
23978 "/api/worker/activity-tasks/poll",
23979 "/api/worker/query-tasks/poll",
23980 ] {
23981 assert!(
23982 server
23983 .request_times(path)
23984 .iter()
23985 .any(|received_at| *received_at >= delay_window_start
23986 && *received_at <= delay_window_end),
23987 "{path} must keep polling while a heartbeat acknowledgement is delayed"
23988 );
23989 }
23990 assert!(
23991 server.request_count("/api/worker/workflow-tasks/snapshot-wait-3/fail") >= 1,
23992 "workflow work must be settled"
23993 );
23994 assert!(
23995 server.request_count("/api/worker/activity-tasks/activity-cancel/complete") >= 1,
23996 "activity work must be settled"
23997 );
23998 assert!(
23999 server.request_count("/api/worker/query-tasks/snapshot-current/complete") >= 1,
24000 "query work must be settled"
24001 );
24002 }
24003
24004 #[tokio::test]
24005 async fn retried_worker_heartbeat_restarts_the_advertised_cadence() {
24006 let server = MockWorkerServer::heartbeat_retry_worker();
24007 let client = Client::builder(server.base_url())
24008 .timeout(Duration::from_secs(2))
24009 .build()
24010 .expect("client");
24011 let observations = Arc::new(Mutex::new(Vec::new()));
24012 let observed = Arc::clone(&observations);
24013 let worker = Worker::new(client, "rust-workers")
24014 .worker_id("heartbeat-retry-worker")
24015 .retry_policy(WorkerRetryPolicy {
24016 max_retries: 1,
24017 initial_backoff: Duration::from_millis(300),
24018 max_backoff: Duration::from_millis(300),
24019 })
24020 .on_worker_heartbeat(move |observation| {
24021 observed
24022 .lock()
24023 .expect("heartbeat observations")
24024 .push(observation.clone());
24025 });
24026
24027 worker
24028 .run_until(tokio::time::sleep(Duration::from_millis(2_700)))
24029 .await
24030 .expect("retryable heartbeat failure must remain bounded and recover");
24031
24032 let observations = observations.lock().expect("heartbeat observations");
24033 assert!(observations.len() >= 3, "heartbeat retry must recover");
24034 assert!(
24035 observations.windows(2).all(|pair| {
24036 pair[1]
24037 .acknowledged_at_unix_millis
24038 .saturating_sub(pair[0].acknowledged_at_unix_millis)
24039 >= 850
24040 }),
24041 "a successful retry must start a fresh advertised cadence: {observations:?}"
24042 );
24043 assert_eq!(
24044 server.request_count("/api/worker/heartbeat"),
24045 observations.len() + 1,
24046 "one retryable failure must add exactly one bounded request"
24047 );
24048 }
24049
24050 #[tokio::test]
24051 async fn query_enabled_worker_ignores_unmatched_signals_then_completes_once() {
24052 let server = MockWorkerServer::waiting_query_worker();
24053 let client = Client::builder(server.base_url())
24054 .timeout(Duration::from_secs(2))
24055 .build()
24056 .expect("client");
24057 let observations = Arc::new(Mutex::new(Vec::new()));
24058 let observed = Arc::clone(&observations);
24059 let mut worker = Worker::new(client, "rust-snapshot-workers")
24060 .worker_id("rust-snapshot-worker")
24061 .poll_timeout(Duration::from_millis(10))
24062 .on_worker_heartbeat(move |observation| {
24063 observed
24064 .lock()
24065 .expect("heartbeat observations")
24066 .push(observation.clone());
24067 });
24068
24069 worker.register_workflow("snapshot", |ctx, _input| async move {
24070 ctx.wait_signal("finish").await?;
24071 Ok(json!({"status": "finished"}))
24072 });
24073 worker.register_query("snapshot", "current", |ctx, _args| async move {
24074 let current = ctx
24075 .signals("increment")
24076 .iter()
24077 .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
24078 .sum::<i64>();
24079 Ok(json!(current))
24080 });
24081 worker.register_update("snapshot", "replace", |_ctx, args| async move { Ok(args) });
24082
24083 worker
24084 .run_until(tokio::time::sleep(Duration::from_millis(3_200)))
24085 .await
24086 .expect("pending workflow and query poller must remain live until shutdown");
24087
24088 assert!(
24089 observations.lock().expect("heartbeat observations").len() >= 4,
24090 "the immediate heartbeat and at least three advertised one-second intervals must be acknowledged"
24091 );
24092 assert!(
24093 server.request_count("/api/worker/workflow-tasks/poll") >= 3,
24094 "workflow polling must continue after empty replay acknowledgements"
24095 );
24096 assert!(
24097 server.request_count("/api/worker/query-tasks/poll") >= 2,
24098 "query polling must continue after serving the current query"
24099 );
24100 assert_eq!(
24101 server.request_body("/api/worker/register")["capabilities"],
24102 json!([
24103 CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY,
24104 DURABLE_SELECTION_CAPABILITY,
24105 MEMO_UPSERTS_CAPABILITY,
24106 TYPED_SEARCH_ATTRIBUTES_CAPABILITY,
24107 QUERY_TASKS_CAPABILITY,
24108 WORKFLOW_UPDATES_CAPABILITY,
24109 MESSAGE_STREAMS_CAPABILITY
24110 ])
24111 );
24112 assert_eq!(
24113 server.request_body("/api/worker/register")["workflow_command_contracts"]["snapshot"],
24114 json!({
24115 "queries": ["current"],
24116 "query_contracts": [],
24117 "signals": [],
24118 "signal_contracts": [],
24119 "updates": ["replace"],
24120 "update_contracts": [],
24121 "update_validators": [],
24122 })
24123 );
24124
24125 let opened = server.request_body("/api/worker/workflow-tasks/snapshot-open/complete");
24126 assert_eq!(
24127 opened["commands"],
24128 json!([{
24129 "type": "open_signal_wait",
24130 "signal_name": "finish",
24131 }])
24132 );
24133
24134 for task_id in ["snapshot-wait-3", "snapshot-wait-5"] {
24135 let fail_path = format!("/api/worker/workflow-tasks/{task_id}/fail");
24136 let completion_path = format!("/api/worker/workflow-tasks/{task_id}/complete");
24137 let failure = server.request_body(&fail_path);
24138 assert_eq!(
24139 failure["failure"]["type"],
24140 WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE
24141 );
24142 assert_eq!(server.request_count(&completion_path), 0);
24143 }
24144
24145 let query_completion =
24146 server.request_body("/api/worker/query-tasks/snapshot-current/complete");
24147 assert_eq!(query_completion["result"], json!(8));
24148
24149 let terminal_path = "/api/worker/workflow-tasks/snapshot-finish/complete";
24150 assert_eq!(
24151 server.request_count(terminal_path),
24152 1,
24153 "the matching signal must settle the workflow exactly once"
24154 );
24155 let terminal = server.request_body(terminal_path);
24156 assert_eq!(terminal["commands"].as_array().map(Vec::len), Some(1));
24157 assert_eq!(terminal["commands"][0]["type"], "complete_workflow");
24158 assert_eq!(
24159 decode_wire_value(&terminal["commands"][0]["result"], DEFAULT_CODEC)
24160 .expect("terminal workflow result"),
24161 json!({"status": "finished"})
24162 );
24163 }
24164
24165 #[tokio::test]
24166 async fn worker_retries_poll_and_heartbeat_transport_failures_independently() {
24167 let server = MockWorkerServer::transient_worker_failures();
24168 let client = Client::builder(server.base_url())
24169 .timeout(Duration::from_secs(2))
24170 .build()
24171 .expect("client");
24172 let mut worker = Worker::new(client, "rust-workers")
24173 .worker_id("retry-worker")
24174 .poll_timeout(Duration::from_millis(10))
24175 .retry_policy(WorkerRetryPolicy {
24176 max_retries: 2,
24177 initial_backoff: Duration::from_millis(1),
24178 max_backoff: Duration::from_millis(1),
24179 });
24180 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
24181 worker.register_activity(
24182 "counter.activity",
24183 |_ctx, _input| async move { Ok(Value::Null) },
24184 );
24185 worker.register_query(
24186 "counter",
24187 "current",
24188 |_ctx, _args| async move { Ok(json!(8)) },
24189 );
24190
24191 worker
24192 .run_until(tokio::time::sleep(Duration::from_millis(75)))
24193 .await
24194 .expect("transient failures must not stop the worker");
24195
24196 for path in [
24197 "/api/worker/heartbeat",
24198 "/api/worker/workflow-tasks/poll",
24199 "/api/worker/activity-tasks/poll",
24200 "/api/worker/query-tasks/poll",
24201 ] {
24202 assert!(
24203 server.request_count(path) >= 2,
24204 "{path} must continue after its transient failure"
24205 );
24206 }
24207 }
24208
24209 #[tokio::test]
24210 async fn worker_continues_after_long_poll_capacity_backpressure() {
24211 let server = MockWorkerServer::capacity_limited_activity_poll();
24212 let client = Client::builder(server.base_url())
24213 .timeout(Duration::from_secs(2))
24214 .build()
24215 .expect("client");
24216 let mut worker = Worker::new(client, "rust-workers")
24217 .worker_id("capacity-worker")
24218 .poll_timeout(Duration::from_millis(10))
24219 .retry_policy(WorkerRetryPolicy {
24220 max_retries: 0,
24221 initial_backoff: Duration::from_millis(1),
24222 max_backoff: Duration::from_millis(1),
24223 });
24224 worker.register_activity("capacity.activity", |_ctx, _input| async move {
24225 Ok(json!({"handled": true}))
24226 });
24227
24228 worker
24229 .run_until(tokio::time::sleep(Duration::from_millis(50)))
24230 .await
24231 .expect("capacity backpressure must not stop the worker");
24232
24233 assert!(
24234 server.request_count("/api/worker/activity-tasks/poll") >= 2,
24235 "the activity poller must continue after capacity backpressure"
24236 );
24237 assert_eq!(
24238 server.request_count("/api/worker/activity-tasks/capacity-activity/complete"),
24239 1,
24240 "the worker must complete work returned after capacity recovers"
24241 );
24242 }
24243
24244 #[test]
24245 fn worker_poll_capacity_backpressure_requires_the_typed_retryable_contract() {
24246 let capacity = Error::Http {
24247 status: reqwest::StatusCode::TOO_MANY_REQUESTS,
24248 body: r#"{"poll_status":"long_poll_capacity_exhausted","retryable":true,"retry_after_seconds":3}"#.to_string(),
24249 };
24250 assert_eq!(
24251 worker_poll_capacity_retry_after(&capacity),
24252 Some(Duration::from_secs(3))
24253 );
24254
24255 let rejected_capacity = Error::Http {
24256 status: reqwest::StatusCode::TOO_MANY_REQUESTS,
24257 body: r#"{"reason":"long_poll_capacity_exhausted","retryable":false,"retry_after_seconds":3}"#.to_string(),
24258 };
24259 assert_eq!(worker_poll_capacity_retry_after(&rejected_capacity), None);
24260 assert!(!worker_operation_is_retryable(&rejected_capacity));
24261
24262 let ordinary_rate_limit = Error::Http {
24263 status: reqwest::StatusCode::TOO_MANY_REQUESTS,
24264 body: r#"{"reason":"rate_limited","retryable":true,"retry_after_seconds":3}"#
24265 .to_string(),
24266 };
24267 assert_eq!(worker_poll_capacity_retry_after(&ordinary_rate_limit), None);
24268 assert!(worker_operation_is_retryable(&ordinary_rate_limit));
24269 }
24270
24271 #[tokio::test]
24272 async fn worker_bounds_transport_retries() {
24273 let server = MockWorkerServer::unavailable_polls();
24274 let client = Client::builder(server.base_url())
24275 .timeout(Duration::from_secs(2))
24276 .build()
24277 .expect("client");
24278 let mut worker = Worker::new(client, "rust-workers")
24279 .worker_id("bounded-retry-worker")
24280 .poll_timeout(Duration::from_millis(10))
24281 .retry_policy(WorkerRetryPolicy {
24282 max_retries: 2,
24283 initial_backoff: Duration::from_millis(1),
24284 max_backoff: Duration::from_millis(1),
24285 });
24286 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
24287
24288 let error = worker.run().await.expect_err("retry bound must terminate");
24289 assert!(matches!(error, Error::Transport(_)));
24290 assert_eq!(
24291 server.request_count("/api/worker/workflow-tasks/poll"),
24292 3,
24293 "one initial request plus exactly two retries"
24294 );
24295 }
24296
24297 #[tokio::test]
24298 async fn worker_retry_policy_can_disable_poll_retries() {
24299 let server = MockWorkerServer::unavailable_polls();
24300 let client = Client::builder(server.base_url())
24301 .timeout(Duration::from_secs(2))
24302 .build()
24303 .expect("client");
24304 let mut worker = Worker::new(client, "rust-workers")
24305 .worker_id("no-retry-worker")
24306 .poll_timeout(Duration::from_millis(10))
24307 .retry_policy(WorkerRetryPolicy {
24308 max_retries: 0,
24309 initial_backoff: Duration::from_millis(1),
24310 max_backoff: Duration::from_millis(1),
24311 });
24312 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
24313
24314 let error = worker
24315 .run_once()
24316 .await
24317 .expect_err("disabled retries must return the first transport failure");
24318 assert!(matches!(error, Error::Transport(_)));
24319 assert_eq!(
24320 server.request_count("/api/worker/workflow-tasks/poll"),
24321 1,
24322 "max_retries=0 must send only the initial request"
24323 );
24324 }
24325
24326 #[tokio::test]
24327 async fn worker_does_not_retry_authentication_failures() {
24328 let server = MockWorkerServer::unauthorized_polls();
24329 let client = Client::builder(server.base_url())
24330 .timeout(Duration::from_secs(2))
24331 .build()
24332 .expect("client");
24333 let mut worker = Worker::new(client, "rust-workers")
24334 .worker_id("unauthorized-worker")
24335 .poll_timeout(Duration::from_millis(10));
24336 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
24337
24338 let error = worker
24339 .run()
24340 .await
24341 .expect_err("authentication must terminate");
24342 let Error::Http { status, body } = error else {
24343 panic!("expected stable HTTP authentication error");
24344 };
24345 assert_eq!(status, reqwest::StatusCode::UNAUTHORIZED);
24346 assert!(body.contains("authentication_failed"));
24347 assert_eq!(
24348 server.request_count("/api/worker/workflow-tasks/poll"),
24349 1,
24350 "authentication failures must not be retried"
24351 );
24352 }
24353
24354 #[derive(Clone, Debug)]
24355 struct CapturedRequest {
24356 headers: String,
24357 method: String,
24358 path: String,
24359 authorization: Option<String>,
24360 namespace: Option<String>,
24361 worker_protocol: Option<String>,
24362 control_protocol: Option<String>,
24363 body: String,
24364 received_at: Instant,
24365 }
24366
24367 struct MockWorkerServer {
24368 addr: SocketAddr,
24369 stop: Arc<AtomicBool>,
24370 requests: Arc<Mutex<Vec<CapturedRequest>>>,
24371 thread: Option<thread::JoinHandle<()>>,
24372 }
24373
24374 type RequestOverride = fn(&str, &str, usize) -> Option<(&'static str, String)>;
24375
24376 #[derive(Clone, Copy, Default)]
24377 struct MockWorkerBehavior {
24378 response_override: Option<fn(&str) -> Option<(&'static str, String)>>,
24379 request_override: Option<RequestOverride>,
24380 storage_refusals: usize,
24381 storage_path: Option<&'static str>,
24382 storage_unavailable: bool,
24383 storage_mid_poll: bool,
24384 storage_activity: bool,
24385 storage_query: bool,
24386 storage_wrong_poll_id: bool,
24387 reject_query_protocol: bool,
24388 reject_query_completion: bool,
24389 waiting_query_worker: bool,
24390 decline_registration: bool,
24391 complete_named_signal: bool,
24392 poll_failures_per_path: usize,
24393 long_poll_capacity_responses_per_path: usize,
24394 heartbeat_failures: usize,
24395 heartbeat_failure_request: Option<usize>,
24396 delayed_heartbeat_request: Option<usize>,
24397 heartbeat_response_delay: Duration,
24398 concurrent_requests: bool,
24399 unauthorized_polls: bool,
24400 reject_registration: bool,
24401 reject_registration_protocol: bool,
24402 reject_deregistration: bool,
24403 reject_deregistration_protocol: bool,
24404 cancelled_activity: bool,
24405 draining_polls: bool,
24406 invalid_task_payload_codec: Option<InvalidTaskPayloadCodec>,
24407 workflow_completion_status: Option<&'static str>,
24408 workflow_completion_body: Option<&'static str>,
24409 }
24410
24411 impl MockWorkerServer {
24412 fn start() -> Self {
24413 Self::start_with_behavior(MockWorkerBehavior::default())
24414 }
24415
24416 fn reject_query_protocol() -> Self {
24417 Self::start_with_behavior(MockWorkerBehavior {
24418 reject_query_protocol: true,
24419 ..MockWorkerBehavior::default()
24420 })
24421 }
24422
24423 fn reject_query_completion() -> Self {
24424 Self::start_with_behavior(MockWorkerBehavior {
24425 reject_query_completion: true,
24426 ..MockWorkerBehavior::default()
24427 })
24428 }
24429
24430 fn waiting_query_worker() -> Self {
24431 Self::start_with_behavior(MockWorkerBehavior {
24432 waiting_query_worker: true,
24433 complete_named_signal: true,
24434 ..MockWorkerBehavior::default()
24435 })
24436 }
24437
24438 fn transient_worker_failures() -> Self {
24439 Self::start_with_behavior(MockWorkerBehavior {
24440 poll_failures_per_path: 1,
24441 heartbeat_failures: 1,
24442 ..MockWorkerBehavior::default()
24443 })
24444 }
24445
24446 fn consecutive_poll_failures(count: usize) -> Self {
24447 Self::start_with_behavior(MockWorkerBehavior {
24448 poll_failures_per_path: count,
24449 ..MockWorkerBehavior::default()
24450 })
24451 }
24452
24453 fn capacity_limited_activity_poll() -> Self {
24454 Self::start_with_behavior(MockWorkerBehavior {
24455 long_poll_capacity_responses_per_path: 1,
24456 ..MockWorkerBehavior::default()
24457 })
24458 }
24459
24460 fn delayed_heartbeat_worker() -> Self {
24461 Self::start_with_behavior(MockWorkerBehavior {
24462 waiting_query_worker: true,
24463 delayed_heartbeat_request: Some(2),
24464 heartbeat_response_delay: Duration::from_millis(1_500),
24465 concurrent_requests: true,
24466 cancelled_activity: true,
24467 ..MockWorkerBehavior::default()
24468 })
24469 }
24470
24471 fn heartbeat_retry_worker() -> Self {
24472 Self::start_with_behavior(MockWorkerBehavior {
24473 waiting_query_worker: true,
24474 heartbeat_failure_request: Some(2),
24475 concurrent_requests: true,
24476 ..MockWorkerBehavior::default()
24477 })
24478 }
24479
24480 fn unavailable_polls() -> Self {
24481 Self::start_with_behavior(MockWorkerBehavior {
24482 poll_failures_per_path: usize::MAX,
24483 ..MockWorkerBehavior::default()
24484 })
24485 }
24486
24487 fn unauthorized_polls() -> Self {
24488 Self::start_with_behavior(MockWorkerBehavior {
24489 unauthorized_polls: true,
24490 ..MockWorkerBehavior::default()
24491 })
24492 }
24493
24494 fn rejected_registration() -> Self {
24495 Self::start_with_behavior(MockWorkerBehavior {
24496 reject_registration: true,
24497 ..MockWorkerBehavior::default()
24498 })
24499 }
24500
24501 fn rejected_registration_protocol() -> Self {
24502 Self::start_with_behavior(MockWorkerBehavior {
24503 reject_registration_protocol: true,
24504 ..MockWorkerBehavior::default()
24505 })
24506 }
24507
24508 fn declined_registration() -> Self {
24509 Self::start_with_behavior(MockWorkerBehavior {
24510 decline_registration: true,
24511 ..MockWorkerBehavior::default()
24512 })
24513 }
24514
24515 fn rejected_deregistration() -> Self {
24516 Self::start_with_behavior(MockWorkerBehavior {
24517 reject_deregistration: true,
24518 ..MockWorkerBehavior::default()
24519 })
24520 }
24521
24522 fn rejected_deregistration_protocol() -> Self {
24523 Self::start_with_behavior(MockWorkerBehavior {
24524 reject_deregistration_protocol: true,
24525 ..MockWorkerBehavior::default()
24526 })
24527 }
24528
24529 fn unauthorized_polls_and_rejected_deregistration() -> Self {
24530 Self::start_with_behavior(MockWorkerBehavior {
24531 unauthorized_polls: true,
24532 reject_deregistration: true,
24533 ..MockWorkerBehavior::default()
24534 })
24535 }
24536
24537 fn cancelled_activity() -> Self {
24538 Self::start_with_behavior(MockWorkerBehavior {
24539 cancelled_activity: true,
24540 ..MockWorkerBehavior::default()
24541 })
24542 }
24543
24544 fn draining_polls() -> Self {
24545 Self::start_with_behavior(MockWorkerBehavior {
24546 draining_polls: true,
24547 ..MockWorkerBehavior::default()
24548 })
24549 }
24550
24551 fn invalid_task_payload_codec(codec: InvalidTaskPayloadCodec) -> Self {
24552 Self::start_with_behavior(MockWorkerBehavior {
24553 invalid_task_payload_codec: Some(codec),
24554 ..MockWorkerBehavior::default()
24555 })
24556 }
24557
24558 fn workflow_completion(status: &'static str, body: &'static str) -> Self {
24559 Self::start_with_behavior(MockWorkerBehavior {
24560 workflow_completion_status: Some(status),
24561 workflow_completion_body: Some(body),
24562 ..MockWorkerBehavior::default()
24563 })
24564 }
24565
24566 fn start_with_behavior(behavior: MockWorkerBehavior) -> Self {
24567 let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock server");
24568 listener
24569 .set_nonblocking(true)
24570 .expect("configure mock listener");
24571 let addr = listener.local_addr().expect("mock server address");
24572 let stop = Arc::new(AtomicBool::new(false));
24573 let server_stop = Arc::clone(&stop);
24574 let requests = Arc::new(Mutex::new(Vec::new()));
24575 let server_requests = Arc::clone(&requests);
24576 let thread = thread::spawn(move || {
24577 let mut request_threads = Vec::new();
24578 while !server_stop.load(Ordering::SeqCst) {
24579 match listener.accept() {
24580 Ok((mut stream, _)) => {
24581 if behavior.concurrent_requests {
24582 let requests = Arc::clone(&server_requests);
24583 request_threads.push(thread::spawn(move || {
24584 handle_mock_worker_request(&mut stream, &requests, behavior)
24585 }));
24586 } else {
24587 handle_mock_worker_request(&mut stream, &server_requests, behavior);
24588 }
24589 }
24590 Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
24591 let mut index = 0;
24592 while index < request_threads.len() {
24593 if request_threads[index].is_finished() {
24594 request_threads
24595 .swap_remove(index)
24596 .join()
24597 .expect("join mock request");
24598 } else {
24599 index += 1;
24600 }
24601 }
24602 thread::sleep(Duration::from_millis(5));
24603 }
24604 Err(_) => break,
24605 }
24606 }
24607 for request_thread in request_threads {
24608 request_thread.join().expect("join mock request");
24609 }
24610 });
24611
24612 Self {
24613 addr,
24614 stop,
24615 requests,
24616 thread: Some(thread),
24617 }
24618 }
24619
24620 fn base_url(&self) -> String {
24621 format!("http://{}", self.addr)
24622 }
24623
24624 fn worker_protocol_for(&self, path: &str) -> Option<String> {
24625 self.requests
24626 .lock()
24627 .expect("captured requests")
24628 .iter()
24629 .find(|request| request.path == path)
24630 .and_then(|request| request.worker_protocol.clone())
24631 }
24632
24633 fn control_protocol_for(&self, path: &str) -> Option<String> {
24634 self.requests
24635 .lock()
24636 .expect("captured requests")
24637 .iter()
24638 .find(|request| request.path == path)
24639 .and_then(|request| request.control_protocol.clone())
24640 }
24641
24642 fn method_for(&self, path: &str) -> Option<String> {
24643 self.requests
24644 .lock()
24645 .expect("captured requests")
24646 .iter()
24647 .find(|request| request.path == path)
24648 .map(|request| request.method.clone())
24649 }
24650
24651 fn authorization_for(&self, path: &str) -> Option<String> {
24652 self.requests
24653 .lock()
24654 .expect("captured requests")
24655 .iter()
24656 .find(|request| request.path == path)
24657 .and_then(|request| request.authorization.clone())
24658 }
24659
24660 fn namespace_for(&self, path: &str) -> Option<String> {
24661 self.requests
24662 .lock()
24663 .expect("captured requests")
24664 .iter()
24665 .find(|request| request.path == path)
24666 .and_then(|request| request.namespace.clone())
24667 }
24668
24669 fn request_count(&self, path: &str) -> usize {
24670 self.requests
24671 .lock()
24672 .expect("captured requests")
24673 .iter()
24674 .filter(|request| request.path == path)
24675 .count()
24676 }
24677
24678 fn captured_paths(&self) -> Vec<String> {
24679 self.requests
24680 .lock()
24681 .expect("captured requests")
24682 .iter()
24683 .map(|request| request.path.clone())
24684 .collect()
24685 }
24686
24687 fn request_times(&self, path: &str) -> Vec<Instant> {
24688 self.requests
24689 .lock()
24690 .expect("captured requests")
24691 .iter()
24692 .filter(|request| request.path == path)
24693 .map(|request| request.received_at)
24694 .collect()
24695 }
24696
24697 fn request_body(&self, path: &str) -> Value {
24698 let requests = self.requests.lock().expect("captured requests");
24699 let body = &requests
24700 .iter()
24701 .find(|request| request.path == path)
24702 .unwrap_or_else(|| panic!("missing request for {path}"))
24703 .body;
24704 serde_json::from_str(body).unwrap_or_else(|error| {
24705 panic!("invalid JSON request body for {path}: {error}: {body:?}")
24706 })
24707 }
24708
24709 fn request_bodies(&self, path: &str) -> Vec<Value> {
24710 self.requests
24711 .lock()
24712 .expect("captured requests")
24713 .iter()
24714 .filter(|request| request.path == path)
24715 .map(|request| {
24716 serde_json::from_str(&request.body).unwrap_or_else(|error| {
24717 panic!(
24718 "invalid JSON request body for {path}: {error}: {:?}",
24719 request.body
24720 )
24721 })
24722 })
24723 .collect()
24724 }
24725 }
24726
24727 impl Drop for MockWorkerServer {
24728 fn drop(&mut self) {
24729 self.stop.store(true, Ordering::SeqCst);
24730 let _ = TcpStream::connect(self.addr);
24731
24732 if let Some(thread) = self.thread.take() {
24733 thread.join().expect("join mock server");
24734 }
24735 }
24736 }
24737
24738 fn handle_mock_worker_request(
24739 stream: &mut TcpStream,
24740 requests: &Arc<Mutex<Vec<CapturedRequest>>>,
24741 behavior: MockWorkerBehavior,
24742 ) {
24743 let _ = stream.set_read_timeout(Some(Duration::from_millis(200)));
24744 let mut buffer = [0_u8; 8192];
24745 let mut request = Vec::new();
24746
24747 loop {
24748 match stream.read(&mut buffer) {
24749 Ok(0) => break,
24750 Ok(read) => {
24751 request.extend_from_slice(&buffer[..read]);
24752 if mock_request_is_complete(&request) {
24753 break;
24754 }
24755 }
24756 Err(error)
24757 if matches!(
24758 error.kind(),
24759 std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
24760 ) =>
24761 {
24762 break;
24763 }
24764 Err(_) => return,
24765 }
24766 }
24767
24768 let request = String::from_utf8_lossy(&request);
24769 let body = request
24770 .split_once("\r\n\r\n")
24771 .map(|(_, body)| body)
24772 .unwrap_or_default();
24773 let path = request
24774 .lines()
24775 .next()
24776 .and_then(|line| line.split_whitespace().nth(1))
24777 .unwrap_or_default();
24778 let method = request
24779 .lines()
24780 .next()
24781 .and_then(|line| line.split_whitespace().next())
24782 .unwrap_or_default();
24783 let authorization = request.lines().find_map(|line| {
24784 let (name, value) = line.split_once(':')?;
24785 name.eq_ignore_ascii_case("Authorization")
24786 .then(|| value.trim().to_string())
24787 });
24788 let namespace = request.lines().find_map(|line| {
24789 let (name, value) = line.split_once(':')?;
24790 name.eq_ignore_ascii_case("X-Namespace")
24791 .then(|| value.trim().to_string())
24792 });
24793 let worker_protocol = request.lines().find_map(|line| {
24794 let (name, value) = line.split_once(':')?;
24795 name.eq_ignore_ascii_case("X-Durable-Workflow-Protocol-Version")
24796 .then(|| value.trim().to_string())
24797 });
24798 let control_protocol = request.lines().find_map(|line| {
24799 let (name, value) = line.split_once(':')?;
24800 name.eq_ignore_ascii_case("X-Durable-Workflow-Control-Plane-Version")
24801 .then(|| value.trim().to_string())
24802 });
24803 let request_number = {
24804 let mut requests = requests.lock().expect("captured requests");
24805 requests.push(CapturedRequest {
24806 headers: request
24807 .split_once("\r\n\r\n")
24808 .map_or("", |(headers, _)| headers)
24809 .to_owned(),
24810 method: method.to_string(),
24811 path: path.to_string(),
24812 authorization,
24813 namespace,
24814 worker_protocol: worker_protocol.clone(),
24815 control_protocol,
24816 body: body.to_string(),
24817 received_at: Instant::now(),
24818 });
24819 requests
24820 .iter()
24821 .filter(|request| request.path == path)
24822 .count()
24823 };
24824
24825 if let Some(response) = behavior
24826 .request_override
24827 .and_then(|handler| handler(path, body, request_number))
24828 {
24829 write_mock_response(stream, response.0, &response.1);
24830 return;
24831 }
24832 if let Some(response) = behavior.response_override.and_then(|handler| handler(path)) {
24833 write_mock_response(stream, response.0, &response.1);
24834 return;
24835 }
24836 if path.ends_with("/poll") && request_number <= behavior.poll_failures_per_path {
24837 return;
24838 }
24839 let pressure_path = behavior
24840 .storage_path
24841 .is_some_and(|part| path.contains(part));
24842 let prior_failures = if path.ends_with("/poll") {
24843 behavior.poll_failures_per_path
24844 } else {
24845 0
24846 };
24847 if pressure_path
24848 && request_number.saturating_sub(prior_failures) <= behavior.storage_refusals
24849 {
24850 let request_body: Value = serde_json::from_str(body).unwrap_or(Value::Null);
24851 let poll_id = path
24852 .ends_with("/poll")
24853 .then(|| request_body["poll_request_id"].as_str().unwrap_or(""));
24854 let mut refusal = storage_refusal(
24855 poll_id,
24856 behavior.storage_unavailable,
24857 behavior.storage_mid_poll,
24858 );
24859 if behavior.storage_wrong_poll_id {
24860 refusal["poll_request_id"] = json!("wrong-poll");
24861 }
24862 write_mock_response(stream, "503 Service Unavailable", &refusal.to_string());
24863 return;
24864 }
24865 if path.contains("/storage-task/")
24866 || path.contains("/storage-activity/")
24867 || path.contains("/storage-query/")
24868 {
24869 write_mock_response(stream, "200 OK", "{}");
24870 return;
24871 }
24872 if behavior.storage_query && path == "/api/worker/query-tasks/poll" && request_number == 1 {
24873 write_mock_response(stream, "200 OK", &json!({"task":{
24874 "query_task_id":"storage-query", "query_task_attempt":7, "workflow_type":"storage.workflow",
24875 "query_name":"state", "workflow_id":"workflow", "run_id":"run", "payload_codec":"avro",
24876 "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC).unwrap(),
24877 "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC).unwrap(),
24878 "history_events":[], "run_status":"waiting", "lease_owner":"storage-worker"
24879 }}).to_string());
24880 return;
24881 }
24882 if behavior.storage_activity
24883 && path == "/api/worker/activity-tasks/poll"
24884 && request_number == 1
24885 {
24886 write_mock_response(stream, "200 OK", &json!({"task":{
24887 "task_id":"storage-activity", "activity_attempt_id":"storage-attempt", "activity_type":"storage.activity",
24888 "payload_codec":"avro", "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC).unwrap(),
24889 "attempt_number":7, "lease_owner":"storage-worker"
24890 }}).to_string());
24891 return;
24892 }
24893
24894 if path == "/api/worker/register" {
24895 if behavior.reject_registration_protocol {
24896 write_mock_response(
24897 stream,
24898 "400 Bad Request",
24899 r#"{"reason":"unsupported_protocol_version","message":"condition-wait occurrence identity requires worker protocol 1.17","supported_version":"1.16","requested_version":"1.17"}"#,
24900 );
24901 return;
24902 }
24903 if behavior.reject_registration {
24904 write_mock_response(
24905 stream,
24906 "503 Service Unavailable",
24907 r#"{"reason":"registration_unavailable","message":"registration failed"}"#,
24908 );
24909 return;
24910 }
24911 }
24912
24913 if path.starts_with("/api/worker/registrations/") {
24914 if behavior.reject_deregistration_protocol {
24915 write_mock_response(
24916 stream,
24917 "400 Bad Request",
24918 r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.17","requested_version":"1.19"}"#,
24919 );
24920 } else if behavior.reject_deregistration {
24921 write_mock_response(
24922 stream,
24923 "403 Forbidden",
24924 r#"{"reason":"authorization_failed","message":"worker cannot deregister"}"#,
24925 );
24926 } else {
24927 write_mock_response(
24928 stream,
24929 "200 OK",
24930 r#"{"worker_id":"deregistered-worker","outcome":"deregistered","recovered_workflow_task_count":2}"#,
24931 );
24932 }
24933 return;
24934 }
24935
24936 let is_poll = matches!(
24937 path,
24938 "/api/worker/workflow-tasks/poll"
24939 | "/api/worker/activity-tasks/poll"
24940 | "/api/worker/query-tasks/poll"
24941 );
24942 if is_poll && request_number <= behavior.long_poll_capacity_responses_per_path {
24943 write_mock_response(
24944 stream,
24945 "429 Too Many Requests",
24946 r#"{"task":null,"poll_status":"long_poll_capacity_exhausted","reason":"long_poll_capacity_exhausted","retryable":true,"retry_after_seconds":1}"#,
24947 );
24948 return;
24949 }
24950 if is_poll && request_number <= behavior.poll_failures_per_path {
24951 return;
24952 }
24953 if path == "/api/worker/heartbeat" && request_number <= behavior.heartbeat_failures {
24954 return;
24955 }
24956 if path == "/api/worker/heartbeat"
24957 && behavior.heartbeat_failure_request == Some(request_number)
24958 {
24959 return;
24960 }
24961 if path == "/api/worker/heartbeat"
24962 && behavior.delayed_heartbeat_request == Some(request_number)
24963 {
24964 thread::sleep(behavior.heartbeat_response_delay);
24965 }
24966 if behavior.unauthorized_polls && is_poll {
24967 write_mock_response(
24968 stream,
24969 "401 Unauthorized",
24970 r#"{"reason":"authentication_failed","message":"invalid worker token"}"#,
24971 );
24972 return;
24973 }
24974 if behavior.draining_polls && is_poll {
24975 write_mock_response(
24976 stream,
24977 "409 Conflict",
24978 r#"{"task":null,"poll_status":"draining","reason":"worker_draining","worker_status":"draining","drain_intent":"draining"}"#,
24979 );
24980 return;
24981 }
24982
24983 if let Some(codec_case) = behavior.invalid_task_payload_codec {
24984 if is_poll && request_number == 1 {
24985 let mut task = match path {
24986 "/api/worker/workflow-tasks/poll" => json!({
24987 "task_id": "codec-workflow",
24988 "workflow_type": "codec.workflow",
24989 "payload_codec": DEFAULT_CODEC,
24990 "workflow_task_attempt": 1,
24991 "lease_owner": "codec-worker"
24992 }),
24993 "/api/worker/activity-tasks/poll" => json!({
24994 "task_id": "codec-activity",
24995 "activity_attempt_id": "codec-activity-attempt",
24996 "activity_type": "codec.activity",
24997 "payload_codec": DEFAULT_CODEC,
24998 "attempt_number": 1,
24999 "lease_owner": "codec-worker"
25000 }),
25001 "/api/worker/query-tasks/poll" => json!({
25002 "query_task_id": "codec-query",
25003 "query_task_attempt": 1,
25004 "workflow_type": "codec.workflow",
25005 "query_name": "known",
25006 "payload_codec": DEFAULT_CODEC,
25007 "lease_owner": "codec-worker"
25008 }),
25009 _ => unreachable!("is_poll limits task codec probe paths"),
25010 };
25011 codec_case.apply(&mut task);
25012 write_mock_response(stream, "200 OK", &json!({"task": task}).to_string());
25013 return;
25014 }
25015
25016 if matches!(
25017 path,
25018 "/api/worker/workflow-tasks/codec-workflow/fail"
25019 | "/api/worker/activity-tasks/codec-activity/fail"
25020 | "/api/worker/query-tasks/codec-query/fail"
25021 ) {
25022 write_mock_response(stream, "200 OK", r#"{"outcome":"failed"}"#);
25023 return;
25024 }
25025 }
25026
25027 if behavior.reject_query_protocol && path.starts_with("/api/worker/query-tasks/") {
25028 let requested_version = worker_protocol.as_deref().unwrap_or("missing");
25029 let body = format!(
25030 r#"{{"reason":"unsupported_protocol_version","message":"Query tasks require worker protocol 1.8 or newer.","supported_version":"1.7","requested_version":"{requested_version}"}}"#
25031 );
25032 write_mock_response(stream, "400 Bad Request", &body);
25033 return;
25034 }
25035
25036 if behavior.reject_query_completion && path == "/api/worker/query-tasks/query-late/complete"
25037 {
25038 write_mock_response(
25039 stream,
25040 "409 Conflict",
25041 r#"{"reason":"query_task_timed_out","message":"query task timed out before completion"}"#,
25042 );
25043 return;
25044 }
25045
25046 if behavior.workflow_completion_status.is_some()
25047 && path == "/api/worker/workflow-tasks/poll"
25048 && request_number == 1
25049 {
25050 write_mock_response(
25051 stream,
25052 "200 OK",
25053 r#"{"task":{"task_id":"workflow-timeout-task","workflow_id":"reused-workflow-id","run_id":"run-selected-timeout","workflow_type":"timeout.workflow","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"workflow_task_attempt":3,"lease_owner":"timeout-worker"}}"#,
25054 );
25055 return;
25056 }
25057
25058 if path == "/api/worker/workflow-tasks/workflow-timeout-task/complete" {
25059 if let (Some(status), Some(body)) = (
25060 behavior.workflow_completion_status,
25061 behavior.workflow_completion_body,
25062 ) {
25063 write_mock_response(stream, status, body);
25064 return;
25065 }
25066 }
25067
25068 if behavior.waiting_query_worker {
25069 if behavior.complete_named_signal
25070 && path == "/api/worker/workflow-tasks/poll"
25071 && request_number == 1
25072 {
25073 let body = json!({
25074 "task": {
25075 "task_id": "snapshot-open",
25076 "workflow_id": "snapshot-1",
25077 "run_id": "snapshot-run-1",
25078 "workflow_type": "snapshot",
25079 "payload_codec": DEFAULT_CODEC,
25080 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25081 .expect("Avro workflow arguments"),
25082 "history_events": [],
25083 "workflow_task_attempt": 1,
25084 "lease_owner": "rust-snapshot-worker"
25085 }
25086 })
25087 .to_string();
25088 write_mock_response(stream, "200 OK", &body);
25089 return;
25090 }
25091
25092 let signal_request = request_number - usize::from(behavior.complete_named_signal);
25093 let signal_request_limit = 2 + usize::from(behavior.complete_named_signal);
25094 if path == "/api/worker/workflow-tasks/poll"
25095 && signal_request >= 1
25096 && signal_request <= signal_request_limit
25097 {
25098 let finish = behavior.complete_named_signal && signal_request == 3;
25099 let amounts = if signal_request == 1 {
25100 vec![3]
25101 } else {
25102 vec![3, 5]
25103 };
25104 let task_id = if signal_request == 1 {
25105 "snapshot-wait-3"
25106 } else if finish {
25107 "snapshot-finish"
25108 } else {
25109 "snapshot-wait-5"
25110 };
25111 let mut history_events = std::iter::once(json!({
25112 "event_type": "SignalWaitOpened",
25113 "payload": {"sequence": 1, "signal_name": "finish"}
25114 }))
25115 .chain(amounts.iter().enumerate().map(|(index, amount)| {
25116 json!({
25117 "event_type": "SignalReceived",
25118 "payload": {
25119 "signal_id": format!("increment-{amount}"),
25120 "signal_name": "increment",
25121 "workflow_sequence": index + 2,
25122 "payload_codec": DEFAULT_CODEC,
25123 "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
25124 .expect("Avro signal envelope")
25125 }
25126 })
25127 }))
25128 .collect::<Vec<_>>();
25129 let (resume_id, resume_name, resume_arguments) = if finish {
25130 history_events.push(json!({
25131 "event_type": "SignalReceived",
25132 "payload": {
25133 "signal_id": "finish",
25134 "signal_name": "finish",
25135 "workflow_sequence": 4,
25136 "payload_codec": DEFAULT_CODEC,
25137 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25138 .expect("Avro finish signal envelope")
25139 }
25140 }));
25141 (
25142 "finish".to_string(),
25143 "finish".to_string(),
25144 encode_value_envelope(&json!([]), DEFAULT_CODEC)
25145 .expect("Avro finish resume signal"),
25146 )
25147 } else {
25148 let amount = amounts.last().expect("amount");
25149 (
25150 format!("increment-{amount}"),
25151 "increment".to_string(),
25152 encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
25153 .expect("Avro increment resume signal"),
25154 )
25155 };
25156 let body = json!({
25157 "task": {
25158 "task_id": task_id,
25159 "workflow_id": "snapshot-1",
25160 "run_id": "snapshot-run-1",
25161 "workflow_type": "snapshot",
25162 "payload_codec": DEFAULT_CODEC,
25163 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25164 .expect("Avro workflow arguments"),
25165 "history_events": history_events,
25166 "workflow_task_attempt": 1,
25167 "workflow_signal_id": resume_id,
25168 "signal_name": resume_name,
25169 "signal_arguments": resume_arguments,
25170 "lease_owner": "rust-snapshot-worker"
25171 }
25172 })
25173 .to_string();
25174 write_mock_response(stream, "200 OK", &body);
25175 return;
25176 }
25177
25178 if path == "/api/worker/query-tasks/poll" && request_number == 1 {
25179 let history_events = [3, 5]
25180 .into_iter()
25181 .enumerate()
25182 .map(|(index, amount)| {
25183 json!({
25184 "event_type": "SignalReceived",
25185 "payload": {
25186 "signal_id": format!("increment-{amount}"),
25187 "signal_name": "increment",
25188 "workflow_sequence": index + 2,
25189 "payload_codec": DEFAULT_CODEC,
25190 "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
25191 .expect("Avro query signal envelope")
25192 }
25193 })
25194 })
25195 .collect::<Vec<_>>();
25196 let body = json!({
25197 "task": {
25198 "query_task_id": "snapshot-current",
25199 "query_task_attempt": 1,
25200 "lease_owner": "rust-snapshot-worker",
25201 "workflow_id": "snapshot-1",
25202 "run_id": "snapshot-run-1",
25203 "workflow_type": "snapshot",
25204 "query_name": "current",
25205 "payload_codec": DEFAULT_CODEC,
25206 "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25207 .expect("Avro workflow arguments"),
25208 "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25209 .expect("Avro query arguments"),
25210 "history_events": history_events,
25211 "run_status": "waiting"
25212 }
25213 })
25214 .to_string();
25215 write_mock_response(stream, "200 OK", &body);
25216 return;
25217 }
25218
25219 if path == "/api/worker/workflow-tasks/snapshot-wait-3/fail"
25220 || path == "/api/worker/workflow-tasks/snapshot-wait-5/fail"
25221 {
25222 write_mock_response(
25223 stream,
25224 "200 OK",
25225 r#"{"outcome":"waiting_for_history","recorded":true}"#,
25226 );
25227 return;
25228 }
25229
25230 if path == "/api/worker/workflow-tasks/snapshot-open/complete" {
25231 write_mock_response(stream, "200 OK", r#"{"outcome":"waiting","recorded":true}"#);
25232 return;
25233 }
25234
25235 if path == "/api/worker/workflow-tasks/snapshot-finish/complete" {
25236 write_mock_response(
25237 stream,
25238 "200 OK",
25239 r#"{"outcome":"completed","run_status":"completed","recorded":true}"#,
25240 );
25241 return;
25242 }
25243
25244 if path == "/api/worker/query-tasks/snapshot-current/complete" {
25245 write_mock_response(stream, "200 OK", r#"{"outcome":"completed"}"#);
25246 return;
25247 }
25248 }
25249
25250 if matches!(
25251 path,
25252 "/api/workflows/typed-1/query/inspect" | "/api/workflows/typed-1/update/replace"
25253 ) {
25254 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
25255 .expect("typed mock result");
25256 let body = json!({
25257 "result": typed_fidelity_probe().into_json().expect("result projection"),
25258 "result_envelope": result,
25259 })
25260 .to_string();
25261 write_mock_response(stream, "200 OK", &body);
25262 return;
25263 }
25264
25265 if path == "/api/workflows/typed-1" {
25266 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
25267 .expect("typed mock result");
25268 let body = json!({
25269 "workflow_id": "typed-1",
25270 "run_id": "run-typed-1",
25271 "workflow_type": "typed.echo",
25272 "status": "completed",
25273 "output": typed_fidelity_probe().into_json().expect("output projection"),
25274 "output_envelope": result,
25275 })
25276 .to_string();
25277 write_mock_response(stream, "200 OK", &body);
25278 return;
25279 }
25280
25281 let (status, body) = match path {
25282 "/api/cluster/info" => ("200 OK", r#"{"limits":{"max_payload_bytes":2097152}}"#),
25283 "/api/health" => ("200 OK", r#"{"status":"ok"}"#),
25284 "/api/workflows" => (
25285 "201 Created",
25286 r#"{"workflow_id":"wf-start-options","run_id":"run-start-options","workflow_type":"rust.timeout"}"#,
25287 ),
25288 "/api/worker/register" if behavior.decline_registration => (
25289 "200 OK",
25290 r#"{"worker_id":"declined-worker","registered":false}"#,
25291 ),
25292 "/api/worker/register" if behavior.waiting_query_worker => (
25293 "200 OK",
25294 r#"{"worker_id":"rust-snapshot-worker","registered":true,"heartbeat_interval_seconds":1}"#,
25295 ),
25296 "/api/worker/register" => (
25297 "200 OK",
25298 r#"{"worker_id":"mock-worker","registered":true,"heartbeat_interval_seconds":3600}"#,
25299 ),
25300 "/api/worker/heartbeat" => ("200 OK", "{}"),
25301 "/api/worker/activity-tasks/poll"
25302 if behavior.cancelled_activity && request_number == 1 =>
25303 {
25304 (
25305 "200 OK",
25306 r#"{"task":{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","activity_type":"cancel-aware","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"rust-cancel-worker"}}"#,
25307 )
25308 }
25309 "/api/worker/activity-tasks/poll"
25310 if behavior.long_poll_capacity_responses_per_path > 0
25311 && request_number
25312 == behavior
25313 .long_poll_capacity_responses_per_path
25314 .saturating_add(1) =>
25315 {
25316 (
25317 "200 OK",
25318 r#"{"task":{"task_id":"capacity-activity","activity_attempt_id":"capacity-attempt","activity_type":"capacity.activity","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"capacity-worker"}}"#,
25319 )
25320 }
25321 "/api/worker/activity-tasks/poll" | "/api/worker/workflow-tasks/poll" => {
25322 ("200 OK", r#"{"task":null}"#)
25323 }
25324 "/api/worker/query-tasks/poll"
25325 if behavior.reject_query_completion && request_number == 1 =>
25326 {
25327 (
25328 "200 OK",
25329 r#"{"task":{"query_task_id":"query-late","query_task_attempt":1,"lease_owner":"late-worker","workflow_id":"counter-late","run_id":"run-late","workflow_type":"counter","query_name":"current","payload_codec":"avro","workflow_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"query_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"run_status":"running"}}"#,
25330 )
25331 }
25332 "/api/worker/query-tasks/poll" => ("200 OK", r#"{"task":null}"#),
25333 "/api/worker/query-tasks/query-capture/complete"
25334 | "/api/worker/query-tasks/query-capture/fail" => ("200 OK", "{}"),
25335 "/api/worker/activity-tasks/activity-cancel/heartbeat" => (
25336 "200 OK",
25337 r#"{"activity_attempt_id":"attempt-cancel","cancel_requested":true,"can_continue":false,"reason":"run_cancelled","run_closed_reason":"cancelled","heartbeat_recorded":false}"#,
25338 ),
25339 "/api/worker/activity-tasks/activity-cancel/complete" => (
25340 "409 Conflict",
25341 r#"{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","reason":"run_cancelled","cancel_requested":true,"can_continue":false,"run_closed_reason":"cancelled"}"#,
25342 ),
25343 "/api/worker/activity-tasks/activity-typed/complete"
25344 | "/api/worker/activity-tasks/activity-typed/fail"
25345 | "/api/worker/activity-tasks/capacity-activity/complete"
25346 | "/api/workflows/typed-1/signal/changed" => ("200 OK", "{}"),
25347 "/api/workflows/counter-1/query/current" => (
25348 "200 OK",
25349 r#"{"workflow_id":"counter-1","query_name":"current","result":{"count":8},"result_envelope":{"codec":"avro","blob":"wwHioz3/VYAiNw4CCmNvdW50BBAA"}}"#,
25350 ),
25351 "/api/workflows/counter-1/query/missing" => (
25352 "404 Not Found",
25353 r#"{"workflow_id":"counter-1","query_name":"missing","reason":"rejected_unknown_query","message":"unknown query"}"#,
25354 ),
25355 "/api/workflows/wf-lifecycle/cancel" => (
25356 "200 OK",
25357 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","reason":"cleanup requested","command_status":"accepted"}"#,
25358 ),
25359 "/api/workflows/wf-lifecycle/terminate" => (
25360 "200 OK",
25361 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","reason":"forced stop","command_status":"accepted"}"#,
25362 ),
25363 "/api/workflows/wf-lifecycle/runs/run-current/cancel" => (
25364 "200 OK",
25365 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","command_status":"accepted"}"#,
25366 ),
25367 "/api/workflows/wf-lifecycle/runs/run-current/terminate" => (
25368 "200 OK",
25369 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","command_status":"accepted"}"#,
25370 ),
25371 "/api/workflows/wf-lifecycle/runs/run-stale/cancel"
25372 | "/api/workflows/wf-lifecycle/runs/run-stale/terminate" => (
25373 "409 Conflict",
25374 r#"{"workflow_id":"wf-lifecycle","run_id":"run-stale","reason":"historical_run_command_rejected","target_scope":"run","message":"Commands cannot target historical runs."}"#,
25375 ),
25376 "/api/workflows/wf-failed" | "/api/workflows/wf-failed/runs/run-failed" => (
25377 "200 OK",
25378 r#"{"workflow_id":"wf-failed","run_id":"run-failed","status":"failed","closed_reason":"failed","error":"payment failed","failure":{"message":"payment failed","failure_category":"application","exception_type":"PaymentError","exception_class":"billing::PaymentError","non_retryable":true,"exception":{"type":"PaymentError","class":"billing::PaymentError","message":"payment failed"},"failures":[{"id":"failure-17","failure_category":"application"}]}}"#,
25379 ),
25380 "/api/workflows/wf-cancelled" => (
25381 "200 OK",
25382 r#"{"workflow_id":"wf-cancelled","run_id":"run-cancelled","status":"cancelled","closed_reason":"cancelled","reason":"cleanup requested"}"#,
25383 ),
25384 "/api/workflows/wf-terminated" => (
25385 "200 OK",
25386 r#"{"workflow_id":"wf-terminated","run_id":"run-terminated","status":"terminated","closed_reason":"terminated","reason":"forced stop"}"#,
25387 ),
25388 "/api/workflows/wf-timed-out" => (
25389 "200 OK",
25390 r#"{"workflow_id":"wf-timed-out","run_id":"run-timed-out","status":"failed","closed_reason":"timed_out","reason":"run_timeout"}"#,
25391 ),
25392 "/api/workflows/wf-waiting" | "/api/workflows/wf-waiting/runs/run-waiting" => (
25393 "200 OK",
25394 r#"{"workflow_id":"wf-waiting","run_id":"run-waiting","status":"waiting"}"#,
25395 ),
25396 "/api/workflows/wf-selected" => (
25397 "200 OK",
25398 r#"{"workflow_id":"wf-selected","run_id":"run-current","status":"completed","output":"current run output"}"#,
25399 ),
25400 "/api/workflows/wf-selected/runs/run-selected" => (
25401 "200 OK",
25402 r#"{"workflow_id":"wf-selected","run_id":"run-selected","status":"cancelled","closed_reason":"cancelled","reason":"selected run cancelled"}"#,
25403 ),
25404 _ => ("404 Not Found", r#"{"message":"not found"}"#),
25405 };
25406 write_mock_response(stream, status, body);
25407 }
25408
25409 fn mock_request_is_complete(request: &[u8]) -> bool {
25410 let Some(header_end) = request
25411 .windows(4)
25412 .position(|window| window == b"\r\n\r\n")
25413 .map(|position| position + 4)
25414 else {
25415 return false;
25416 };
25417 let headers = String::from_utf8_lossy(&request[..header_end]);
25418 let content_length = headers.lines().find_map(|line| {
25419 let (name, value) = line.split_once(':')?;
25420 name.eq_ignore_ascii_case("content-length")
25421 .then(|| value.trim().parse::<usize>().ok())
25422 .flatten()
25423 });
25424
25425 request.len() >= header_end + content_length.unwrap_or(0)
25426 }
25427
25428 fn write_mock_response(stream: &mut TcpStream, status: &str, body: &str) {
25429 let response = format!(
25430 "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
25431 body.len()
25432 );
25433
25434 let _ = stream.write_all(response.as_bytes());
25435 let _ = stream.flush();
25436 }
25437}