1#![doc = include_str!("../README.md")]
2
3use std::{
4 any::{type_name, Any, TypeId},
5 collections::{BTreeMap, HashMap},
6 future::Future,
7 io::{self, Read},
8 pin::Pin,
9 sync::{
10 atomic::{AtomicBool, Ordering},
11 Arc, Mutex, OnceLock,
12 },
13 task::{Context as TaskContext, Poll},
14 time::{Duration, Instant, SystemTime, UNIX_EPOCH},
15};
16
17use apache_avro::{from_avro_datum, to_avro_datum, types::Value as AvroDatum, Schema};
18use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
19use chrono::DateTime;
20use futures_util::{future::OptionFuture, task::noop_waker_ref};
21use serde::{
22 de::DeserializeOwned,
23 ser::{SerializeMap, SerializeSeq},
24 Deserialize, Deserializer, Serialize, Serializer,
25};
26pub use serde_json::{json, Value};
27use sha2::{Digest, Sha256};
28use thiserror::Error;
29pub use uuid::Uuid;
30
31pub const WORKER_PROTOCOL_VERSION: &str = "1.19";
32pub const PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION: &str = "1.18";
34pub const CONTROL_PLANE_VERSION: &str = "2";
35pub const DEFAULT_CODEC: &str = "avro";
36pub const SDK_VERSION: &str = concat!("durable-workflow-rust/", env!("CARGO_PKG_VERSION"));
37pub const CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY: &str =
39 "condition_wait_occurrence_identity";
40pub const MEMO_UPSERTS_CAPABILITY: &str = "memo_upserts";
42pub const QUERY_TASKS_CAPABILITY: &str = "query_tasks";
44pub const TYPED_SEARCH_ATTRIBUTES_CAPABILITY: &str = "typed_search_attributes";
46pub const WORKFLOW_UPDATES_CAPABILITY: &str = "workflow_updates";
48pub const MESSAGE_STREAMS_CAPABILITY: &str = "message_streams";
50pub const DURABLE_SELECTION_CAPABILITY: &str = "durable_selection";
52pub const MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.15";
53pub const MESSAGE_STREAM_SIGNAL: &str = "__durable_workflow_message_stream";
54pub const MESSAGE_STREAM_SCHEMA: &str = "durable-workflow.v2.message-stream.message";
55pub const MESSAGE_STREAM_CURSOR_SCHEMA: &str = "durable-workflow.v2.message-stream.cursor";
56pub const MESSAGE_STREAM_MAX_BATCH: usize = 100;
57pub const QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
59pub const SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
61pub const MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.14";
63pub const TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.16";
65pub const CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.9";
67pub const CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.17";
69pub const DURABLE_SELECTION_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.19";
71
72pub fn worker_protocol_supports_message_streams(version: &str) -> bool {
73 let Some((major, minor)) = version.split_once('.') else {
74 return false;
75 };
76 major == "1" && minor.parse::<u64>().is_ok_and(|minor| minor >= 15)
77}
78
79fn validate_user_signal_name(signal_name: &str) -> Result<()> {
80 if signal_name == MESSAGE_STREAM_SIGNAL {
81 return Err(Error::Codec(format!(
82 "signal name {MESSAGE_STREAM_SIGNAL:?} is reserved by the workflow runtime"
83 )));
84 }
85 Ok(())
86}
87
88const MAX_LONG_POLL_TIMEOUT_SECONDS: u64 = 60;
89const WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE: &str =
90 "Workflow task waiting for scheduled history.";
91const WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE: &str = "WorkflowTaskWaitingForHistory";
92const MISSING_TASK_PAYLOAD_CODEC: &str = "\0missing-task-payload-codec";
93const NULL_TASK_PAYLOAD_CODEC: &str = "\0null-task-payload-codec";
94const NON_STRING_TASK_PAYLOAD_CODEC: &str = "\0non-string-task-payload-codec";
95const MAX_MEMO_ENTRIES: usize = 100;
96const MAX_MEMO_VALUE_SIZE_BYTES: usize = 10_240;
97const MAX_MEMO_TOTAL_SIZE_BYTES: usize = 65_536;
98
99const QUERY_TASK_FINAL_REJECTION_REASONS: &[&str] = &[
100 "lease_expired",
101 "query_task_not_found",
102 "query_task_not_leased",
103 "query_task_timed_out",
104];
105
106pub fn portable_worker_affinity_capability_manifest() -> Value {
108 json!({
109 "local_activities": {
110 "supported": false,
111 "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
112 "reason": "rust_worker_does_not_execute_record_local_activity",
113 },
114 "worker_sessions": {
115 "supported": false,
116 "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
117 "reason": "rust_worker_has_no_typed_session_lifecycle",
118 },
119 "sticky_execution": {
120 "supported": false,
121 "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
122 "reason": "rust_worker_uses_complete_durable_history_replay",
123 },
124 })
125}
126
127pub const AVRO_VALUE_SCHEMA_JSON: &str =
129 include_str!("../schema/durable_workflow.protocol.Value.v1.avsc");
130pub const AVRO_VALUE_SCHEMA_FINGERPRINT_HEX: &str = "e2a33dff55802237";
131pub const AVRO_VALUE_SCHEMA_FINGERPRINT: [u8; 8] = [0xe2, 0xa3, 0x3d, 0xff, 0x55, 0x80, 0x22, 0x37];
132const AVRO_SINGLE_OBJECT_MAGIC: [u8; 2] = [0xc3, 0x01];
133
134static AVRO_VALUE_SCHEMA: OnceLock<std::result::Result<Schema, String>> = OnceLock::new();
135static AVRO_VALUE_ORDERED_MAP_ENCODING_SCHEMA: OnceLock<std::result::Result<Schema, String>> =
136 OnceLock::new();
137
138#[derive(Clone, Copy)]
139enum RequestProtocol {
140 ControlPlane,
141 Worker(&'static str),
142}
143
144pub type Result<T> = std::result::Result<T, Error>;
145
146#[derive(Debug, Error)]
147pub enum Error {
148 #[error("transport error: {0}")]
149 Transport(#[from] reqwest::Error),
150 #[error(
151 "invalid Durable Workflow base URL: omit the SDK-owned /api suffix and pass the Server or Cloud runtime base URL; the SDK appends /api automatically"
152 )]
153 InvalidBaseUrl,
154 #[error("json error: {0}")]
155 Json(#[from] serde_json::Error),
156 #[error("http {status}: {body}")]
157 Http {
158 status: reqwest::StatusCode,
159 body: String,
160 },
161 #[error("codec error: {0}")]
162 Codec(String),
163 #[error(transparent)]
164 QueryFailed(QueryFailure),
165 #[error(transparent)]
166 Protocol(ProtocolFailure),
167 #[error(transparent)]
168 NonDeterministicReplay(ReplayFailure),
169 #[error(transparent)]
170 ChildWorkflowFailed(ChildWorkflowFailure),
171 #[error(transparent)]
172 ActivityFailed(ActivityFailure),
173 #[error(transparent)]
174 ParallelFailed(ParallelFailure),
175 #[error(transparent)]
176 SagaCompensationFailed(SagaCompensationFailure),
177 #[error(transparent)]
178 InvalidParallelGroup(ParallelGroupError),
179 #[error(transparent)]
180 DurableOperationCancelled(DurableOperationCancelled),
181 #[error(transparent)]
182 WorkflowCancellationRequested(WorkflowCancellationRequested),
183 #[error(transparent)]
184 WorkflowCommandRejected(WorkflowCommandRejection),
185 #[error(transparent)]
186 WorkflowFailed(WorkflowTerminalOutcome),
187 #[error(transparent)]
188 WorkflowCancelled(WorkflowTerminalOutcome),
189 #[error(transparent)]
190 WorkflowTerminated(WorkflowTerminalOutcome),
191 #[error(transparent)]
192 WorkflowTimedOut(WorkflowTerminalOutcome),
193 #[error(transparent)]
194 ActivityTaskRejected(ActivityTaskRejection),
195 #[error("workflow handler {0:?} is not registered")]
196 WorkflowNotRegistered(String),
197 #[error("activity handler {0:?} is not registered")]
198 ActivityNotRegistered(String),
199 #[error(
200 "{handler_kind} handler {handler_name:?} {value_kind} type {rust_type} is incompatible with the fixed Avro Value codec: {message}"
201 )]
202 HandlerType {
203 handler_kind: HandlerKind,
204 handler_name: String,
205 value_kind: HandlerValueKind,
206 rust_type: &'static str,
207 message: String,
208 },
209 #[error("workflow future yielded without emitting a durable command")]
210 WorkflowYieldedWithoutCommand,
211 #[error(
212 "workflow_stream_command_identity_missing: workflow stream authoring requires a non-empty server-provided workflow_command_id"
213 )]
214 MissingWorkflowCommandIdentity,
215 #[error("workflow state lock is poisoned")]
216 WorkflowStatePoisoned,
217 #[error("timer duration is too large for the worker protocol")]
218 TimerDurationOverflow,
219 #[error(transparent)]
220 InvalidConditionWaitOptions(#[from] ConditionWaitOptionsError),
221 #[error(transparent)]
222 InvalidSearchAttributeUpdate(#[from] SearchAttributeUpdateError),
223 #[error("operation timed out")]
224 Timeout,
225 #[error(
226 "missing {role}-plane credentials: configure ClientBuilder::{role}_token or ClientBuilder::token; a {opposite_role}-plane token cannot authorize this request"
227 )]
228 MissingRoleCredentials {
229 role: &'static str,
230 opposite_role: &'static str,
231 },
232 #[error("worker loop error: {0}")]
233 WorkerLoop(String),
234 #[error(
235 "workflow command contract for {workflow_type:?} declares update validators, but this Rust SDK cannot execute synchronous pre-accept update validation"
236 )]
237 UnsupportedUpdateValidators { workflow_type: String },
238 #[error("{primary}; worker deregistration also failed: {deregistration}")]
239 WorkerShutdown {
240 primary: Box<Error>,
241 deregistration: Box<Error>,
242 },
243 #[error("invalid child workflow options: {0}")]
244 InvalidChildWorkflowOptions(String),
245 #[error("invalid workflow memo update: {0}")]
246 InvalidMemoUpdate(String),
247 #[error(
248 "workflow_memo_updates_unavailable: the connected runtime did not advertise workflow memo update support"
249 )]
250 WorkflowMemoUpdatesUnavailable,
251 #[error(transparent)]
252 InvalidActivityOptions(ActivityOptionsError),
253 #[error(transparent)]
254 InvalidContinueAsNewOptions(#[from] ContinueAsNewOptionsError),
255 #[doc(hidden)]
256 #[error("workflow requested continue as new")]
257 ContinueAsNew(ContinueAsNewRequest),
258}
259
260#[derive(Clone, Debug, Error, PartialEq, Eq)]
262pub enum ConditionWaitOptionsError {
263 #[error("condition_key must be non-empty")]
264 EmptyKey,
265 #[error("condition_definition_fingerprint must be non-empty")]
266 EmptyPredicateIdentity,
267 #[error("condition timeout is too large for the worker protocol")]
268 TimeoutOverflow,
269}
270
271#[derive(Clone, Debug, PartialEq, Eq)]
278pub struct ConditionWaitOptions {
279 condition_key: String,
280 predicate_identity: String,
281 timeout: Option<Duration>,
282}
283
284impl ConditionWaitOptions {
285 pub fn new(condition_key: impl Into<String>, predicate_identity: impl Into<String>) -> Self {
286 Self {
287 condition_key: condition_key.into(),
288 predicate_identity: predicate_identity.into(),
289 timeout: None,
290 }
291 }
292
293 pub fn timeout(mut self, timeout: Duration) -> Self {
294 self.timeout = Some(timeout);
295 self
296 }
297
298 fn validate(
299 &self,
300 ) -> std::result::Result<ValidatedConditionWaitOptions, ConditionWaitOptionsError> {
301 let condition_key = self.condition_key.trim();
302 if condition_key.is_empty() {
303 return Err(ConditionWaitOptionsError::EmptyKey);
304 }
305 let predicate_identity = self.predicate_identity.trim();
306 if predicate_identity.is_empty() {
307 return Err(ConditionWaitOptionsError::EmptyPredicateIdentity);
308 }
309 let timeout_seconds = self
310 .timeout
311 .map(|timeout| {
312 timeout
313 .as_secs()
314 .checked_add(u64::from(timeout.subsec_nanos() > 0))
315 .ok_or(ConditionWaitOptionsError::TimeoutOverflow)
316 })
317 .transpose()?;
318
319 Ok(ValidatedConditionWaitOptions {
320 condition_key: condition_key.to_string(),
321 predicate_identity: predicate_identity.to_string(),
322 timeout_seconds,
323 })
324 }
325}
326
327#[derive(Clone, Debug, PartialEq, Eq)]
328struct ValidatedConditionWaitOptions {
329 condition_key: String,
330 predicate_identity: String,
331 timeout_seconds: Option<u64>,
332}
333
334const CONDITION_WAIT_OCCURRENCE_PREFIX: &str = "rust:condition-wait:";
335
336#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
338#[serde(rename_all = "snake_case")]
339pub enum ConditionWaitResult {
340 Satisfied,
341 TimedOut,
342}
343
344impl ConditionWaitResult {
345 pub fn is_satisfied(self) -> bool {
346 self == Self::Satisfied
347 }
348
349 pub fn is_timed_out(self) -> bool {
350 self == Self::TimedOut
351 }
352}
353
354#[doc(hidden)]
356pub fn __condition_definition_fingerprint(source: &str) -> String {
357 let mut digest = Sha256::new();
358 digest.update(b"durable-workflow-rust.wait-condition.v1\0");
359 digest.update(source.as_bytes());
360 format!("sha256:{:x}", digest.finalize())
361}
362
363#[macro_export]
369macro_rules! wait_condition {
370 ($ctx:expr, $key:expr, timeout: $timeout:expr, $predicate:expr $(,)?) => {{
371 $ctx.wait_condition(
372 $crate::ConditionWaitOptions::new(
373 $key,
374 $crate::__condition_definition_fingerprint(concat!(
375 module_path!(),
376 "\0",
377 stringify!($predicate)
378 )),
379 )
380 .timeout($timeout),
381 $predicate,
382 )
383 }};
384 ($ctx:expr, $key:expr, $predicate:expr $(,)?) => {{
385 $ctx.wait_condition(
386 $crate::ConditionWaitOptions::new(
387 $key,
388 $crate::__condition_definition_fingerprint(concat!(
389 module_path!(),
390 "\0",
391 stringify!($predicate)
392 )),
393 ),
394 $predicate,
395 )
396 }};
397}
398
399const MAX_SEARCH_ATTRIBUTES_PER_UPDATE: usize = 100;
400const MAX_SEARCH_ATTRIBUTE_KEY_LENGTH: usize = 64;
401const MAX_SEARCH_ATTRIBUTE_STRING_LENGTH: usize = 255;
402const MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH: usize = 255;
403const MAX_SEARCH_ATTRIBUTE_UPDATE_BYTES: usize = 65_536;
404
405#[derive(Clone, Debug, Error, PartialEq, Eq)]
407pub enum SearchAttributeUpdateError {
408 #[error("search-attribute update requires at least one attribute")]
409 Empty,
410 #[error("search attribute key {0:?} must be 1-64 URL-safe ASCII characters")]
411 InvalidKey(String),
412 #[error("search-attribute update exceeds the limit of 100 attributes")]
413 TooManyAttributes,
414 #[error("search attribute {key:?} {kind} value exceeds {limit} bytes")]
415 ValueTooLong {
416 key: String,
417 kind: &'static str,
418 limit: usize,
419 },
420 #[error(
421 "search attribute {0:?} must not contain an empty string value; use delete() to remove it"
422 )]
423 EmptyString(String),
424 #[error("search attribute {0:?} has a non-finite float value")]
425 NonFiniteFloat(String),
426 #[error("search attribute {0:?} must use an RFC 3339 datetime with an explicit timezone")]
427 InvalidDateTime(String),
428 #[error("search-attribute update exceeds the 65536-byte protocol limit")]
429 PayloadTooLarge,
430}
431
432#[derive(Clone, Debug, PartialEq)]
434pub enum SearchAttributeValue {
435 String(String),
436 Keyword(String),
437 KeywordList(Vec<String>),
438 Int(i64),
439 Float(f64),
440 Bool(bool),
441 DateTime(String),
442 Delete,
443}
444
445impl SearchAttributeValue {
446 fn type_name(&self) -> Option<&'static str> {
447 match self {
448 Self::String(_) => Some("string"),
449 Self::Keyword(_) => Some("keyword"),
450 Self::KeywordList(_) => Some("keyword_list"),
451 Self::Int(_) => Some("int"),
452 Self::Float(_) => Some("float"),
453 Self::Bool(_) => Some("bool"),
454 Self::DateTime(_) => Some("datetime"),
455 Self::Delete => None,
456 }
457 }
458
459 fn normalized(self, key: &str) -> std::result::Result<Self, SearchAttributeUpdateError> {
460 let normalize_string = |value: String, kind: &'static str, limit: usize| {
461 let value = value.trim().to_string();
462 if value.is_empty() {
463 return Err(SearchAttributeUpdateError::EmptyString(key.to_string()));
464 }
465 if value.len() > limit {
466 return Err(SearchAttributeUpdateError::ValueTooLong {
467 key: key.to_string(),
468 kind,
469 limit,
470 });
471 }
472 Ok(value)
473 };
474
475 match self {
476 Self::String(value) => Ok(Self::String(normalize_string(
477 value,
478 "string",
479 MAX_SEARCH_ATTRIBUTE_STRING_LENGTH,
480 )?)),
481 Self::Keyword(value) => Ok(Self::Keyword(normalize_string(
482 value,
483 "keyword",
484 MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH,
485 )?)),
486 Self::KeywordList(values) => {
487 let values = values
488 .into_iter()
489 .map(|value| {
490 let value = value.trim().to_string();
491 if value.len() > MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH {
492 return Err(SearchAttributeUpdateError::ValueTooLong {
493 key: key.to_string(),
494 kind: "keyword-list entry",
495 limit: MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH,
496 });
497 }
498 Ok(value)
499 })
500 .collect::<std::result::Result<Vec<_>, _>>()?;
501 Ok(Self::KeywordList(values))
502 }
503 Self::Float(value) if !value.is_finite() => {
504 Err(SearchAttributeUpdateError::NonFiniteFloat(key.to_string()))
505 }
506 Self::DateTime(value) => {
507 let value =
508 normalize_string(value, "datetime", MAX_SEARCH_ATTRIBUTE_STRING_LENGTH)?;
509 if DateTime::parse_from_rfc3339(&value).is_err() {
510 return Err(SearchAttributeUpdateError::InvalidDateTime(key.to_string()));
511 }
512 Ok(Self::DateTime(value))
513 }
514 value => Ok(value),
515 }
516 }
517
518 fn into_json(self) -> Value {
519 match self {
520 Self::String(value) | Self::Keyword(value) | Self::DateTime(value) => {
521 Value::String(value)
522 }
523 Self::KeywordList(values) => {
524 Value::Array(values.into_iter().map(Value::String).collect())
525 }
526 Self::Int(value) => json!(value),
527 Self::Float(value) => json!(value),
528 Self::Bool(value) => json!(value),
529 Self::Delete => Value::Null,
530 }
531 }
532}
533
534#[derive(Clone, Debug, Default, PartialEq)]
536pub struct SearchAttributeUpdate {
537 attributes: BTreeMap<String, SearchAttributeValue>,
538}
539
540impl SearchAttributeUpdate {
541 pub fn new() -> Self {
542 Self::default()
543 }
544
545 pub fn set(
546 mut self,
547 key: impl Into<String>,
548 value: SearchAttributeValue,
549 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
550 let key = key.into();
551 validate_search_attribute_key(&key)?;
552 if !self.attributes.contains_key(&key)
553 && self.attributes.len() >= MAX_SEARCH_ATTRIBUTES_PER_UPDATE
554 {
555 return Err(SearchAttributeUpdateError::TooManyAttributes);
556 }
557 self.attributes.insert(key.clone(), value.normalized(&key)?);
558 self.validate_size()?;
559 Ok(self)
560 }
561
562 pub fn string(
563 self,
564 key: impl Into<String>,
565 value: impl Into<String>,
566 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
567 self.set(key, SearchAttributeValue::String(value.into()))
568 }
569
570 pub fn keyword(
571 self,
572 key: impl Into<String>,
573 value: impl Into<String>,
574 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
575 self.set(key, SearchAttributeValue::Keyword(value.into()))
576 }
577
578 pub fn keyword_list<I, V>(
579 self,
580 key: impl Into<String>,
581 values: I,
582 ) -> std::result::Result<Self, SearchAttributeUpdateError>
583 where
584 I: IntoIterator<Item = V>,
585 V: Into<String>,
586 {
587 self.set(
588 key,
589 SearchAttributeValue::KeywordList(values.into_iter().map(Into::into).collect()),
590 )
591 }
592
593 pub fn int(
594 self,
595 key: impl Into<String>,
596 value: i64,
597 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
598 self.set(key, SearchAttributeValue::Int(value))
599 }
600
601 pub fn float(
602 self,
603 key: impl Into<String>,
604 value: f64,
605 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
606 self.set(key, SearchAttributeValue::Float(value))
607 }
608
609 pub fn bool(
610 self,
611 key: impl Into<String>,
612 value: bool,
613 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
614 self.set(key, SearchAttributeValue::Bool(value))
615 }
616
617 pub fn datetime(
618 self,
619 key: impl Into<String>,
620 value: impl Into<String>,
621 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
622 self.set(key, SearchAttributeValue::DateTime(value.into()))
623 }
624
625 pub fn delete(
626 self,
627 key: impl Into<String>,
628 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
629 self.set(key, SearchAttributeValue::Delete)
630 }
631
632 fn validate_size(&self) -> std::result::Result<(), SearchAttributeUpdateError> {
633 let (attributes, _) = self.clone().into_wire_parts();
634 if serde_json::to_vec(&attributes)
635 .map(|payload| payload.len() > MAX_SEARCH_ATTRIBUTE_UPDATE_BYTES)
636 .unwrap_or(true)
637 {
638 return Err(SearchAttributeUpdateError::PayloadTooLarge);
639 }
640 Ok(())
641 }
642
643 fn into_wire_parts(self) -> (Value, BTreeMap<String, String>) {
644 let mut attributes = serde_json::Map::new();
645 let mut attribute_types = BTreeMap::new();
646 for (key, value) in self.attributes {
647 if let Some(type_name) = value.type_name() {
648 attribute_types.insert(key.clone(), type_name.to_string());
649 }
650 attributes.insert(key, value.into_json());
651 }
652 (Value::Object(attributes), attribute_types)
653 }
654
655 fn validate(&self) -> std::result::Result<(), SearchAttributeUpdateError> {
656 if self.attributes.is_empty() {
657 return Err(SearchAttributeUpdateError::Empty);
658 }
659 self.validate_size()
660 }
661}
662
663fn validate_search_attribute_key(key: &str) -> std::result::Result<(), SearchAttributeUpdateError> {
664 let valid = !key.is_empty()
665 && key.len() <= MAX_SEARCH_ATTRIBUTE_KEY_LENGTH
666 && key
667 .bytes()
668 .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-' | b':'));
669 if valid {
670 Ok(())
671 } else {
672 Err(SearchAttributeUpdateError::InvalidKey(key.to_string()))
673 }
674}
675
676#[derive(Clone, Copy, Debug, PartialEq, Eq)]
678pub enum HandlerKind {
679 Workflow,
680 Activity,
681}
682
683impl std::fmt::Display for HandlerKind {
684 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
685 formatter.write_str(match self {
686 Self::Workflow => "workflow",
687 Self::Activity => "activity",
688 })
689 }
690}
691
692#[derive(Clone, Copy, Debug, PartialEq, Eq)]
694pub enum HandlerValueKind {
695 Input,
696 Result,
697}
698
699impl std::fmt::Display for HandlerValueKind {
700 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
701 formatter.write_str(match self {
702 Self::Input => "input",
703 Self::Result => "result",
704 })
705 }
706}
707
708#[derive(Clone, Copy, Debug, PartialEq, Eq)]
710pub enum WorkflowCommandKind {
711 Cancel,
712 Terminate,
713}
714
715impl WorkflowCommandKind {
716 fn as_str(self) -> &'static str {
717 match self {
718 Self::Cancel => "cancel",
719 Self::Terminate => "terminate",
720 }
721 }
722}
723
724#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
726pub struct WorkflowCommandOptions {
727 #[serde(skip_serializing_if = "Option::is_none")]
728 pub reason: Option<String>,
729 #[serde(skip_serializing_if = "Option::is_none")]
730 pub request_id: Option<String>,
731}
732
733#[derive(Clone, Debug, PartialEq, Eq)]
740pub struct WorkflowStartOptions {
741 pub execution_timeout_seconds: u64,
742 pub run_timeout_seconds: u64,
743}
744
745impl Default for WorkflowStartOptions {
746 fn default() -> Self {
747 Self {
748 execution_timeout_seconds: 3600,
749 run_timeout_seconds: 600,
750 }
751 }
752}
753
754impl WorkflowStartOptions {
755 pub fn new() -> Self {
756 Self::default()
757 }
758
759 pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
760 self.execution_timeout_seconds = seconds;
761 self
762 }
763
764 pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
765 self.run_timeout_seconds = seconds;
766 self
767 }
768
769 fn validate(&self) -> Result<()> {
770 if self.execution_timeout_seconds == 0 {
771 return Err(Error::Codec(
772 "execution_timeout_seconds must be at least 1".to_string(),
773 ));
774 }
775 if self.run_timeout_seconds == 0 {
776 return Err(Error::Codec(
777 "run_timeout_seconds must be at least 1".to_string(),
778 ));
779 }
780 if self.run_timeout_seconds > self.execution_timeout_seconds {
781 return Err(Error::Codec(
782 "run_timeout_seconds cannot exceed execution_timeout_seconds".to_string(),
783 ));
784 }
785
786 Ok(())
787 }
788}
789
790#[derive(Clone, Debug, Default, PartialEq, Eq)]
795pub struct ContinueAsNewOptions {
796 pub workflow_type: Option<String>,
797 pub task_queue: Option<String>,
798}
799
800impl ContinueAsNewOptions {
801 pub fn new() -> Self {
802 Self::default()
803 }
804
805 pub fn workflow_type(mut self, workflow_type: impl Into<String>) -> Self {
806 self.workflow_type = Some(workflow_type.into());
807 self
808 }
809
810 pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
811 self.task_queue = Some(task_queue.into());
812 self
813 }
814
815 fn validate(&self) -> std::result::Result<(), ContinueAsNewOptionsError> {
816 for (field, value) in [
817 ("workflow_type", self.workflow_type.as_deref()),
818 ("task_queue", self.task_queue.as_deref()),
819 ] {
820 if value.is_some_and(|value| value.trim().is_empty()) {
821 return Err(ContinueAsNewOptionsError {
822 field,
823 message: format!("{field} must not be empty"),
824 });
825 }
826 }
827 Ok(())
828 }
829}
830
831#[derive(Clone, Debug, Error, PartialEq, Eq)]
833#[error("invalid continue-as-new option {field}: {message}")]
834pub struct ContinueAsNewOptionsError {
835 pub field: &'static str,
836 pub message: String,
837}
838
839#[derive(Clone, Debug, Default, PartialEq, Eq)]
841pub struct WorkflowHistoryBudget {
842 pub event_count: u64,
843 pub size_bytes: Option<u64>,
844 pub continue_as_new_recommended: bool,
845 pub pressure: Option<String>,
846}
847
848#[doc(hidden)]
849#[derive(Clone, Debug)]
850pub struct ContinueAsNewRequest {
851 arguments: AvroValue,
852 options: ContinueAsNewOptions,
853}
854
855impl WorkflowCommandOptions {
856 pub fn new() -> Self {
857 Self::default()
858 }
859
860 pub fn reason(mut self, reason: impl Into<String>) -> Self {
861 self.reason = Some(reason.into());
862 self
863 }
864
865 pub fn request_id(mut self, request_id: impl Into<String>) -> Self {
866 self.request_id = Some(request_id.into());
867 self
868 }
869}
870
871#[derive(Clone, Debug, PartialEq)]
873pub struct WorkflowCommandResult {
874 pub command: WorkflowCommandKind,
875 pub workflow_id: String,
876 pub run_id: Option<String>,
877 pub outcome: Option<String>,
878 pub reason: Option<String>,
879 pub command_status: Option<String>,
880 pub raw: Value,
881}
882
883#[derive(Clone, Debug, Error)]
885#[error("workflow {command:?} rejected ({reason}, HTTP {status}): {message}")]
886pub struct WorkflowCommandRejection {
887 pub command: WorkflowCommandKind,
888 pub status: u16,
889 pub reason: String,
890 pub message: String,
891 pub workflow_id: String,
892 pub run_id: Option<String>,
893 pub target_scope: Option<String>,
894 pub body: Value,
895}
896
897#[derive(Clone, Copy, Debug, PartialEq, Eq)]
899pub enum WorkflowTerminalKind {
900 Failed,
901 Cancelled,
902 Terminated,
903 TimedOut,
904}
905
906#[derive(Clone, Debug, Error)]
912#[error("workflow {workflow_id} run {run_id:?} ended as {kind:?} ({reason})")]
913pub struct WorkflowTerminalOutcome {
914 pub kind: WorkflowTerminalKind,
915 pub workflow_id: String,
916 pub run_id: Option<String>,
917 pub reason: String,
918 pub failure_category: Option<String>,
919 pub failure_id: Option<String>,
920 pub exception_type: Option<String>,
921 pub exception_class: Option<String>,
922 pub non_retryable: Option<bool>,
923 pub message: Option<String>,
924 pub exception: Option<Value>,
925 pub raw: Value,
926}
927
928#[derive(Clone, Debug, Error)]
930#[error("activity task {operation} rejected ({reason}, HTTP {status})")]
931pub struct ActivityTaskRejection {
932 pub operation: String,
933 pub status: u16,
934 pub reason: String,
935 pub task_id: String,
936 pub activity_attempt_id: String,
937 pub cancel_requested: bool,
938 pub can_continue: Option<bool>,
939 pub run_closed_reason: Option<String>,
940 pub body: Value,
941}
942
943#[derive(Clone, Copy, Debug, PartialEq, Eq)]
945pub enum ActivityOptionsErrorKind {
946 EmptyTaskQueue,
947 EmptyRetryPolicy,
948 InvalidMaxAttempts,
949 BackoffWithoutRetryBudget,
950 TooManyBackoffIntervals,
951 InvalidBackoffCoefficient,
952 BackoffGenerationTooLarge,
953 BackoffOverflow,
954 EmptyNonRetryableErrorType,
955 TimeoutNotPositive,
956 TimeoutOverflow,
957 TimeoutOrder,
958}
959
960#[derive(Clone, Debug, Error, PartialEq, Eq)]
962#[error("invalid activity options ({kind:?}, {field:?}): {message}")]
963pub struct ActivityOptionsError {
964 pub kind: ActivityOptionsErrorKind,
965 pub field: Option<&'static str>,
966 pub message: String,
967}
968
969impl ActivityOptionsError {
970 fn new(
971 kind: ActivityOptionsErrorKind,
972 field: Option<&'static str>,
973 message: impl Into<String>,
974 ) -> Self {
975 Self {
976 kind,
977 field,
978 message: message.into(),
979 }
980 }
981}
982
983#[derive(Clone, Copy, Debug, PartialEq, Eq)]
985pub enum ActivityFailureKind {
986 Failed,
987 Cancelled,
988 TimedOut,
989}
990
991#[derive(Clone, Debug, Error)]
996#[error("activity failed ({reason}): {message}")]
997pub struct ActivityFailure {
998 pub kind: ActivityFailureKind,
999 pub reason: String,
1000 pub message: String,
1001 pub activity_execution_id: Option<String>,
1002 pub activity_attempt_id: Option<String>,
1003 pub activity_type: Option<String>,
1004 pub activity_class: Option<String>,
1005 pub attempt_number: Option<u64>,
1006 pub failure_id: Option<String>,
1007 pub failure_category: Option<String>,
1008 pub timeout_kind: Option<String>,
1009 pub non_retryable: bool,
1010 pub exception_type: Option<String>,
1011 pub exception_class: Option<String>,
1012 pub code: Option<Value>,
1013 pub exception: Option<Value>,
1014}
1015
1016#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1018pub enum ChildWorkflowFailureKind {
1019 Failed,
1020 Cancelled,
1021 Terminated,
1022}
1023
1024#[derive(Clone, Debug, Error)]
1030#[error("child workflow failed ({reason}): {message}")]
1031pub struct ChildWorkflowFailure {
1032 pub kind: ChildWorkflowFailureKind,
1033 pub reason: String,
1034 pub message: String,
1035 pub parent_workflow_id: Option<String>,
1036 pub parent_workflow_run_id: Option<String>,
1037 pub child_workflow_id: Option<String>,
1038 pub child_workflow_run_id: Option<String>,
1039 pub child_workflow_type: Option<String>,
1040 pub failure_id: Option<String>,
1041 pub failure_category: Option<String>,
1042 pub exception_type: Option<String>,
1043 pub exception_class: Option<String>,
1044 pub non_retryable: bool,
1045 pub code: Option<Value>,
1046 pub exception: Option<Value>,
1047}
1048
1049#[derive(Clone, Debug, PartialEq, Eq)]
1051pub struct WorkflowIdentity {
1052 pub workflow_id: Option<String>,
1053 pub run_id: Option<String>,
1054}
1055
1056#[derive(Clone, Debug, PartialEq)]
1058pub struct ChildWorkflowResult {
1059 pub parent: WorkflowIdentity,
1060 pub child: WorkflowIdentity,
1061 pub child_workflow_type: Option<String>,
1062 pub result: Value,
1063}
1064
1065#[derive(Clone, Debug, PartialEq)]
1067pub struct ChildWorkflowAvroResult {
1068 pub parent: WorkflowIdentity,
1069 pub child: WorkflowIdentity,
1070 pub child_workflow_type: Option<String>,
1071 pub result: AvroValue,
1072}
1073
1074#[derive(Clone, Debug, Deserialize, Hash, PartialEq, Eq, Serialize)]
1076#[serde(untagged)]
1077pub enum SelectionKey {
1078 Index(usize),
1079 Name(String),
1080}
1081
1082impl From<usize> for SelectionKey {
1083 fn from(value: usize) -> Self {
1084 Self::Index(value)
1085 }
1086}
1087
1088impl From<String> for SelectionKey {
1089 fn from(value: String) -> Self {
1090 Self::Name(value)
1091 }
1092}
1093
1094impl From<&str> for SelectionKey {
1095 fn from(value: &str) -> Self {
1096 Self::Name(value.to_string())
1097 }
1098}
1099
1100#[derive(Clone, Debug, Error, PartialEq, Eq)]
1102#[error("selected {operation_kind} operation {operation_identity} was explicitly cancelled")]
1103pub struct DurableOperationCancelled {
1104 pub selection_group_id: String,
1105 pub member_key: SelectionKey,
1106 pub member_index: usize,
1107 pub operation_kind: String,
1108 pub operation_identity: String,
1109}
1110
1111#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
1117pub struct ParallelGroupMetadata {
1118 pub parallel_group_id: String,
1119 pub parallel_group_kind: String,
1120 pub parallel_group_base_sequence: u64,
1121 pub parallel_group_size: usize,
1122 pub parallel_group_index: usize,
1123 #[serde(default, skip_serializing_if = "Option::is_none")]
1124 pub parallel_group_mode: Option<String>,
1125 #[serde(default, skip_serializing_if = "Option::is_none")]
1126 pub selection_member_key: Option<SelectionKey>,
1127 #[serde(default, skip_serializing_if = "Option::is_none")]
1128 pub selection_member_index: Option<usize>,
1129 #[serde(default, skip_serializing_if = "Option::is_none")]
1130 pub selection_member_base_sequence: Option<u64>,
1131 #[serde(default, skip_serializing_if = "Option::is_none")]
1132 pub selection_member_size: Option<usize>,
1133 #[serde(default, skip_serializing_if = "Option::is_none")]
1134 pub selection_member_kind: Option<String>,
1135}
1136
1137#[derive(Clone, Debug, PartialEq)]
1139pub enum ParallelResult {
1140 Activity(Value),
1141 ChildWorkflow(ChildWorkflowResult),
1142 Timer,
1143 Signal(Vec<Value>),
1144 Condition(ConditionWaitResult),
1145 Group(Vec<ParallelResult>),
1146}
1147
1148#[derive(Clone, Debug, PartialEq)]
1150pub enum ParallelAvroResult {
1151 Activity(AvroValue),
1152 ChildWorkflow(ChildWorkflowAvroResult),
1153 Timer,
1154 Signal(Vec<AvroValue>),
1155 Condition(ConditionWaitResult),
1156 Group(Vec<ParallelAvroResult>),
1157}
1158
1159impl ParallelAvroResult {
1160 fn into_json_result(self) -> Result<ParallelResult> {
1161 match self {
1162 Self::Activity(value) => Ok(ParallelResult::Activity(value.into_json()?)),
1163 Self::ChildWorkflow(result) => Ok(ParallelResult::ChildWorkflow(ChildWorkflowResult {
1164 parent: result.parent,
1165 child: result.child,
1166 child_workflow_type: result.child_workflow_type,
1167 result: result.result.into_json()?,
1168 })),
1169 Self::Timer => Ok(ParallelResult::Timer),
1170 Self::Signal(values) => Ok(ParallelResult::Signal(
1171 values
1172 .into_iter()
1173 .map(AvroValue::into_json)
1174 .collect::<Result<Vec<_>>>()?,
1175 )),
1176 Self::Condition(result) => Ok(ParallelResult::Condition(result)),
1177 Self::Group(results) => Ok(ParallelResult::Group(
1178 results
1179 .into_iter()
1180 .map(Self::into_json_result)
1181 .collect::<Result<Vec<_>>>()?,
1182 )),
1183 }
1184 }
1185}
1186
1187#[derive(Clone, Debug, PartialEq)]
1189pub struct ParallelCompletion {
1190 pub member_path: Vec<usize>,
1191 pub result: ParallelResult,
1192}
1193
1194#[derive(Debug, Error)]
1202#[error("parallel group {group_id} member {member_path:?} failed: {cause}")]
1203pub struct ParallelFailure {
1204 pub group_id: String,
1205 pub member_path: Vec<usize>,
1206 pub group_path: Vec<ParallelGroupMetadata>,
1207 pub completed: Vec<ParallelCompletion>,
1208 #[source]
1209 pub cause: Box<Error>,
1210}
1211
1212#[derive(Clone, Debug, Error, PartialEq, Eq)]
1214#[error("invalid deterministic parallel group ({reason}): {message}")]
1215pub struct ParallelGroupError {
1216 pub reason: &'static str,
1217 pub member_path: Vec<usize>,
1218 pub message: String,
1219}
1220
1221#[derive(Clone, Debug, Error, PartialEq, Eq)]
1223#[error("workflow cancellation was requested")]
1224pub struct WorkflowCancellationRequested;
1225
1226#[derive(Debug, Error)]
1228#[error(
1229 "saga forward execution failed; compensation activity {compensation_activity_type} (registration {compensation_registration_order}) also failed: {compensation_failure}"
1230)]
1231pub struct SagaCompensationFailure {
1232 pub initiating_failure: Box<Error>,
1233 pub compensation_failure: Box<Error>,
1234 pub compensation_activity_type: String,
1235 pub compensation_registration_order: usize,
1236}
1237
1238#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
1240pub enum ParentClosePolicy {
1241 #[default]
1242 Abandon,
1243 RequestCancel,
1244 Terminate,
1245}
1246
1247impl ParentClosePolicy {
1248 fn as_str(self) -> &'static str {
1249 match self {
1250 Self::Abandon => "abandon",
1251 Self::RequestCancel => "request_cancel",
1252 Self::Terminate => "terminate",
1253 }
1254 }
1255}
1256
1257#[derive(Clone, Debug, Default, PartialEq, Eq)]
1259pub struct ChildWorkflowRetryPolicy {
1260 pub max_attempts: Option<u32>,
1261 pub backoff_seconds: Vec<u64>,
1262 pub non_retryable_error_types: Vec<String>,
1263}
1264
1265#[derive(Clone, Debug, PartialEq, Eq)]
1269pub struct ChildWorkflowOptions {
1270 pub task_queue: String,
1271 pub parent_close_policy: ParentClosePolicy,
1272 pub retry_policy: Option<ChildWorkflowRetryPolicy>,
1273 pub execution_timeout_seconds: Option<u64>,
1274 pub run_timeout_seconds: Option<u64>,
1275}
1276
1277impl ChildWorkflowOptions {
1278 pub fn new(task_queue: impl Into<String>) -> Self {
1279 Self {
1280 task_queue: task_queue.into(),
1281 parent_close_policy: ParentClosePolicy::Abandon,
1282 retry_policy: None,
1283 execution_timeout_seconds: None,
1284 run_timeout_seconds: None,
1285 }
1286 }
1287
1288 pub fn parent_close_policy(mut self, policy: ParentClosePolicy) -> Self {
1289 self.parent_close_policy = policy;
1290 self
1291 }
1292
1293 pub fn retry_policy(mut self, policy: ChildWorkflowRetryPolicy) -> Self {
1294 self.retry_policy = Some(policy);
1295 self
1296 }
1297
1298 pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
1299 self.execution_timeout_seconds = Some(seconds);
1300 self
1301 }
1302
1303 pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
1304 self.run_timeout_seconds = Some(seconds);
1305 self
1306 }
1307}
1308
1309#[derive(Clone, Debug, PartialEq, Eq)]
1311pub enum ActivityBackoff {
1312 Explicit(Vec<Duration>),
1315 Exponential {
1317 initial_interval: Duration,
1318 coefficient: u32,
1319 maximum_interval: Option<Duration>,
1320 },
1321}
1322
1323#[derive(Clone, Debug, Default, PartialEq, Eq)]
1325pub struct ActivityRetryPolicy {
1326 pub max_attempts: Option<u32>,
1327 pub backoff: Option<ActivityBackoff>,
1328 pub non_retryable_error_types: Vec<String>,
1329}
1330
1331impl ActivityRetryPolicy {
1332 pub fn new(max_attempts: u32) -> Self {
1334 Self {
1335 max_attempts: Some(max_attempts),
1336 ..Self::default()
1337 }
1338 }
1339
1340 pub fn backoff_intervals(mut self, intervals: impl IntoIterator<Item = Duration>) -> Self {
1341 self.backoff = Some(ActivityBackoff::Explicit(intervals.into_iter().collect()));
1342 self
1343 }
1344
1345 pub fn exponential_backoff(
1346 mut self,
1347 initial_interval: Duration,
1348 coefficient: u32,
1349 maximum_interval: Option<Duration>,
1350 ) -> Self {
1351 self.backoff = Some(ActivityBackoff::Exponential {
1352 initial_interval,
1353 coefficient,
1354 maximum_interval,
1355 });
1356 self
1357 }
1358
1359 pub fn non_retryable_error_type(mut self, error_type: impl Into<String>) -> Self {
1360 self.non_retryable_error_types.push(error_type.into());
1361 self
1362 }
1363
1364 pub fn non_retryable_error_types(
1365 mut self,
1366 error_types: impl IntoIterator<Item = impl Into<String>>,
1367 ) -> Self {
1368 self.non_retryable_error_types
1369 .extend(error_types.into_iter().map(Into::into));
1370 self
1371 }
1372}
1373
1374#[derive(Clone, Debug, Default, PartialEq, Eq)]
1379pub struct ActivityOptions {
1380 pub task_queue: Option<String>,
1381 pub retry_policy: Option<ActivityRetryPolicy>,
1382 pub start_to_close_timeout: Option<Duration>,
1383 pub schedule_to_start_timeout: Option<Duration>,
1384 pub schedule_to_close_timeout: Option<Duration>,
1385 pub heartbeat_timeout: Option<Duration>,
1386}
1387
1388impl ActivityOptions {
1389 pub fn new() -> Self {
1390 Self::default()
1391 }
1392
1393 pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
1394 self.task_queue = Some(task_queue.into());
1395 self
1396 }
1397
1398 pub fn retry_policy(mut self, policy: ActivityRetryPolicy) -> Self {
1399 self.retry_policy = Some(policy);
1400 self
1401 }
1402
1403 pub fn start_to_close_timeout(mut self, timeout: Duration) -> Self {
1404 self.start_to_close_timeout = Some(timeout);
1405 self
1406 }
1407
1408 pub fn schedule_to_start_timeout(mut self, timeout: Duration) -> Self {
1409 self.schedule_to_start_timeout = Some(timeout);
1410 self
1411 }
1412
1413 pub fn schedule_to_close_timeout(mut self, timeout: Duration) -> Self {
1414 self.schedule_to_close_timeout = Some(timeout);
1415 self
1416 }
1417
1418 pub fn heartbeat_timeout(mut self, timeout: Duration) -> Self {
1419 self.heartbeat_timeout = Some(timeout);
1420 self
1421 }
1422
1423 fn validate(&self) -> std::result::Result<ValidatedActivityOptions, ActivityOptionsError> {
1424 if self
1425 .task_queue
1426 .as_deref()
1427 .is_some_and(|queue| queue.trim().is_empty())
1428 {
1429 return Err(ActivityOptionsError::new(
1430 ActivityOptionsErrorKind::EmptyTaskQueue,
1431 Some("task_queue"),
1432 "task_queue must not be empty",
1433 ));
1434 }
1435
1436 for (field, value) in [
1437 ("start_to_close_timeout", self.start_to_close_timeout),
1438 ("schedule_to_start_timeout", self.schedule_to_start_timeout),
1439 ("schedule_to_close_timeout", self.schedule_to_close_timeout),
1440 ("heartbeat_timeout", self.heartbeat_timeout),
1441 ] {
1442 if value.is_some_and(|value| value.is_zero()) {
1443 return Err(ActivityOptionsError::new(
1444 ActivityOptionsErrorKind::TimeoutNotPositive,
1445 Some(field),
1446 format!("{field} must be positive"),
1447 ));
1448 }
1449 }
1450
1451 validate_timeout_order(
1452 "heartbeat_timeout",
1453 self.heartbeat_timeout,
1454 "start_to_close_timeout",
1455 self.start_to_close_timeout,
1456 )?;
1457 validate_timeout_order(
1458 "start_to_close_timeout",
1459 self.start_to_close_timeout,
1460 "schedule_to_close_timeout",
1461 self.schedule_to_close_timeout,
1462 )?;
1463 validate_timeout_order(
1464 "schedule_to_start_timeout",
1465 self.schedule_to_start_timeout,
1466 "schedule_to_close_timeout",
1467 self.schedule_to_close_timeout,
1468 )?;
1469
1470 Ok(ValidatedActivityOptions {
1471 task_queue: self.task_queue.clone(),
1472 retry_policy: self
1473 .retry_policy
1474 .as_ref()
1475 .map(validate_activity_retry_policy)
1476 .transpose()?,
1477 start_to_close_timeout: timeout_seconds(
1478 "start_to_close_timeout",
1479 self.start_to_close_timeout,
1480 )?,
1481 schedule_to_start_timeout: timeout_seconds(
1482 "schedule_to_start_timeout",
1483 self.schedule_to_start_timeout,
1484 )?,
1485 schedule_to_close_timeout: timeout_seconds(
1486 "schedule_to_close_timeout",
1487 self.schedule_to_close_timeout,
1488 )?,
1489 heartbeat_timeout: timeout_seconds("heartbeat_timeout", self.heartbeat_timeout)?,
1490 })
1491 }
1492}
1493
1494pub enum ParallelOperation {
1500 Activity {
1501 activity_type: String,
1502 options: ActivityOptions,
1503 arguments: Result<AvroValue>,
1504 },
1505 ChildWorkflow {
1506 workflow_type: String,
1507 options: ChildWorkflowOptions,
1508 arguments: Result<AvroValue>,
1509 },
1510 Timer(Duration),
1511 Signal(String),
1512 Condition {
1513 options: ConditionWaitOptions,
1514 predicate: Box<dyn Fn() -> Result<bool> + Send + 'static>,
1515 },
1516 Group(Vec<ParallelOperation>),
1517}
1518
1519impl ParallelOperation {
1520 pub fn activity<T: Serialize>(activity_type: impl Into<String>, args: T) -> Self {
1521 Self::activity_with_options(activity_type, ActivityOptions::new(), args)
1522 }
1523
1524 pub fn activity_with_options<T: Serialize>(
1525 activity_type: impl Into<String>,
1526 options: ActivityOptions,
1527 args: T,
1528 ) -> Self {
1529 Self::Activity {
1530 activity_type: activity_type.into(),
1531 options,
1532 arguments: AvroValue::from_serialize(&args),
1533 }
1534 }
1535
1536 pub fn child_workflow<T: Serialize>(
1537 workflow_type: impl Into<String>,
1538 options: ChildWorkflowOptions,
1539 args: T,
1540 ) -> Self {
1541 Self::ChildWorkflow {
1542 workflow_type: workflow_type.into(),
1543 options,
1544 arguments: AvroValue::from_serialize(&args),
1545 }
1546 }
1547
1548 pub fn timer(duration: Duration) -> Self {
1549 Self::Timer(duration)
1550 }
1551
1552 pub fn signal(signal_name: impl Into<String>) -> Self {
1553 Self::Signal(signal_name.into())
1554 }
1555
1556 pub fn condition<F>(options: ConditionWaitOptions, predicate: F) -> Self
1557 where
1558 F: Fn() -> Result<bool> + Send + 'static,
1559 {
1560 Self::Condition {
1561 options,
1562 predicate: Box::new(predicate),
1563 }
1564 }
1565
1566 pub fn group(operations: Vec<ParallelOperation>) -> Self {
1567 Self::Group(operations)
1568 }
1569}
1570
1571#[derive(Clone, Debug)]
1572struct ValidatedActivityOptions {
1573 task_queue: Option<String>,
1574 retry_policy: Option<Value>,
1575 start_to_close_timeout: Option<u64>,
1576 schedule_to_start_timeout: Option<u64>,
1577 schedule_to_close_timeout: Option<u64>,
1578 heartbeat_timeout: Option<u64>,
1579}
1580
1581fn validate_timeout_order(
1582 smaller_name: &'static str,
1583 smaller: Option<Duration>,
1584 larger_name: &'static str,
1585 larger: Option<Duration>,
1586) -> std::result::Result<(), ActivityOptionsError> {
1587 if matches!((smaller, larger), (Some(smaller), Some(larger)) if smaller > larger) {
1588 return Err(ActivityOptionsError::new(
1589 ActivityOptionsErrorKind::TimeoutOrder,
1590 Some(smaller_name),
1591 format!("{smaller_name} must be <= {larger_name}"),
1592 ));
1593 }
1594 Ok(())
1595}
1596
1597fn timeout_seconds(
1598 field: &'static str,
1599 value: Option<Duration>,
1600) -> std::result::Result<Option<u64>, ActivityOptionsError> {
1601 value
1602 .map(|value| {
1603 activity_protocol_seconds(value).ok_or_else(|| {
1604 ActivityOptionsError::new(
1605 ActivityOptionsErrorKind::TimeoutOverflow,
1606 Some(field),
1607 format!("{field} is too large for the worker protocol"),
1608 )
1609 })
1610 })
1611 .transpose()
1612}
1613
1614fn duration_seconds_ceil(value: Duration) -> Option<u64> {
1615 value
1616 .as_secs()
1617 .checked_add(u64::from(value.subsec_nanos() > 0))
1618}
1619
1620fn activity_protocol_seconds(value: Duration) -> Option<u64> {
1621 duration_seconds_ceil(value).filter(|seconds| *seconds <= i64::MAX as u64)
1622}
1623
1624fn validate_activity_retry_policy(
1625 policy: &ActivityRetryPolicy,
1626) -> std::result::Result<Value, ActivityOptionsError> {
1627 if policy.max_attempts.is_none()
1628 && policy.backoff.is_none()
1629 && policy.non_retryable_error_types.is_empty()
1630 {
1631 return Err(ActivityOptionsError::new(
1632 ActivityOptionsErrorKind::EmptyRetryPolicy,
1633 Some("retry_policy"),
1634 "retry_policy must configure at least one field",
1635 ));
1636 }
1637 if policy.max_attempts == Some(0) {
1638 return Err(ActivityOptionsError::new(
1639 ActivityOptionsErrorKind::InvalidMaxAttempts,
1640 Some("retry_policy.max_attempts"),
1641 "max_attempts must be >= 1",
1642 ));
1643 }
1644 if policy
1645 .non_retryable_error_types
1646 .iter()
1647 .any(|error_type| error_type.trim().is_empty())
1648 {
1649 return Err(ActivityOptionsError::new(
1650 ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
1651 Some("retry_policy.non_retryable_error_types"),
1652 "non_retryable_error_types must not contain empty values",
1653 ));
1654 }
1655
1656 let backoff_seconds = match &policy.backoff {
1657 None => None,
1658 Some(backoff) => {
1659 let max_attempts = policy.max_attempts.ok_or_else(|| {
1660 ActivityOptionsError::new(
1661 ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
1662 Some("retry_policy.backoff"),
1663 "backoff requires max_attempts",
1664 )
1665 })?;
1666 let retry_count = max_attempts.saturating_sub(1) as usize;
1667 let intervals = match backoff {
1668 ActivityBackoff::Explicit(intervals) => {
1669 if intervals.len() > retry_count {
1670 return Err(ActivityOptionsError::new(
1671 ActivityOptionsErrorKind::TooManyBackoffIntervals,
1672 Some("retry_policy.backoff"),
1673 "backoff interval count must not exceed max_attempts - 1",
1674 ));
1675 }
1676 intervals.clone()
1677 }
1678 ActivityBackoff::Exponential {
1679 initial_interval,
1680 coefficient,
1681 maximum_interval,
1682 } => {
1683 if *coefficient < 1 {
1684 return Err(ActivityOptionsError::new(
1685 ActivityOptionsErrorKind::InvalidBackoffCoefficient,
1686 Some("retry_policy.backoff.coefficient"),
1687 "backoff coefficient must be >= 1",
1688 ));
1689 }
1690 if retry_count > 10_000 {
1691 return Err(ActivityOptionsError::new(
1692 ActivityOptionsErrorKind::BackoffGenerationTooLarge,
1693 Some("retry_policy.max_attempts"),
1694 "generated backoff supports at most 10000 retry intervals",
1695 ));
1696 }
1697 let mut current = *initial_interval;
1698 let mut intervals = Vec::with_capacity(retry_count);
1699 for _ in 0..retry_count {
1700 let interval = maximum_interval
1701 .map(|maximum| current.min(maximum))
1702 .unwrap_or(current);
1703 intervals.push(interval);
1704 if maximum_interval.is_some_and(|maximum| interval == maximum) {
1705 break;
1706 }
1707 current = current.checked_mul(*coefficient).ok_or_else(|| {
1708 ActivityOptionsError::new(
1709 ActivityOptionsErrorKind::BackoffOverflow,
1710 Some("retry_policy.backoff"),
1711 "generated backoff interval overflowed",
1712 )
1713 })?;
1714 }
1715 intervals
1716 }
1717 };
1718 Some(
1719 intervals
1720 .into_iter()
1721 .map(|interval| {
1722 activity_protocol_seconds(interval).ok_or_else(|| {
1723 ActivityOptionsError::new(
1724 ActivityOptionsErrorKind::BackoffOverflow,
1725 Some("retry_policy.backoff"),
1726 "backoff interval is too large for the worker protocol",
1727 )
1728 })
1729 })
1730 .collect::<std::result::Result<Vec<_>, _>>()?,
1731 )
1732 }
1733 };
1734
1735 let mut encoded = serde_json::Map::new();
1736 if let Some(max_attempts) = policy.max_attempts {
1737 encoded.insert("max_attempts".to_string(), json!(max_attempts));
1738 }
1739 if let Some(backoff_seconds) = backoff_seconds {
1740 encoded.insert("backoff_seconds".to_string(), json!(backoff_seconds));
1741 }
1742 if !policy.non_retryable_error_types.is_empty() {
1743 let mut canonical_error_types = Vec::new();
1744 for error_type in policy
1745 .non_retryable_error_types
1746 .iter()
1747 .map(|error_type| error_type.trim())
1748 {
1749 if !canonical_error_types.contains(&error_type) {
1750 canonical_error_types.push(error_type);
1751 }
1752 }
1753 encoded.insert(
1754 "non_retryable_error_types".to_string(),
1755 json!(canonical_error_types),
1756 );
1757 }
1758 Ok(Value::Object(encoded))
1759}
1760
1761#[derive(Clone, Debug, Error)]
1764#[error("non-deterministic workflow replay ({reason}) at sequence {sequence:?}: {message}")]
1765pub struct ReplayFailure {
1766 pub reason: String,
1767 pub sequence: Option<u64>,
1768 pub expected: Option<String>,
1769 pub actual: Option<String>,
1770 pub message: String,
1771}
1772
1773impl ReplayFailure {
1774 fn new(
1775 reason: impl Into<String>,
1776 sequence: Option<u64>,
1777 expected: Option<String>,
1778 actual: Option<String>,
1779 message: impl Into<String>,
1780 ) -> Self {
1781 Self {
1782 reason: reason.into(),
1783 sequence,
1784 expected,
1785 actual,
1786 message: message.into(),
1787 }
1788 }
1789}
1790
1791#[derive(Clone, Debug, Error)]
1793#[error("query failed ({reason}, HTTP {status}): {message}")]
1794pub struct QueryFailure {
1795 pub status: u16,
1796 pub reason: String,
1797 pub message: String,
1798 pub body: Value,
1799}
1800
1801#[derive(Clone, Debug, Error)]
1803#[error("protocol rejected ({reason}, HTTP {status}): {message}")]
1804pub struct ProtocolFailure {
1805 pub status: u16,
1806 pub reason: String,
1807 pub message: String,
1808 pub supported_version: Option<String>,
1809 pub requested_version: Option<String>,
1810 pub body: Value,
1811}
1812
1813#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
1814pub struct PayloadEnvelope {
1815 pub codec: String,
1816 pub blob: String,
1817}
1818
1819impl PayloadEnvelope {
1820 pub fn avro<T: Serialize>(value: &T) -> Result<Self> {
1821 encode_payload(value, DEFAULT_CODEC)
1822 }
1823
1824 pub fn avro_value(value: &AvroValue) -> Result<Self> {
1827 encode_avro_value(value)
1828 }
1829}
1830
1831#[derive(Clone, Debug)]
1833pub enum AvroValue {
1834 Null,
1835 Boolean(bool),
1836 Long(i64),
1837 Double(f64),
1838 Bytes(Vec<u8>),
1839 String(String),
1840 Array(Vec<AvroValue>),
1841 Map(BTreeMap<String, AvroValue>),
1842}
1843
1844impl PartialEq for AvroValue {
1845 fn eq(&self, other: &Self) -> bool {
1846 match (self, other) {
1847 (Self::Null, Self::Null) => true,
1848 (Self::Boolean(left), Self::Boolean(right)) => left == right,
1849 (Self::Long(left), Self::Long(right)) => left == right,
1850 (Self::Double(left), Self::Double(right)) => left.to_bits() == right.to_bits(),
1851 (Self::Bytes(left), Self::Bytes(right)) => left == right,
1852 (Self::String(left), Self::String(right)) => left == right,
1853 (Self::Array(left), Self::Array(right)) => left == right,
1854 (Self::Map(left), Self::Map(right)) => left == right,
1855 _ => false,
1856 }
1857 }
1858}
1859
1860impl AvroValue {
1861 fn from_serialize<T: Serialize>(value: &T) -> Result<Self> {
1862 Self::from_serde_value(
1863 serde_value::to_value(value).map_err(|error| {
1864 Error::Codec(format!("could not adapt value for Avro: {error}"))
1865 })?,
1866 )
1867 }
1868
1869 fn from_serde_value(value: serde_value::Value) -> Result<Self> {
1870 use serde_value::Value as SerdeValue;
1871
1872 match value {
1873 SerdeValue::Unit => Ok(Self::Null),
1874 SerdeValue::Bool(value) => Ok(Self::Boolean(value)),
1875 SerdeValue::I8(value) => Ok(Self::Long(i64::from(value))),
1876 SerdeValue::I16(value) => Ok(Self::Long(i64::from(value))),
1877 SerdeValue::I32(value) => Ok(Self::Long(i64::from(value))),
1878 SerdeValue::I64(value) => Ok(Self::Long(value)),
1879 SerdeValue::U8(value) => Ok(Self::Long(i64::from(value))),
1880 SerdeValue::U16(value) => Ok(Self::Long(i64::from(value))),
1881 SerdeValue::U32(value) => Ok(Self::Long(i64::from(value))),
1882 SerdeValue::U64(value) => i64::try_from(value).map(Self::Long).map_err(|_| {
1883 Error::Codec(
1884 "integer_overflow: Avro Value long must be within signed 64-bit range"
1885 .to_string(),
1886 )
1887 }),
1888 SerdeValue::F32(value) => Self::finite_double(f64::from(value)),
1889 SerdeValue::F64(value) => Self::finite_double(value),
1890 SerdeValue::Char(value) => Ok(Self::String(value.to_string())),
1891 SerdeValue::String(value) => Ok(Self::String(value)),
1892 SerdeValue::Bytes(value) => Ok(Self::Bytes(value)),
1893 SerdeValue::Option(None) => Ok(Self::Null),
1894 SerdeValue::Option(Some(value)) | SerdeValue::Newtype(value) => {
1895 Self::from_serde_value(*value)
1896 }
1897 SerdeValue::Seq(values) => values
1898 .into_iter()
1899 .map(Self::from_serde_value)
1900 .collect::<Result<Vec<_>>>()
1901 .map(Self::Array),
1902 SerdeValue::Map(values) => values
1903 .into_iter()
1904 .map(|(key, value)| {
1905 let SerdeValue::String(key) = key else {
1906 return Err(Error::Codec(
1907 "invalid_map_key: Avro Value map keys must be strings".to_string(),
1908 ));
1909 };
1910
1911 Ok((key, Self::from_serde_value(value)?))
1912 })
1913 .collect::<Result<BTreeMap<_, _>>>()
1914 .map(Self::Map),
1915 }
1916 }
1917
1918 fn finite_double(value: f64) -> Result<Self> {
1919 if !value.is_finite() {
1920 return Err(Error::Codec(
1921 "non_finite_float: Avro Value doubles must be finite".to_string(),
1922 ));
1923 }
1924
1925 Ok(Self::Double(value))
1926 }
1927
1928 fn into_json(self) -> Result<Value> {
1929 match self {
1930 Self::Null => Ok(Value::Null),
1931 Self::Boolean(value) => Ok(Value::Bool(value)),
1932 Self::Long(value) => Ok(Value::Number(value.into())),
1933 Self::Double(value) => serde_json::Number::from_f64(value)
1934 .map(Value::Number)
1935 .ok_or_else(|| {
1936 Error::Codec(
1937 "non_finite_float: decoded Avro Value double is not finite".to_string(),
1938 )
1939 }),
1940 Self::Bytes(value) => Ok(json!({
1941 "$type": "bytes",
1942 "base64": BASE64.encode(value),
1943 })),
1944 Self::String(value) => Ok(Value::String(value)),
1945 Self::Array(values) => values
1946 .into_iter()
1947 .map(Self::into_json)
1948 .collect::<Result<Vec<_>>>()
1949 .map(Value::Array),
1950 Self::Map(values) => values
1951 .into_iter()
1952 .map(|(key, value)| Ok((key, value.into_json()?)))
1953 .collect::<Result<serde_json::Map<_, _>>>()
1954 .map(Value::Object),
1955 }
1956 }
1957
1958 fn into_serde_value(self) -> serde_value::Value {
1959 use serde_value::Value as SerdeValue;
1960
1961 match self {
1962 Self::Null => SerdeValue::Unit,
1963 Self::Boolean(value) => SerdeValue::Bool(value),
1964 Self::Long(value) => SerdeValue::I64(value),
1965 Self::Double(value) => SerdeValue::F64(value),
1966 Self::Bytes(value) => SerdeValue::Bytes(value),
1967 Self::String(value) => SerdeValue::String(value),
1968 Self::Array(values) => {
1969 SerdeValue::Seq(values.into_iter().map(Self::into_serde_value).collect())
1970 }
1971 Self::Map(values) => SerdeValue::Map(
1972 values
1973 .into_iter()
1974 .map(|(key, value)| (SerdeValue::String(key), value.into_serde_value()))
1975 .collect(),
1976 ),
1977 }
1978 }
1979
1980 pub fn deserialize<T: DeserializeOwned>(self) -> Result<T> {
1981 self.into_serde_value().deserialize_into().map_err(|error| {
1982 Error::Codec(format!(
1983 "avro_value_type_mismatch: could not adapt decoded value: {error}"
1984 ))
1985 })
1986 }
1987}
1988
1989impl Serialize for AvroValue {
1990 fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
1991 where
1992 S: Serializer,
1993 {
1994 match self {
1995 Self::Null => serializer.serialize_unit(),
1996 Self::Boolean(value) => serializer.serialize_bool(*value),
1997 Self::Long(value) => serializer.serialize_i64(*value),
1998 Self::Double(value) => serializer.serialize_f64(*value),
1999 Self::Bytes(value) => serializer.serialize_bytes(value),
2000 Self::String(value) => serializer.serialize_str(value),
2001 Self::Array(values) => {
2002 let mut sequence = serializer.serialize_seq(Some(values.len()))?;
2003 for value in values {
2004 sequence.serialize_element(value)?;
2005 }
2006 sequence.end()
2007 }
2008 Self::Map(values) => {
2009 let mut map = serializer.serialize_map(Some(values.len()))?;
2010 for (key, value) in values {
2011 map.serialize_entry(key, value)?;
2012 }
2013 map.end()
2014 }
2015 }
2016 }
2017}
2018
2019pub fn encode_avro_value(value: &AvroValue) -> Result<PayloadEnvelope> {
2020 let datum = avro_value_to_datum(value)?;
2021 let datum = to_avro_datum(avro_value_ordered_map_encoding_schema()?, datum)
2022 .map_err(|err| Error::Codec(format!("avro_value_encode_failed: {err}")))?;
2023 let mut bytes = Vec::with_capacity(datum.len() + 10);
2024 bytes.extend_from_slice(&AVRO_SINGLE_OBJECT_MAGIC);
2025 bytes.extend_from_slice(&AVRO_VALUE_SCHEMA_FINGERPRINT);
2026 bytes.extend_from_slice(&datum);
2027 Ok(PayloadEnvelope {
2028 codec: DEFAULT_CODEC.to_string(),
2029 blob: BASE64.encode(bytes),
2030 })
2031}
2032
2033pub fn decode_avro_value(envelope: &PayloadEnvelope) -> Result<AvroValue> {
2034 if envelope.codec != DEFAULT_CODEC {
2035 return Err(unsupported_payload_codec(&envelope.codec));
2036 }
2037 decode_avro_value_blob(&envelope.blob)
2038}
2039
2040pub fn encode_payload<T: Serialize>(value: &T, codec: &str) -> Result<PayloadEnvelope> {
2041 let blob = match codec {
2042 DEFAULT_CODEC => encode_avro_value(&AvroValue::from_serialize(value)?)?.blob,
2043 other => return Err(unsupported_payload_codec(other)),
2044 };
2045
2046 Ok(PayloadEnvelope {
2047 codec: codec.to_string(),
2048 blob,
2049 })
2050}
2051
2052pub fn decode_payload<T: DeserializeOwned>(envelope: &PayloadEnvelope) -> Result<T> {
2053 match envelope.codec.as_str() {
2054 DEFAULT_CODEC => decode_avro_value(envelope)?.deserialize(),
2055 other => Err(unsupported_payload_codec(other)),
2056 }
2057}
2058
2059fn handler_type_error<T>(
2060 handler_kind: HandlerKind,
2061 handler_name: &str,
2062 value_kind: HandlerValueKind,
2063 message: impl Into<String>,
2064) -> Error {
2065 Error::HandlerType {
2066 handler_kind,
2067 handler_name: handler_name.to_string(),
2068 value_kind,
2069 rust_type: type_name::<T>(),
2070 message: message.into(),
2071 }
2072}
2073
2074fn decode_handler_input<T: DeserializeOwned>(
2075 arguments: AvroValue,
2076 handler_kind: HandlerKind,
2077 handler_name: &str,
2078) -> Result<T> {
2079 let argument = match arguments {
2080 AvroValue::Array(mut arguments) if arguments.len() == 1 => {
2081 arguments.pop().expect("one typed handler argument")
2082 }
2083 AvroValue::Array(arguments) if arguments.is_empty() => AvroValue::Null,
2084 AvroValue::Array(arguments) => {
2085 return Err(handler_type_error::<T>(
2086 handler_kind,
2087 handler_name,
2088 HandlerValueKind::Input,
2089 format!(
2090 "typed handlers accept one request value, but the task carried {} arguments",
2091 arguments.len()
2092 ),
2093 ));
2094 }
2095 argument => argument,
2096 };
2097
2098 argument.deserialize().map_err(|error| {
2099 handler_type_error::<T>(
2100 handler_kind,
2101 handler_name,
2102 HandlerValueKind::Input,
2103 error.to_string(),
2104 )
2105 })
2106}
2107
2108fn encode_handler_result<T: Serialize>(
2109 result: &T,
2110 handler_kind: HandlerKind,
2111 handler_name: &str,
2112) -> Result<AvroValue> {
2113 AvroValue::from_serialize(result).map_err(|error| {
2114 handler_type_error::<T>(
2115 handler_kind,
2116 handler_name,
2117 HandlerValueKind::Result,
2118 error.to_string(),
2119 )
2120 })
2121}
2122
2123fn decode_handler_result<T: DeserializeOwned>(
2124 result: AvroValue,
2125 handler_kind: HandlerKind,
2126 handler_name: &str,
2127) -> Result<T> {
2128 result.deserialize().map_err(|error| {
2129 handler_type_error::<T>(
2130 handler_kind,
2131 handler_name,
2132 HandlerValueKind::Result,
2133 error.to_string(),
2134 )
2135 })
2136}
2137
2138#[cfg(test)]
2139fn encode_value_envelope(value: &Value, codec: &str) -> Result<Value> {
2140 Ok(serde_json::to_value(encode_payload(value, codec)?)?)
2141}
2142
2143fn decode_wire_value(value: &Value, fallback_codec: &str) -> Result<Value> {
2144 validate_payload_codec(fallback_codec)?;
2145
2146 if value.is_null() {
2147 return Ok(Value::Null);
2148 }
2149
2150 if let Some((codec, blob)) = payload_envelope_parts(value)? {
2151 return decode_blob(blob, codec);
2152 }
2153
2154 if let Some(blob) = value.as_str() {
2155 return decode_blob(blob, fallback_codec);
2156 }
2157
2158 Err(untagged_payload_value())
2159}
2160
2161fn encode_typed_envelope(value: &AvroValue, codec: &str) -> Result<Value> {
2162 let envelope = match codec {
2163 DEFAULT_CODEC => encode_avro_value(value)?,
2164 other => return Err(unsupported_payload_codec(other)),
2165 };
2166 Ok(serde_json::to_value(envelope)?)
2167}
2168
2169fn decode_wire_avro_value(value: &Value, fallback_codec: &str) -> Result<AvroValue> {
2170 validate_payload_codec(fallback_codec)?;
2171
2172 if value.is_null() {
2173 return Ok(AvroValue::Null);
2174 }
2175
2176 if let Some((codec, blob)) = payload_envelope_parts(value)? {
2177 validate_payload_codec(codec)?;
2178 return decode_avro_value_blob(blob);
2179 }
2180
2181 if let Some(blob) = value.as_str() {
2182 return match fallback_codec {
2183 DEFAULT_CODEC => decode_avro_value_blob(blob),
2184 other => Err(unsupported_payload_codec(other)),
2185 };
2186 }
2187
2188 Err(untagged_payload_value())
2189}
2190
2191fn normalize_avro_arguments(value: AvroValue) -> AvroValue {
2192 match value {
2193 AvroValue::Null => AvroValue::Array(Vec::new()),
2194 AvroValue::Array(_) => value,
2195 other => AvroValue::Array(vec![other]),
2196 }
2197}
2198
2199fn decode_blob(blob: &str, codec: &str) -> Result<Value> {
2200 match codec {
2201 DEFAULT_CODEC => decode_avro_value_blob(blob)?.into_json(),
2202 other => Err(unsupported_payload_codec(other)),
2203 }
2204}
2205
2206fn validate_payload_codec(codec: &str) -> Result<()> {
2207 match codec {
2208 DEFAULT_CODEC => Ok(()),
2209 MISSING_TASK_PAYLOAD_CODEC => {
2210 Err(invalid_task_payload_codec("task payload_codec is missing"))
2211 }
2212 NULL_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec("task payload_codec is null")),
2213 NON_STRING_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec(
2214 "task payload_codec must be a string",
2215 )),
2216 other => Err(unsupported_payload_codec(other)),
2217 }
2218}
2219
2220fn invalid_task_payload_codec(reason: &str) -> Error {
2221 Error::Codec(format!(
2222 "unsupported_payload_codec: {reason}; Durable Workflow 2.0 requires an explicit string payload_codec=\"avro\" before worker task execution"
2223 ))
2224}
2225
2226fn payload_envelope_parts(value: &Value) -> Result<Option<(&str, &str)>> {
2227 let Some(object) = value.as_object() else {
2228 return Ok(None);
2229 };
2230 if !object.contains_key("codec") && !object.contains_key("blob") {
2231 return Ok(None);
2232 }
2233
2234 let codec = object
2235 .get("codec")
2236 .and_then(Value::as_str)
2237 .ok_or_else(invalid_payload_envelope)?;
2238 validate_payload_codec(codec)?;
2239 let blob = object
2240 .get("blob")
2241 .and_then(Value::as_str)
2242 .ok_or_else(invalid_payload_envelope)?;
2243 Ok(Some((codec, blob)))
2244}
2245
2246fn invalid_payload_envelope() -> Error {
2247 Error::Codec(
2248 "invalid_payload_envelope: durable payloads must use an object with string codec=\"avro\" and blob fields"
2249 .to_string(),
2250 )
2251}
2252
2253fn validate_workflow_task_commands(commands: &[Value]) -> Result<()> {
2254 for command in commands {
2255 let Some(command) = command.as_object() else {
2256 continue;
2257 };
2258 let Some(command_type) = command.get("type").and_then(Value::as_str) else {
2259 continue;
2260 };
2261 let Some(payload_field) = workflow_command_payload_field(command_type) else {
2262 continue;
2263 };
2264
2265 if let Some(codec) = command.get("payload_codec") {
2266 let codec = codec.as_str().ok_or_else(invalid_payload_envelope)?;
2267 validate_payload_codec(codec)?;
2268 }
2269
2270 let payload = command
2271 .get(payload_field)
2272 .ok_or_else(invalid_payload_envelope)?;
2273 validate_outbound_payload_envelope(payload)?;
2274 }
2275 Ok(())
2276}
2277
2278fn workflow_completion_protocol_version(commands: &[Value]) -> &'static str {
2279 if commands.iter().any(|command| {
2280 command.get("type").and_then(Value::as_str) == Some("open_condition_wait")
2281 && command
2282 .get("condition_wait_occurrence_id")
2283 .and_then(Value::as_str)
2284 .is_some_and(|occurrence_id| !occurrence_id.is_empty())
2285 }) {
2286 CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
2287 } else if commands.iter().any(|command| {
2288 command.get("type").and_then(Value::as_str) == Some("upsert_search_attributes")
2289 && command.get("attribute_types").is_some()
2290 }) {
2291 TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION
2292 } else if commands
2293 .iter()
2294 .any(|command| command.get("type").and_then(Value::as_str) == Some("upsert_memo"))
2295 {
2296 MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION
2297 } else if commands
2298 .iter()
2299 .any(|command| command.get("type").and_then(Value::as_str) == Some("open_condition_wait"))
2300 {
2301 CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION
2302 } else if commands.iter().any(|command| {
2303 command.get("type").and_then(Value::as_str) == Some("upsert_search_attributes")
2304 }) {
2305 SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION
2306 } else {
2307 WORKER_PROTOCOL_VERSION
2308 }
2309}
2310
2311fn workflow_completion_protocol_version_with_message_streams(
2312 commands: &[Value],
2313 has_message_stream_metadata: bool,
2314) -> &'static str {
2315 let command_protocol = workflow_completion_protocol_version(commands);
2316 if has_message_stream_metadata && !worker_protocol_supports_message_streams(command_protocol) {
2317 MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION
2318 } else {
2319 command_protocol
2320 }
2321}
2322
2323fn workflow_command_payload_field(command_type: &str) -> Option<&'static str> {
2324 match command_type {
2325 "complete_workflow" | "complete_update" | "record_side_effect" => Some("result"),
2326 "schedule_activity" | "start_child_workflow" | "continue_as_new" => Some("arguments"),
2327 "start_service_operation" => Some("request_payload"),
2328 "upsert_memo" => Some("entries"),
2329 _ => None,
2330 }
2331}
2332
2333fn validate_outbound_payload_envelope(value: &Value) -> Result<()> {
2334 let Some((codec, blob)) = payload_envelope_parts(value)? else {
2335 return Err(untagged_payload_value());
2336 };
2337 validate_payload_codec(codec)?;
2338 decode_avro_value_blob(blob)?;
2339 Ok(())
2340}
2341
2342fn unsupported_payload_codec(codec: &str) -> Error {
2343 Error::Codec(format!(
2344 "unsupported_payload_codec: workflow payload codec {codec:?} is not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
2345 ))
2346}
2347
2348fn untagged_payload_value() -> Error {
2349 Error::Codec(
2350 "unsupported_payload_codec: untagged durable payload values are not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
2351 .to_string(),
2352 )
2353}
2354
2355fn decode_avro_value_blob(blob: &str) -> Result<AvroValue> {
2356 let bytes = BASE64.decode(blob).map_err(|err| {
2357 Error::Codec(format!(
2358 "invalid_payload_framing: expected strict base64 Avro single-object bytes: {err}"
2359 ))
2360 })?;
2361
2362 if serde_json::from_slice::<Value>(&bytes).is_ok() {
2363 return Err(unsupported_payload_codec("json"));
2364 }
2365
2366 if bytes.len() < 10 || bytes[..2] != AVRO_SINGLE_OBJECT_MAGIC {
2367 return Err(Error::Codec(
2368 "invalid_payload_framing: expected Avro single-object magic c301".to_string(),
2369 ));
2370 }
2371
2372 let fingerprint: [u8; 8] = bytes[2..10]
2373 .try_into()
2374 .map_err(|_| Error::Codec("invalid Avro fingerprint length".to_string()))?;
2375 if fingerprint != AVRO_VALUE_SCHEMA_FINGERPRINT {
2376 return Err(Error::Codec(format!(
2377 "unsupported_payload_schema: unknown CRC-64-AVRO fingerprint {}",
2378 fingerprint
2379 .iter()
2380 .map(|byte| format!("{byte:02x}"))
2381 .collect::<String>()
2382 )));
2383 }
2384
2385 let mut datum_reader = StrictAvroDatumReader::new(&bytes[10..]);
2386 let datum = from_avro_datum(avro_value_schema()?, &mut datum_reader, None);
2390 if datum_reader.truncated {
2391 return Err(Error::Codec(
2392 "invalid_payload_framing: truncated Avro Value datum".to_string(),
2393 ));
2394 }
2395 let datum = datum.map_err(|err| {
2396 Error::Codec(format!(
2397 "invalid_payload_framing: malformed Avro Value datum: {err}"
2398 ))
2399 })?;
2400 if datum_reader.remaining() != 0 {
2401 return Err(Error::Codec(format!(
2402 "invalid_payload_framing: {} trailing bytes after Avro Value datum",
2403 datum_reader.remaining()
2404 )));
2405 }
2406 avro_value_from_datum(datum)
2407}
2408
2409struct StrictAvroDatumReader<'a> {
2410 bytes: &'a [u8],
2411 offset: usize,
2412 truncated: bool,
2413}
2414
2415impl<'a> StrictAvroDatumReader<'a> {
2416 fn new(bytes: &'a [u8]) -> Self {
2417 Self {
2418 bytes,
2419 offset: 0,
2420 truncated: false,
2421 }
2422 }
2423
2424 fn remaining(&self) -> usize {
2425 self.bytes.len() - self.offset
2426 }
2427}
2428
2429impl Read for StrictAvroDatumReader<'_> {
2430 fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
2431 let count = buffer.len().min(self.remaining());
2432 buffer[..count].copy_from_slice(&self.bytes[self.offset..self.offset + count]);
2433 self.offset += count;
2434 if count < buffer.len() {
2435 self.truncated = true;
2436 }
2437
2438 Ok(count)
2439 }
2440}
2441
2442fn avro_value_to_datum(value: &AvroValue) -> Result<AvroDatum> {
2443 let branch = match value {
2444 AvroValue::Null => AvroDatum::Union(0, Box::new(AvroDatum::Null)),
2445 AvroValue::Boolean(value) => AvroDatum::Union(
2446 1,
2447 Box::new(AvroDatum::Record(vec![(
2448 "boolean".to_string(),
2449 AvroDatum::Boolean(*value),
2450 )])),
2451 ),
2452 AvroValue::Long(value) => AvroDatum::Union(
2453 2,
2454 Box::new(AvroDatum::Record(vec![(
2455 "long".to_string(),
2456 AvroDatum::Long(*value),
2457 )])),
2458 ),
2459 AvroValue::Double(value) => {
2460 if !value.is_finite() {
2461 return Err(Error::Codec(
2462 "non_finite_float: Avro Value doubles must be finite".to_string(),
2463 ));
2464 }
2465 AvroDatum::Union(
2466 3,
2467 Box::new(AvroDatum::Record(vec![(
2468 "double".to_string(),
2469 AvroDatum::Double(*value),
2470 )])),
2471 )
2472 }
2473 AvroValue::Bytes(value) => AvroDatum::Union(
2474 4,
2475 Box::new(AvroDatum::Record(vec![(
2476 "bytes".to_string(),
2477 AvroDatum::Bytes(value.clone()),
2478 )])),
2479 ),
2480 AvroValue::String(value) => AvroDatum::Union(
2481 5,
2482 Box::new(AvroDatum::Record(vec![(
2483 "string".to_string(),
2484 AvroDatum::String(value.clone()),
2485 )])),
2486 ),
2487 AvroValue::Array(values) => AvroDatum::Union(
2488 6,
2489 Box::new(AvroDatum::Record(vec![(
2490 "items".to_string(),
2491 AvroDatum::Array(
2492 values
2493 .iter()
2494 .map(avro_value_to_datum)
2495 .collect::<Result<Vec<_>>>()?,
2496 ),
2497 )])),
2498 ),
2499 AvroValue::Map(values) => AvroDatum::Union(
2500 7,
2501 Box::new(AvroDatum::Record(vec![(
2502 "entries".to_string(),
2503 AvroDatum::Array(
2504 values
2505 .iter()
2506 .map(|(key, value)| {
2507 Ok(AvroDatum::Record(vec![
2508 ("key".to_string(), AvroDatum::String(key.clone())),
2509 ("value".to_string(), avro_value_to_datum(value)?),
2510 ]))
2511 })
2512 .collect::<Result<Vec<_>>>()?,
2513 ),
2514 )])),
2515 ),
2516 };
2517 Ok(AvroDatum::Record(vec![("value".to_string(), branch)]))
2518}
2519
2520fn avro_value_from_datum(datum: AvroDatum) -> Result<AvroValue> {
2521 let AvroDatum::Record(mut outer) = datum else {
2522 return Err(Error::Codec(
2523 "invalid_payload_framing: datum is not a Value record".to_string(),
2524 ));
2525 };
2526 let (_, branch) = outer
2527 .pop()
2528 .filter(|(name, _)| name == "value")
2529 .ok_or_else(|| Error::Codec("invalid_payload_framing: Value field missing".to_string()))?;
2530 let AvroDatum::Union(_, branch) = branch else {
2531 return Err(Error::Codec(
2532 "invalid_payload_framing: invalid Value union".to_string(),
2533 ));
2534 };
2535 match *branch {
2536 AvroDatum::Null => Ok(AvroValue::Null),
2537 AvroDatum::Record(mut fields) => {
2538 let (name, value) = fields.pop().ok_or_else(|| {
2539 Error::Codec("invalid_payload_framing: empty Value branch".to_string())
2540 })?;
2541 match (name.as_str(), value) {
2542 ("boolean", AvroDatum::Boolean(value)) => Ok(AvroValue::Boolean(value)),
2543 ("long", AvroDatum::Long(value)) => Ok(AvroValue::Long(value)),
2544 ("double", AvroDatum::Double(value)) if value.is_finite() => {
2545 Ok(AvroValue::Double(value))
2546 }
2547 ("bytes", AvroDatum::Bytes(value)) => Ok(AvroValue::Bytes(value)),
2548 ("string", AvroDatum::String(value)) => Ok(AvroValue::String(value)),
2549 ("items", AvroDatum::Array(values)) => values
2550 .into_iter()
2551 .map(avro_value_from_datum)
2552 .collect::<Result<Vec<_>>>()
2553 .map(AvroValue::Array),
2554 ("entries", AvroDatum::Map(values)) => values
2555 .into_iter()
2556 .map(|(key, value)| Ok((key, avro_value_from_datum(value)?)))
2557 .collect::<Result<BTreeMap<_, _>>>()
2558 .map(AvroValue::Map),
2559 _ => Err(Error::Codec(
2560 "invalid_payload_framing: unknown Value branch".to_string(),
2561 )),
2562 }
2563 }
2564 _ => Err(Error::Codec(
2565 "invalid_payload_framing: invalid Value branch".to_string(),
2566 )),
2567 }
2568}
2569
2570fn avro_value_schema() -> Result<&'static Schema> {
2571 match AVRO_VALUE_SCHEMA.get_or_init(|| {
2572 Schema::parse_str(AVRO_VALUE_SCHEMA_JSON)
2573 .map_err(|err| format!("could not parse Avro Value schema: {err}"))
2574 }) {
2575 Ok(schema) => Ok(schema),
2576 Err(message) => Err(Error::Codec(message.clone())),
2577 }
2578}
2579
2580fn avro_value_ordered_map_encoding_schema() -> Result<&'static Schema> {
2581 match AVRO_VALUE_ORDERED_MAP_ENCODING_SCHEMA.get_or_init(|| {
2582 let mut schema: Value = serde_json::from_str(AVRO_VALUE_SCHEMA_JSON)
2587 .map_err(|err| format!("could not read packaged Avro Value schema: {err}"))?;
2588 let entries_schema = schema
2589 .pointer_mut("/fields/0/type/7/fields/0/type")
2590 .ok_or_else(|| "packaged Avro Value map schema is missing".to_string())?;
2591 if *entries_schema != json!({"type": "map", "values": "Value"}) {
2592 return Err("packaged Avro Value map schema changed unexpectedly".to_string());
2593 }
2594 *entries_schema = json!({
2595 "type": "array",
2596 "items": {
2597 "type": "record",
2598 "name": "MapEntry",
2599 "fields": [
2600 {"name": "key", "type": "string"},
2601 {"name": "value", "type": "Value"}
2602 ]
2603 }
2604 });
2605 Schema::parse_str(&schema.to_string())
2606 .map_err(|err| format!("could not parse ordered-map Avro Value schema: {err}"))
2607 }) {
2608 Ok(schema) => Ok(schema),
2609 Err(message) => Err(Error::Codec(message.clone())),
2610 }
2611}
2612
2613#[derive(Clone, Debug)]
2614pub struct Client {
2615 http: reqwest::Client,
2616 base_url: String,
2617 token: Option<String>,
2618 control_token: Option<String>,
2619 worker_token: Option<String>,
2620 namespace: String,
2621}
2622
2623impl Client {
2624 pub fn new(base_url: impl Into<String>) -> Result<Self> {
2625 Self::builder(base_url).build()
2626 }
2627
2628 pub fn builder(base_url: impl Into<String>) -> ClientBuilder {
2629 ClientBuilder {
2630 base_url: base_url.into(),
2631 token: None,
2632 control_token: None,
2633 worker_token: None,
2634 namespace: "default".to_string(),
2635 timeout: Duration::from_secs(60),
2636 }
2637 }
2638
2639 pub async fn health(&self) -> Result<Value> {
2640 self.request_json(
2641 reqwest::Method::GET,
2642 "/health",
2643 RequestProtocol::ControlPlane,
2644 Option::<&Value>::None,
2645 )
2646 .await
2647 }
2648
2649 pub async fn cluster_info(&self) -> Result<Value> {
2650 self.request_json(
2651 reqwest::Method::GET,
2652 "/cluster/info",
2653 RequestProtocol::ControlPlane,
2654 Option::<&Value>::None,
2655 )
2656 .await
2657 }
2658
2659 pub async fn start_workflow<T: Serialize>(
2660 &self,
2661 workflow_type: &str,
2662 task_queue: &str,
2663 workflow_id: &str,
2664 input: T,
2665 ) -> Result<WorkflowHandle> {
2666 self.start_workflow_with_options(
2667 workflow_type,
2668 task_queue,
2669 workflow_id,
2670 WorkflowStartOptions::default(),
2671 input,
2672 )
2673 .await
2674 }
2675
2676 pub async fn start_workflow_with_options<T: Serialize>(
2679 &self,
2680 workflow_type: &str,
2681 task_queue: &str,
2682 workflow_id: &str,
2683 options: WorkflowStartOptions,
2684 input: T,
2685 ) -> Result<WorkflowHandle> {
2686 options.validate()?;
2687 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2688 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2689 let body = json!({
2690 "workflow_id": workflow_id,
2691 "workflow_type": workflow_type,
2692 "task_queue": task_queue,
2693 "input": input_envelope,
2694 "execution_timeout_seconds": options.execution_timeout_seconds,
2695 "run_timeout_seconds": options.run_timeout_seconds
2696 });
2697
2698 let data: Value = self
2699 .request_json(
2700 reqwest::Method::POST,
2701 "/workflows",
2702 RequestProtocol::ControlPlane,
2703 Some(&body),
2704 )
2705 .await?;
2706
2707 Ok(WorkflowHandle {
2708 client: self.clone(),
2709 workflow_id: data
2710 .get("workflow_id")
2711 .and_then(Value::as_str)
2712 .unwrap_or(workflow_id)
2713 .to_string(),
2714 run_id: data
2715 .get("run_id")
2716 .and_then(Value::as_str)
2717 .map(str::to_string),
2718 workflow_type: data
2719 .get("workflow_type")
2720 .and_then(Value::as_str)
2721 .unwrap_or(workflow_type)
2722 .to_string(),
2723 })
2724 }
2725
2726 pub async fn signal_workflow<T: Serialize>(
2727 &self,
2728 workflow_id: &str,
2729 signal_name: &str,
2730 input: T,
2731 ) -> Result<Value> {
2732 self.signal_workflow_target(workflow_id, None, signal_name, input)
2733 .await
2734 }
2735
2736 pub async fn append_message_stream<T: Serialize>(
2738 &self,
2739 workflow_id: &str,
2740 stream_name: &str,
2741 message_id: &str,
2742 input: T,
2743 ) -> Result<Value> {
2744 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2745 let body = json!({
2746 "message_id": message_id,
2747 "input": encode_typed_envelope(&input, DEFAULT_CODEC)?
2748 });
2749 self.request_json(
2750 reqwest::Method::POST,
2751 &format!("/workflows/{workflow_id}/message-streams/{stream_name}/messages"),
2752 RequestProtocol::ControlPlane,
2753 Some(&body),
2754 )
2755 .await
2756 }
2757
2758 pub async fn signal_workflow_run<T: Serialize>(
2760 &self,
2761 workflow_id: &str,
2762 run_id: &str,
2763 signal_name: &str,
2764 input: T,
2765 ) -> Result<Value> {
2766 self.signal_workflow_target(workflow_id, Some(run_id), signal_name, input)
2767 .await
2768 }
2769
2770 async fn signal_workflow_target<T: Serialize>(
2771 &self,
2772 workflow_id: &str,
2773 run_id: Option<&str>,
2774 signal_name: &str,
2775 input: T,
2776 ) -> Result<Value> {
2777 validate_user_signal_name(signal_name)?;
2778 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2779 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2780 let body = json!({
2781 "input": input_envelope
2782 });
2783 let path = match run_id {
2784 Some(run_id) => {
2785 format!("/workflows/{workflow_id}/runs/{run_id}/signal/{signal_name}")
2786 }
2787 None => format!("/workflows/{workflow_id}/signal/{signal_name}"),
2788 };
2789 self.request_json(
2790 reqwest::Method::POST,
2791 &path,
2792 RequestProtocol::ControlPlane,
2793 Some(&body),
2794 )
2795 .await
2796 }
2797
2798 pub async fn cancel_workflow(
2800 &self,
2801 workflow_id: &str,
2802 options: WorkflowCommandOptions,
2803 ) -> Result<WorkflowCommandResult> {
2804 self.workflow_command(workflow_id, None, WorkflowCommandKind::Cancel, options)
2805 .await
2806 }
2807
2808 pub async fn cancel_workflow_run(
2810 &self,
2811 workflow_id: &str,
2812 run_id: &str,
2813 options: WorkflowCommandOptions,
2814 ) -> Result<WorkflowCommandResult> {
2815 self.workflow_command(
2816 workflow_id,
2817 Some(run_id),
2818 WorkflowCommandKind::Cancel,
2819 options,
2820 )
2821 .await
2822 }
2823
2824 pub async fn terminate_workflow(
2826 &self,
2827 workflow_id: &str,
2828 options: WorkflowCommandOptions,
2829 ) -> Result<WorkflowCommandResult> {
2830 self.workflow_command(workflow_id, None, WorkflowCommandKind::Terminate, options)
2831 .await
2832 }
2833
2834 pub async fn terminate_workflow_run(
2836 &self,
2837 workflow_id: &str,
2838 run_id: &str,
2839 options: WorkflowCommandOptions,
2840 ) -> Result<WorkflowCommandResult> {
2841 self.workflow_command(
2842 workflow_id,
2843 Some(run_id),
2844 WorkflowCommandKind::Terminate,
2845 options,
2846 )
2847 .await
2848 }
2849
2850 async fn workflow_command(
2851 &self,
2852 workflow_id: &str,
2853 run_id: Option<&str>,
2854 command: WorkflowCommandKind,
2855 options: WorkflowCommandOptions,
2856 ) -> Result<WorkflowCommandResult> {
2857 let path = match run_id {
2858 Some(run_id) => format!(
2859 "/workflows/{workflow_id}/runs/{run_id}/{}",
2860 command.as_str()
2861 ),
2862 None => format!("/workflows/{workflow_id}/{}", command.as_str()),
2863 };
2864 let data = match self
2865 .request_json(
2866 reqwest::Method::POST,
2867 &path,
2868 RequestProtocol::ControlPlane,
2869 Some(&options),
2870 )
2871 .await
2872 {
2873 Ok(data) => data,
2874 Err(Error::Http { status, body }) => {
2875 return Err(Error::WorkflowCommandRejected(workflow_command_rejection(
2876 command,
2877 status,
2878 body,
2879 workflow_id,
2880 run_id,
2881 )));
2882 }
2883 Err(error) => return Err(error),
2884 };
2885
2886 Ok(workflow_command_result(command, data, workflow_id, run_id))
2887 }
2888
2889 pub async fn query_workflow<T: Serialize>(
2895 &self,
2896 workflow_id: &str,
2897 query_name: &str,
2898 input: T,
2899 ) -> Result<Value> {
2900 self.query_workflow_target(workflow_id, None, query_name, input)
2901 .await
2902 }
2903
2904 pub async fn query_workflow_run<T: Serialize>(
2906 &self,
2907 workflow_id: &str,
2908 run_id: &str,
2909 query_name: &str,
2910 input: T,
2911 ) -> Result<Value> {
2912 self.query_workflow_target(workflow_id, Some(run_id), query_name, input)
2913 .await
2914 }
2915
2916 pub async fn query_workflow_avro_value<T: Serialize>(
2918 &self,
2919 workflow_id: &str,
2920 query_name: &str,
2921 input: T,
2922 ) -> Result<AvroValue> {
2923 self.query_workflow_avro_value_target(workflow_id, None, query_name, input)
2924 .await
2925 }
2926
2927 pub async fn query_workflow_run_avro_value<T: Serialize>(
2929 &self,
2930 workflow_id: &str,
2931 run_id: &str,
2932 query_name: &str,
2933 input: T,
2934 ) -> Result<AvroValue> {
2935 self.query_workflow_avro_value_target(workflow_id, Some(run_id), query_name, input)
2936 .await
2937 }
2938
2939 async fn query_workflow_avro_value_target<T: Serialize>(
2940 &self,
2941 workflow_id: &str,
2942 run_id: Option<&str>,
2943 query_name: &str,
2944 input: T,
2945 ) -> Result<AvroValue> {
2946 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2947 let body = json!({"input": encode_typed_envelope(&input, DEFAULT_CODEC)?});
2948 let path = match run_id {
2949 Some(run_id) => {
2950 format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
2951 }
2952 None => format!("/workflows/{workflow_id}/query/{query_name}"),
2953 };
2954 let response: Value = match self
2955 .request_json(
2956 reqwest::Method::POST,
2957 &path,
2958 RequestProtocol::ControlPlane,
2959 Some(&body),
2960 )
2961 .await
2962 {
2963 Ok(response) => response,
2964 Err(Error::Http { status, body }) => {
2965 return Err(Error::QueryFailed(query_failure(status, body)));
2966 }
2967 Err(error) => return Err(error),
2968 };
2969
2970 let envelope = response
2971 .get("result_envelope")
2972 .filter(|envelope| !envelope.is_null())
2973 .ok_or_else(|| {
2974 Error::Codec(
2975 "missing_payload_envelope: typed query result requires result_envelope"
2976 .to_string(),
2977 )
2978 })?;
2979 decode_wire_avro_value(envelope, DEFAULT_CODEC)
2980 }
2981
2982 async fn query_workflow_target<T: Serialize>(
2983 &self,
2984 workflow_id: &str,
2985 run_id: Option<&str>,
2986 query_name: &str,
2987 input: T,
2988 ) -> Result<Value> {
2989 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2990 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2991 let body = json!({
2992 "input": input_envelope
2993 });
2994 let path = match run_id {
2995 Some(run_id) => {
2996 format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
2997 }
2998 None => format!("/workflows/{workflow_id}/query/{query_name}"),
2999 };
3000 let response: Value = match self
3001 .request_json(
3002 reqwest::Method::POST,
3003 &path,
3004 RequestProtocol::ControlPlane,
3005 Some(&body),
3006 )
3007 .await
3008 {
3009 Ok(response) => response,
3010 Err(Error::Http { status, body }) => {
3011 return Err(Error::QueryFailed(query_failure(status, body)));
3012 }
3013 Err(error) => return Err(error),
3014 };
3015
3016 if let Some(envelope) = response
3017 .get("result_envelope")
3018 .filter(|envelope| !envelope.is_null())
3019 {
3020 return decode_wire_value(envelope, DEFAULT_CODEC);
3021 }
3022
3023 Ok(response.get("result").cloned().unwrap_or(Value::Null))
3024 }
3025
3026 pub async fn update_workflow<T: Serialize>(
3028 &self,
3029 workflow_id: &str,
3030 update_name: &str,
3031 input: T,
3032 request_id: Option<&str>,
3033 ) -> Result<Value> {
3034 let response = self
3035 .update_workflow_response(workflow_id, update_name, input, request_id)
3036 .await?;
3037 if let Some(envelope) = response
3038 .get("result_envelope")
3039 .filter(|envelope| !envelope.is_null())
3040 {
3041 return decode_wire_value(envelope, DEFAULT_CODEC);
3042 }
3043 Ok(response.get("result").cloned().unwrap_or(response))
3044 }
3045
3046 pub async fn update_workflow_avro_value<T: Serialize>(
3048 &self,
3049 workflow_id: &str,
3050 update_name: &str,
3051 input: T,
3052 request_id: Option<&str>,
3053 ) -> Result<AvroValue> {
3054 let response = self
3055 .update_workflow_response(workflow_id, update_name, input, request_id)
3056 .await?;
3057 let envelope = response
3058 .get("result_envelope")
3059 .filter(|envelope| !envelope.is_null())
3060 .ok_or_else(|| {
3061 Error::Codec(
3062 "missing_payload_envelope: typed update result requires result_envelope"
3063 .to_string(),
3064 )
3065 })?;
3066 decode_wire_avro_value(envelope, DEFAULT_CODEC)
3067 }
3068
3069 async fn update_workflow_response<T: Serialize>(
3070 &self,
3071 workflow_id: &str,
3072 update_name: &str,
3073 input: T,
3074 request_id: Option<&str>,
3075 ) -> Result<Value> {
3076 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
3077 let mut body = json!({
3078 "input": encode_typed_envelope(&input, DEFAULT_CODEC)?,
3079 "wait_for": "completed",
3080 });
3081 if let Some(request_id) = request_id {
3082 body["request_id"] = json!(request_id);
3083 }
3084 self.request_json(
3085 reqwest::Method::POST,
3086 &format!("/workflows/{workflow_id}/update/{update_name}"),
3087 RequestProtocol::ControlPlane,
3088 Some(&body),
3089 )
3090 .await
3091 }
3092
3093 pub async fn describe_workflow(&self, workflow_id: &str) -> Result<WorkflowDescription> {
3094 let path = format!("/workflows/{workflow_id}");
3095 let mut data: WorkflowDescription = self
3096 .request_json(
3097 reqwest::Method::GET,
3098 &path,
3099 RequestProtocol::ControlPlane,
3100 Option::<&Value>::None,
3101 )
3102 .await?;
3103 data.decode_payloads()?;
3104 Ok(data)
3105 }
3106
3107 pub async fn describe_workflow_run(
3109 &self,
3110 workflow_id: &str,
3111 run_id: &str,
3112 ) -> Result<WorkflowDescription> {
3113 let path = format!("/workflows/{workflow_id}/runs/{run_id}");
3114 let mut data: WorkflowDescription = self
3115 .request_json(
3116 reqwest::Method::GET,
3117 &path,
3118 RequestProtocol::ControlPlane,
3119 Option::<&Value>::None,
3120 )
3121 .await?;
3122 data.decode_payloads()?;
3123 Ok(data)
3124 }
3125
3126 fn workflow_stream_path(workflow_id: &str, run_id: &str, stream_name: Option<&str>) -> String {
3127 let mut path = format!(
3128 "/workflows/{}/runs/{}/streams",
3129 percent_encode_path_segment(workflow_id),
3130 percent_encode_path_segment(run_id),
3131 );
3132 if let Some(stream_name) = stream_name {
3133 path.push('/');
3134 path.push_str(&percent_encode_path_segment(stream_name));
3135 }
3136 path
3137 }
3138
3139 pub async fn list_workflow_streams(
3141 &self,
3142 workflow_id: &str,
3143 run_id: &str,
3144 ) -> Result<Vec<WorkflowStreamDescription>> {
3145 let response: WorkflowStreamListResponse = self
3146 .request_json(
3147 reqwest::Method::GET,
3148 &Self::workflow_stream_path(workflow_id, run_id, None),
3149 RequestProtocol::ControlPlane,
3150 Option::<&Value>::None,
3151 )
3152 .await?;
3153 Ok(response.streams)
3154 }
3155
3156 pub async fn describe_workflow_stream(
3158 &self,
3159 workflow_id: &str,
3160 run_id: &str,
3161 stream_name: &str,
3162 ) -> Result<WorkflowStreamDescription> {
3163 let response: WorkflowStreamDescriptionResponse = self
3164 .request_json(
3165 reqwest::Method::GET,
3166 &Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3167 RequestProtocol::ControlPlane,
3168 Option::<&Value>::None,
3169 )
3170 .await?;
3171 Ok(response.stream)
3172 }
3173
3174 pub async fn subscribe_workflow_stream(
3181 &self,
3182 workflow_id: &str,
3183 run_id: &str,
3184 stream_name: &str,
3185 from_offset: u64,
3186 max_items: usize,
3187 wait: Duration,
3188 ) -> Result<WorkflowStreamPage> {
3189 let max_items = max_items.clamp(1, 500);
3190 let wait_seconds = wait.as_secs().min(MAX_LONG_POLL_TIMEOUT_SECONDS);
3191 let path = format!(
3192 "{}/items?from={from_offset}&max_items={max_items}&wait_seconds={wait_seconds}",
3193 Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3194 );
3195 let response: WorkflowStreamPageResponse = self
3196 .request_json_with_timeout(
3197 reqwest::Method::GET,
3198 &path,
3199 RequestProtocol::ControlPlane,
3200 Option::<&Value>::None,
3201 Duration::from_secs(wait_seconds.saturating_add(5).max(5)),
3202 )
3203 .await?;
3204
3205 let items = response
3206 .items
3207 .into_iter()
3208 .map(|raw| {
3209 let offset = raw.get("offset").and_then(Value::as_u64).unwrap_or(0);
3210 let envelope = raw.get("payload").cloned();
3211 let payload = envelope
3212 .as_ref()
3213 .filter(|value| value.get("blob").is_some())
3214 .map(|value| decode_wire_avro_value(value, DEFAULT_CODEC))
3215 .transpose()?
3216 .map(AvroValue::into_json)
3217 .transpose()?;
3218 Ok(WorkflowStreamItem {
3219 offset,
3220 payload,
3221 payload_envelope: envelope,
3222 payload_reference: raw
3223 .get("payload_reference")
3224 .and_then(Value::as_str)
3225 .map(str::to_string),
3226 payload_codec: raw
3227 .get("payload_codec")
3228 .and_then(Value::as_str)
3229 .map(str::to_string),
3230 idempotency_key: raw
3231 .get("idempotency_key")
3232 .and_then(Value::as_str)
3233 .map(str::to_string),
3234 item_type: raw
3235 .get("item_type")
3236 .and_then(Value::as_str)
3237 .map(str::to_string),
3238 content_type: raw
3239 .get("content_type")
3240 .and_then(Value::as_str)
3241 .map(str::to_string),
3242 origin: raw
3243 .get("origin")
3244 .and_then(Value::as_str)
3245 .map(str::to_string),
3246 origin_reference: raw
3247 .get("origin_reference")
3248 .and_then(Value::as_str)
3249 .map(str::to_string),
3250 emitted_at: raw
3251 .get("emitted_at")
3252 .and_then(Value::as_str)
3253 .map(str::to_string),
3254 raw,
3255 })
3256 })
3257 .collect::<Result<Vec<_>>>()?;
3258 Ok(WorkflowStreamPage {
3259 stream: response.stream,
3260 items,
3261 next_offset: response.next_offset,
3262 terminal: response.terminal,
3263 })
3264 }
3265
3266 pub async fn append_workflow_stream(
3268 &self,
3269 workflow_id: &str,
3270 run_id: &str,
3271 stream_name: &str,
3272 items: &[WorkflowStreamAppendItem],
3273 max_pending_items: Option<u64>,
3274 ) -> Result<WorkflowStreamAppendResult> {
3275 if items.is_empty() {
3276 return Err(Error::Codec(
3277 "workflow_stream_items_empty: append requires at least one item".to_string(),
3278 ));
3279 }
3280 let mut body = json!({
3281 "items": items
3282 .iter()
3283 .map(|item| item.wire_value(None))
3284 .collect::<Vec<_>>(),
3285 });
3286 if let Some(max_pending_items) = max_pending_items {
3287 if max_pending_items == 0 {
3288 return Err(Error::Codec(
3289 "workflow_stream_pending_limit_invalid: max_pending_items must be positive"
3290 .to_string(),
3291 ));
3292 }
3293 body["max_pending_items"] = json!(max_pending_items);
3294 }
3295 let response: WorkflowStreamAppendResponse = self
3296 .request_json(
3297 reqwest::Method::POST,
3298 &format!(
3299 "{}/items",
3300 Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3301 ),
3302 RequestProtocol::ControlPlane,
3303 Some(&body),
3304 )
3305 .await?;
3306 Ok(WorkflowStreamAppendResult {
3307 stream: response.stream,
3308 accepted_offsets: response.accepted_offsets,
3309 accepted: response.accepted,
3310 deduped: response.deduped,
3311 })
3312 }
3313
3314 pub async fn close_workflow_stream(
3316 &self,
3317 workflow_id: &str,
3318 run_id: &str,
3319 stream_name: &str,
3320 error_reason: Option<&str>,
3321 retention_seconds: Option<u64>,
3322 ) -> Result<WorkflowStreamDescription> {
3323 let mut body = json!({});
3324 if let Some(error_reason) = error_reason {
3325 body["error_reason"] = json!(error_reason);
3326 }
3327 if let Some(retention_seconds) = retention_seconds {
3328 if retention_seconds == 0 {
3329 return Err(Error::Codec(
3330 "workflow_stream_retention_invalid: retention_seconds must be positive"
3331 .to_string(),
3332 ));
3333 }
3334 body["retention_seconds"] = json!(retention_seconds);
3335 }
3336 let response: WorkflowStreamDescriptionResponse = self
3337 .request_json(
3338 reqwest::Method::POST,
3339 &format!(
3340 "{}/close",
3341 Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3342 ),
3343 RequestProtocol::ControlPlane,
3344 Some(&body),
3345 )
3346 .await?;
3347 Ok(response.stream)
3348 }
3349
3350 pub async fn register_worker(
3351 &self,
3352 worker_id: &str,
3353 task_queue: &str,
3354 supported_workflow_types: Vec<String>,
3355 supported_activity_types: Vec<String>,
3356 max_concurrent_workflow_tasks: usize,
3357 max_concurrent_activity_tasks: usize,
3358 ) -> Result<RegisterWorkerResponse> {
3359 self.register_worker_with_capabilities(
3360 worker_id,
3361 task_queue,
3362 supported_workflow_types,
3363 supported_activity_types,
3364 max_concurrent_workflow_tasks,
3365 max_concurrent_activity_tasks,
3366 Vec::new(),
3367 )
3368 .await
3369 }
3370
3371 pub async fn register_worker_with_capabilities(
3373 &self,
3374 worker_id: &str,
3375 task_queue: &str,
3376 supported_workflow_types: Vec<String>,
3377 supported_activity_types: Vec<String>,
3378 max_concurrent_workflow_tasks: usize,
3379 max_concurrent_activity_tasks: usize,
3380 capabilities: Vec<String>,
3381 ) -> Result<RegisterWorkerResponse> {
3382 self.register_worker_with_command_contracts(
3383 worker_id,
3384 task_queue,
3385 supported_workflow_types,
3386 supported_activity_types,
3387 max_concurrent_workflow_tasks,
3388 max_concurrent_activity_tasks,
3389 capabilities,
3390 Value::Object(serde_json::Map::new()),
3391 )
3392 .await
3393 }
3394
3395 #[allow(clippy::too_many_arguments)]
3402 pub async fn register_worker_with_command_contracts(
3403 &self,
3404 worker_id: &str,
3405 task_queue: &str,
3406 supported_workflow_types: Vec<String>,
3407 supported_activity_types: Vec<String>,
3408 max_concurrent_workflow_tasks: usize,
3409 max_concurrent_activity_tasks: usize,
3410 capabilities: Vec<String>,
3411 workflow_command_contracts: Value,
3412 ) -> Result<RegisterWorkerResponse> {
3413 if let Some(contracts) = workflow_command_contracts.as_object() {
3414 for (workflow_type, contract) in contracts {
3415 let Some(update_validators) = contract.get("update_validators") else {
3416 continue;
3417 };
3418 if !update_validators
3419 .as_array()
3420 .is_some_and(|validators| validators.is_empty())
3421 {
3422 return Err(Error::UnsupportedUpdateValidators {
3423 workflow_type: workflow_type.clone(),
3424 });
3425 }
3426 }
3427 }
3428
3429 let mut body = json!({
3430 "worker_id": worker_id,
3431 "task_queue": task_queue,
3432 "runtime": "rust",
3433 "sdk_version": SDK_VERSION,
3434 "supported_workflow_types": supported_workflow_types,
3435 "supported_activity_types": supported_activity_types,
3436 "capabilities": capabilities,
3437 "capability_manifest": portable_worker_affinity_capability_manifest(),
3438 "max_concurrent_workflow_tasks": max_concurrent_workflow_tasks,
3439 "max_concurrent_activity_tasks": max_concurrent_activity_tasks
3440 });
3441 if workflow_command_contracts
3442 .as_object()
3443 .is_some_and(|contracts| !contracts.is_empty())
3444 {
3445 body["workflow_command_contracts"] = workflow_command_contracts;
3446 }
3447
3448 self.request_json(
3449 reqwest::Method::POST,
3450 "/worker/register",
3451 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3452 Some(&body),
3453 )
3454 .await
3455 }
3456
3457 pub async fn deregister_worker_registration(
3463 &self,
3464 worker_id: &str,
3465 ) -> Result<WorkerDeregistrationEnvelope> {
3466 let path = format!(
3467 "/worker/registrations/{}",
3468 percent_encode_path_segment(worker_id)
3469 );
3470 self.request_json(
3471 reqwest::Method::DELETE,
3472 &path,
3473 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3474 Option::<&Value>::None,
3475 )
3476 .await
3477 }
3478
3479 pub async fn poll_query_task(
3481 &self,
3482 worker_id: &str,
3483 task_queue: &str,
3484 timeout: Duration,
3485 ) -> Result<Option<QueryTask>> {
3486 Ok(self
3487 .poll_query_task_response(worker_id, task_queue, timeout)
3488 .await?
3489 .task)
3490 }
3491
3492 pub async fn poll_query_task_response(
3494 &self,
3495 worker_id: &str,
3496 task_queue: &str,
3497 timeout: Duration,
3498 ) -> Result<PollQueryTaskResponse> {
3499 let poll_request_id = unique_request_id("rust-query-poll");
3500 self.poll_query_task_response_with_request_id(
3501 worker_id,
3502 task_queue,
3503 timeout,
3504 &poll_request_id,
3505 1,
3506 )
3507 .await
3508 }
3509
3510 async fn poll_query_task_response_with_request_id(
3511 &self,
3512 worker_id: &str,
3513 task_queue: &str,
3514 timeout: Duration,
3515 poll_request_id: &str,
3516 transport_retries: usize,
3517 ) -> Result<PollQueryTaskResponse> {
3518 let timeout_seconds = long_poll_timeout_seconds(timeout);
3519 let body = json!({
3520 "worker_id": worker_id,
3521 "task_queue": task_queue,
3522 "poll_request_id": poll_request_id,
3523 "timeout_seconds": timeout_seconds,
3524 });
3525 self.poll_request_json(
3526 "/worker/query-tasks/poll",
3527 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3528 &body,
3529 timeout + Duration::from_secs(5),
3530 transport_retries,
3531 )
3532 .await
3533 }
3534
3535 pub async fn complete_query_task<T: Serialize>(
3537 &self,
3538 query_task_id: &str,
3539 lease_owner: &str,
3540 query_task_attempt: u64,
3541 result: T,
3542 codec: &str,
3543 ) -> Result<Value> {
3544 let typed_result = AvroValue::from_serialize(&result)?;
3545 let result_envelope = encode_typed_envelope(&typed_result, codec)?;
3546 self.complete_query_task_with_envelope(
3547 query_task_id,
3548 lease_owner,
3549 query_task_attempt,
3550 typed_result.into_json()?,
3551 result_envelope,
3552 )
3553 .await
3554 }
3555
3556 async fn complete_query_task_with_envelope(
3557 &self,
3558 query_task_id: &str,
3559 lease_owner: &str,
3560 query_task_attempt: u64,
3561 result: Value,
3562 result_envelope: Value,
3563 ) -> Result<Value> {
3564 let body = json!({
3565 "lease_owner": lease_owner,
3566 "query_task_attempt": query_task_attempt,
3567 "result": result,
3568 "result_envelope": result_envelope,
3569 });
3570 let path = format!("/worker/query-tasks/{query_task_id}/complete");
3571 let response = self
3572 .request_json(
3573 reqwest::Method::POST,
3574 &path,
3575 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3576 Some(&body),
3577 )
3578 .await;
3579 query_task_response(response)
3580 }
3581
3582 pub async fn fail_query_task(
3584 &self,
3585 query_task_id: &str,
3586 lease_owner: &str,
3587 query_task_attempt: u64,
3588 message: impl Into<String>,
3589 reason: impl Into<String>,
3590 failure_type: impl Into<String>,
3591 ) -> Result<Value> {
3592 let body = json!({
3593 "lease_owner": lease_owner,
3594 "query_task_attempt": query_task_attempt,
3595 "failure": {
3596 "message": message.into(),
3597 "reason": reason.into(),
3598 "type": failure_type.into(),
3599 }
3600 });
3601 let path = format!("/worker/query-tasks/{query_task_id}/fail");
3602 let response = self
3603 .request_json(
3604 reqwest::Method::POST,
3605 &path,
3606 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3607 Some(&body),
3608 )
3609 .await;
3610 query_task_response(response)
3611 }
3612
3613 pub async fn heartbeat_worker(
3614 &self,
3615 worker_id: &str,
3616 workflow_available: usize,
3617 activity_available: usize,
3618 ) -> Result<Value> {
3619 let body = json!({
3620 "worker_id": worker_id,
3621 "task_slots": {
3622 "workflow_available": workflow_available,
3623 "activity_available": activity_available
3624 },
3625 "process_metrics": {
3626 "process_id": std::process::id(),
3627 "process_uptime_seconds": 0
3628 }
3629 });
3630
3631 self.request_json(
3632 reqwest::Method::POST,
3633 "/worker/heartbeat",
3634 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3635 Some(&body),
3636 )
3637 .await
3638 }
3639
3640 pub async fn poll_workflow_task(
3641 &self,
3642 worker_id: &str,
3643 task_queue: &str,
3644 timeout: Duration,
3645 ) -> Result<Option<WorkflowTask>> {
3646 Ok(self
3647 .poll_workflow_task_response(worker_id, task_queue, timeout)
3648 .await?
3649 .task)
3650 }
3651
3652 pub async fn poll_workflow_task_response(
3653 &self,
3654 worker_id: &str,
3655 task_queue: &str,
3656 timeout: Duration,
3657 ) -> Result<PollWorkflowTaskResponse> {
3658 let poll_request_id = unique_request_id("rust-workflow-poll");
3659 self.poll_workflow_task_response_with_request_id(
3660 worker_id,
3661 task_queue,
3662 timeout,
3663 &poll_request_id,
3664 1,
3665 )
3666 .await
3667 }
3668
3669 async fn poll_workflow_task_response_with_request_id(
3670 &self,
3671 worker_id: &str,
3672 task_queue: &str,
3673 timeout: Duration,
3674 poll_request_id: &str,
3675 transport_retries: usize,
3676 ) -> Result<PollWorkflowTaskResponse> {
3677 let body = json!({
3678 "worker_id": worker_id,
3679 "task_queue": task_queue,
3680 "poll_request_id": poll_request_id,
3681 "timeout_seconds": long_poll_timeout_seconds(timeout),
3682 });
3683 let mut data: PollWorkflowTaskResponse = self
3684 .poll_request_json(
3685 "/worker/workflow-tasks/poll",
3686 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3687 &body,
3688 timeout + Duration::from_secs(5),
3689 transport_retries,
3690 )
3691 .await?;
3692
3693 if let Some(task) = data.task.as_mut() {
3694 self.fetch_remaining_workflow_history(worker_id, task)
3695 .await?;
3696 }
3697
3698 Ok(data)
3699 }
3700
3701 async fn fetch_remaining_workflow_history(
3702 &self,
3703 worker_id: &str,
3704 task: &mut WorkflowTask,
3705 ) -> Result<()> {
3706 let mut next_token = task.next_history_page_token.clone();
3707
3708 while let Some(token) = next_token.take().filter(|token| !token.is_empty()) {
3709 let lease_owner = task
3710 .lease_owner
3711 .clone()
3712 .unwrap_or_else(|| worker_id.to_string());
3713 let page = self
3714 .workflow_task_history_page(
3715 &task.task_id,
3716 &lease_owner,
3717 task.workflow_task_attempt,
3718 &token,
3719 )
3720 .await?;
3721
3722 task.append_history_page(page);
3723
3724 if task.next_history_page_token.as_deref() == Some(token.as_str()) {
3725 return Err(Error::Codec(
3726 "workflow history pagination returned the same page token".to_string(),
3727 ));
3728 }
3729
3730 next_token = task.next_history_page_token.clone();
3731 }
3732
3733 Ok(())
3734 }
3735
3736 async fn workflow_task_history_page(
3737 &self,
3738 task_id: &str,
3739 lease_owner: &str,
3740 workflow_task_attempt: u64,
3741 next_history_page_token: &str,
3742 ) -> Result<WorkflowTaskHistoryPage> {
3743 let body = json!({
3744 "lease_owner": lease_owner,
3745 "workflow_task_attempt": workflow_task_attempt,
3746 "next_history_page_token": next_history_page_token
3747 });
3748 let path = format!("/worker/workflow-tasks/{task_id}/history");
3749
3750 self.request_json(
3751 reqwest::Method::POST,
3752 &path,
3753 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3754 Some(&body),
3755 )
3756 .await
3757 }
3758
3759 pub async fn complete_workflow_task(
3760 &self,
3761 task_id: &str,
3762 lease_owner: &str,
3763 workflow_task_attempt: u64,
3764 commands: Vec<Value>,
3765 ) -> Result<Value> {
3766 self.complete_workflow_task_with_message_streams(
3767 task_id,
3768 lease_owner,
3769 workflow_task_attempt,
3770 commands,
3771 Vec::new(),
3772 Vec::new(),
3773 )
3774 .await
3775 }
3776
3777 async fn complete_workflow_task_with_message_streams(
3778 &self,
3779 task_id: &str,
3780 lease_owner: &str,
3781 workflow_task_attempt: u64,
3782 commands: Vec<Value>,
3783 message_stream_cursors: Vec<Value>,
3784 message_stream_waits: Vec<Value>,
3785 ) -> Result<Value> {
3786 validate_workflow_task_commands(&commands)?;
3787 let has_message_stream_metadata =
3788 !message_stream_cursors.is_empty() || !message_stream_waits.is_empty();
3789 if has_message_stream_metadata
3790 && !worker_protocol_supports_message_streams(WORKER_PROTOCOL_VERSION)
3791 {
3792 return Err(Error::Codec(
3793 "message_streams_unavailable: message stream completion metadata requires worker protocol 1.15 or newer"
3794 .to_string(),
3795 ));
3796 }
3797 let protocol_version = workflow_completion_protocol_version_with_message_streams(
3798 &commands,
3799 has_message_stream_metadata,
3800 );
3801 let mut body = json!({
3802 "lease_owner": lease_owner,
3803 "workflow_task_attempt": workflow_task_attempt,
3804 "commands": commands
3805 });
3806 if !message_stream_cursors.is_empty() {
3807 body["message_stream_cursors"] = Value::Array(message_stream_cursors);
3808 }
3809 if !message_stream_waits.is_empty() {
3810 body["message_stream_waits"] = Value::Array(message_stream_waits);
3811 }
3812 let path = format!("/worker/workflow-tasks/{task_id}/complete");
3813 self.request_json(
3814 reqwest::Method::POST,
3815 &path,
3816 RequestProtocol::Worker(protocol_version),
3817 Some(&body),
3818 )
3819 .await
3820 }
3821
3822 pub async fn fail_workflow_task(
3823 &self,
3824 task_id: &str,
3825 lease_owner: &str,
3826 workflow_task_attempt: u64,
3827 message: impl Into<String>,
3828 ) -> Result<Value> {
3829 self.fail_workflow_task_with_type(
3830 task_id,
3831 lease_owner,
3832 workflow_task_attempt,
3833 message,
3834 "RustWorkflowTaskFailure",
3835 )
3836 .await
3837 }
3838
3839 async fn fail_workflow_task_with_type(
3840 &self,
3841 task_id: &str,
3842 lease_owner: &str,
3843 workflow_task_attempt: u64,
3844 message: impl Into<String>,
3845 failure_type: &str,
3846 ) -> Result<Value> {
3847 let body = json!({
3848 "lease_owner": lease_owner,
3849 "workflow_task_attempt": workflow_task_attempt,
3850 "failure": {
3851 "message": message.into(),
3852 "type": failure_type
3853 }
3854 });
3855 let path = format!("/worker/workflow-tasks/{task_id}/fail");
3856 self.request_json(
3857 reqwest::Method::POST,
3858 &path,
3859 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3860 Some(&body),
3861 )
3862 .await
3863 }
3864
3865 pub async fn poll_activity_task(
3866 &self,
3867 worker_id: &str,
3868 task_queue: &str,
3869 timeout: Duration,
3870 ) -> Result<Option<ActivityTask>> {
3871 Ok(self
3872 .poll_activity_task_response(worker_id, task_queue, timeout)
3873 .await?
3874 .task)
3875 }
3876
3877 pub async fn poll_activity_task_response(
3879 &self,
3880 worker_id: &str,
3881 task_queue: &str,
3882 timeout: Duration,
3883 ) -> Result<PollActivityTaskResponse> {
3884 let poll_request_id = unique_request_id("rust-activity-poll");
3885 self.poll_activity_task_response_with_request_id(
3886 worker_id,
3887 task_queue,
3888 timeout,
3889 &poll_request_id,
3890 1,
3891 )
3892 .await
3893 }
3894
3895 async fn poll_activity_task_response_with_request_id(
3896 &self,
3897 worker_id: &str,
3898 task_queue: &str,
3899 timeout: Duration,
3900 poll_request_id: &str,
3901 transport_retries: usize,
3902 ) -> Result<PollActivityTaskResponse> {
3903 let body = json!({
3904 "worker_id": worker_id,
3905 "task_queue": task_queue,
3906 "poll_request_id": poll_request_id,
3907 "timeout_seconds": long_poll_timeout_seconds(timeout),
3908 });
3909 let data: PollActivityTaskResponse = self
3910 .poll_request_json(
3911 "/worker/activity-tasks/poll",
3912 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3913 &body,
3914 timeout + Duration::from_secs(5),
3915 transport_retries,
3916 )
3917 .await?;
3918 Ok(data)
3919 }
3920
3921 pub async fn complete_activity_task<T: Serialize>(
3922 &self,
3923 task_id: &str,
3924 activity_attempt_id: &str,
3925 lease_owner: &str,
3926 result: T,
3927 codec: &str,
3928 ) -> Result<Value> {
3929 let result = encode_typed_envelope(&AvroValue::from_serialize(&result)?, codec)?;
3930 let body = json!({
3931 "activity_attempt_id": activity_attempt_id,
3932 "lease_owner": lease_owner,
3933 "result": result
3934 });
3935 let path = format!("/worker/activity-tasks/{task_id}/complete");
3936 activity_task_response(
3937 self.request_json(
3938 reqwest::Method::POST,
3939 &path,
3940 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3941 Some(&body),
3942 )
3943 .await,
3944 "complete",
3945 task_id,
3946 activity_attempt_id,
3947 )
3948 }
3949
3950 pub async fn fail_activity_task(
3951 &self,
3952 task_id: &str,
3953 activity_attempt_id: &str,
3954 lease_owner: &str,
3955 message: impl Into<String>,
3956 non_retryable: bool,
3957 ) -> Result<Value> {
3958 let body = json!({
3959 "activity_attempt_id": activity_attempt_id,
3960 "lease_owner": lease_owner,
3961 "failure": {
3962 "message": message.into(),
3963 "type": "RustActivityFailure",
3964 "non_retryable": non_retryable
3965 }
3966 });
3967 let path = format!("/worker/activity-tasks/{task_id}/fail");
3968 activity_task_response(
3969 self.request_json(
3970 reqwest::Method::POST,
3971 &path,
3972 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3973 Some(&body),
3974 )
3975 .await,
3976 "fail",
3977 task_id,
3978 activity_attempt_id,
3979 )
3980 }
3981
3982 pub async fn heartbeat_activity_task<T: Serialize>(
3983 &self,
3984 task_id: &str,
3985 activity_attempt_id: &str,
3986 lease_owner: &str,
3987 details: T,
3988 ) -> Result<ActivityHeartbeatResponse> {
3989 let details = encode_typed_envelope(&AvroValue::from_serialize(&details)?, DEFAULT_CODEC)?;
3990 let body = json!({
3991 "activity_attempt_id": activity_attempt_id,
3992 "lease_owner": lease_owner,
3993 "details": details
3994 });
3995 let path = format!("/worker/activity-tasks/{task_id}/heartbeat");
3996 activity_task_response(
3997 self.request_json(
3998 reqwest::Method::POST,
3999 &path,
4000 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
4001 Some(&body),
4002 )
4003 .await,
4004 "heartbeat",
4005 task_id,
4006 activity_attempt_id,
4007 )
4008 }
4009
4010 async fn request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
4011 &self,
4012 method: reqwest::Method,
4013 path: &str,
4014 protocol: RequestProtocol,
4015 body: Option<&B>,
4016 ) -> Result<T> {
4017 self.request_json_with_timeout(method, path, protocol, body, Duration::from_secs(60))
4018 .await
4019 }
4020
4021 async fn request_json_with_timeout<T: DeserializeOwned, B: Serialize + ?Sized>(
4022 &self,
4023 method: reqwest::Method,
4024 path: &str,
4025 protocol: RequestProtocol,
4026 body: Option<&B>,
4027 timeout: Duration,
4028 ) -> Result<T> {
4029 let auth_token = self.auth_token(protocol)?;
4030 let mut request = self
4031 .http
4032 .request(method, format!("{}/api{}", self.base_url, path))
4033 .timeout(timeout)
4034 .header(reqwest::header::ACCEPT, "application/json")
4035 .header(reqwest::header::CONTENT_TYPE, "application/json")
4036 .header("X-Namespace", &self.namespace);
4037
4038 match protocol {
4039 RequestProtocol::Worker(version) => {
4040 request = request.header("X-Durable-Workflow-Protocol-Version", version);
4041 }
4042 RequestProtocol::ControlPlane => {
4043 request = request.header(
4044 "X-Durable-Workflow-Control-Plane-Version",
4045 CONTROL_PLANE_VERSION,
4046 );
4047 }
4048 }
4049
4050 if let Some(token) = auth_token {
4051 request = request.bearer_auth(token);
4052 }
4053
4054 if let Some(body) = body {
4055 request = request.json(body);
4056 }
4057
4058 let response = request.send().await?;
4059 let status = response.status();
4060 let bytes = response.bytes().await?;
4061
4062 if !status.is_success() {
4063 let body = String::from_utf8_lossy(&bytes).to_string();
4064 if let Some(protocol) = protocol_failure(status, &body) {
4065 return Err(Error::Protocol(protocol));
4066 }
4067 return Err(Error::Http { status, body });
4068 }
4069
4070 if bytes.is_empty() {
4071 return Ok(serde_json::from_value(Value::Null)?);
4072 }
4073
4074 Ok(serde_json::from_slice(&bytes)?)
4075 }
4076
4077 async fn poll_request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
4078 &self,
4079 path: &str,
4080 protocol: RequestProtocol,
4081 body: &B,
4082 timeout: Duration,
4083 max_retries: usize,
4084 ) -> Result<T> {
4085 let mut retries = 0;
4086
4087 loop {
4088 let response = self
4089 .request_json_with_timeout(
4090 reqwest::Method::POST,
4091 path,
4092 protocol,
4093 Some(body),
4094 timeout,
4095 )
4096 .await;
4097
4098 match response {
4099 Err(Error::Transport(_)) if retries < max_retries => retries += 1,
4100 response => return worker_poll_response(response),
4101 }
4102 }
4103 }
4104
4105 fn auth_token(&self, protocol: RequestProtocol) -> Result<Option<&str>> {
4106 match protocol {
4107 RequestProtocol::Worker(_) => {
4108 if let Some(token) = self.worker_token.as_deref().or(self.token.as_deref()) {
4109 return Ok(Some(token));
4110 }
4111 if self.control_token.is_some() {
4112 return Err(Error::MissingRoleCredentials {
4113 role: "worker",
4114 opposite_role: "control",
4115 });
4116 }
4117 Ok(None)
4118 }
4119 RequestProtocol::ControlPlane => {
4120 if let Some(token) = self.control_token.as_deref().or(self.token.as_deref()) {
4121 return Ok(Some(token));
4122 }
4123 if self.worker_token.is_some() {
4124 return Err(Error::MissingRoleCredentials {
4125 role: "control",
4126 opposite_role: "worker",
4127 });
4128 }
4129 Ok(None)
4130 }
4131 }
4132 }
4133}
4134
4135fn query_failure(status: reqwest::StatusCode, raw_body: String) -> QueryFailure {
4136 let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
4137 let reason = body
4138 .get("reason")
4139 .and_then(Value::as_str)
4140 .unwrap_or("query_rejected")
4141 .to_string();
4142 let message = body
4143 .get("message")
4144 .or_else(|| body.get("error"))
4145 .and_then(Value::as_str)
4146 .unwrap_or("workflow query was rejected")
4147 .to_string();
4148
4149 QueryFailure {
4150 status: status.as_u16(),
4151 reason,
4152 message,
4153 body,
4154 }
4155}
4156
4157fn workflow_command_result(
4158 command: WorkflowCommandKind,
4159 data: Value,
4160 workflow_id: &str,
4161 run_id: Option<&str>,
4162) -> WorkflowCommandResult {
4163 WorkflowCommandResult {
4164 command,
4165 workflow_id: data
4166 .get("workflow_id")
4167 .and_then(Value::as_str)
4168 .unwrap_or(workflow_id)
4169 .to_string(),
4170 run_id: data
4171 .get("run_id")
4172 .and_then(Value::as_str)
4173 .or(run_id)
4174 .map(str::to_string),
4175 outcome: data
4176 .get("outcome")
4177 .and_then(Value::as_str)
4178 .map(str::to_string),
4179 reason: data
4180 .get("reason")
4181 .and_then(Value::as_str)
4182 .map(str::to_string),
4183 command_status: data
4184 .get("command_status")
4185 .and_then(Value::as_str)
4186 .map(str::to_string),
4187 raw: data,
4188 }
4189}
4190
4191fn workflow_command_rejection(
4192 command: WorkflowCommandKind,
4193 status: reqwest::StatusCode,
4194 raw_body: String,
4195 workflow_id: &str,
4196 run_id: Option<&str>,
4197) -> WorkflowCommandRejection {
4198 let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
4199 WorkflowCommandRejection {
4200 command,
4201 status: status.as_u16(),
4202 reason: body
4203 .get("reason")
4204 .and_then(Value::as_str)
4205 .unwrap_or("workflow_command_rejected")
4206 .to_string(),
4207 message: body
4208 .get("message")
4209 .or_else(|| body.get("error"))
4210 .and_then(Value::as_str)
4211 .unwrap_or("workflow lifecycle command was rejected")
4212 .to_string(),
4213 workflow_id: body
4214 .get("workflow_id")
4215 .and_then(Value::as_str)
4216 .unwrap_or(workflow_id)
4217 .to_string(),
4218 run_id: body
4219 .get("run_id")
4220 .and_then(Value::as_str)
4221 .or(run_id)
4222 .map(str::to_string),
4223 target_scope: body
4224 .get("target_scope")
4225 .and_then(Value::as_str)
4226 .map(str::to_string),
4227 body,
4228 }
4229}
4230
4231fn query_task_response(response: Result<Value>) -> Result<Value> {
4232 match response {
4233 Err(Error::Http { status, body }) => Err(Error::QueryFailed(query_failure(status, body))),
4234 response => response,
4235 }
4236}
4237
4238fn worker_poll_response<T: DeserializeOwned>(response: Result<T>) -> Result<T> {
4239 match response {
4240 Err(Error::Http { status, body })
4241 if status == reqwest::StatusCode::CONFLICT && worker_poll_body_is_stop(&body) =>
4242 {
4243 Ok(serde_json::from_str(&body)?)
4244 }
4245 response => response,
4246 }
4247}
4248
4249fn worker_poll_body_is_stop(body: &str) -> bool {
4250 serde_json::from_str::<Value>(body)
4251 .ok()
4252 .is_some_and(|body| {
4253 worker_poll_is_stop(
4254 body.get("poll_status").and_then(Value::as_str),
4255 body.get("reason").and_then(Value::as_str),
4256 )
4257 })
4258}
4259
4260fn worker_poll_is_stop(poll_status: Option<&str>, reason: Option<&str>) -> bool {
4261 matches!(poll_status, Some("draining" | "stopped"))
4262 || matches!(reason, Some("worker_draining" | "worker_stopped"))
4263}
4264
4265fn query_task_rejection_is_final(error: &Error) -> bool {
4266 matches!(
4267 error,
4268 Error::QueryFailed(failure)
4269 if QUERY_TASK_FINAL_REJECTION_REASONS.contains(&failure.reason.as_str())
4270 )
4271}
4272
4273fn activity_task_response<T>(
4274 response: Result<T>,
4275 operation: &str,
4276 task_id: &str,
4277 activity_attempt_id: &str,
4278) -> Result<T> {
4279 match response {
4280 Err(Error::Http { status, body }) => {
4281 let body = serde_json::from_str(&body).unwrap_or_else(|_| json!({"message": body}));
4282 Err(Error::ActivityTaskRejected(ActivityTaskRejection {
4283 operation: operation.to_string(),
4284 status: status.as_u16(),
4285 reason: body
4286 .get("reason")
4287 .and_then(Value::as_str)
4288 .unwrap_or("activity_task_rejected")
4289 .to_string(),
4290 task_id: body
4291 .get("task_id")
4292 .and_then(Value::as_str)
4293 .unwrap_or(task_id)
4294 .to_string(),
4295 activity_attempt_id: body
4296 .get("activity_attempt_id")
4297 .and_then(Value::as_str)
4298 .unwrap_or(activity_attempt_id)
4299 .to_string(),
4300 cancel_requested: body
4301 .get("cancel_requested")
4302 .and_then(Value::as_bool)
4303 .unwrap_or(false),
4304 can_continue: body.get("can_continue").and_then(Value::as_bool),
4305 run_closed_reason: body
4306 .get("run_closed_reason")
4307 .and_then(Value::as_str)
4308 .map(str::to_string),
4309 body,
4310 }))
4311 }
4312 response => response,
4313 }
4314}
4315
4316fn activity_task_rejection_is_final(error: &Error) -> bool {
4317 matches!(
4318 error,
4319 Error::ActivityTaskRejected(rejection)
4320 if matches!(
4321 rejection.reason.as_str(),
4322 "run_cancelled"
4323 | "run_terminated"
4324 | "attempt_closed"
4325 | "stale_attempt"
4326 | "activity_cancelled"
4327 | "task_cancelled"
4328 | "run_closed"
4329 | "activity_not_running"
4330 | "attempt_not_found"
4331 )
4332 )
4333}
4334
4335fn workflow_task_completion_is_terminal_timeout(
4336 error: &Error,
4337 task_id: &str,
4338 workflow_task_attempt: u64,
4339 run_id: Option<&str>,
4340) -> bool {
4341 let Error::Http { status, body } = error else {
4342 return false;
4343 };
4344 if *status != reqwest::StatusCode::CONFLICT {
4345 return false;
4346 }
4347
4348 let Some(run_id) = run_id else {
4349 return false;
4350 };
4351 let Ok(body) = serde_json::from_str::<Value>(body) else {
4352 return false;
4353 };
4354
4355 body.get("recorded").and_then(Value::as_bool) == Some(false)
4356 && body.get("reason").and_then(Value::as_str) == Some("run_timed_out")
4357 && body.get("run_status").and_then(Value::as_str) == Some("failed")
4358 && body.get("run_id").and_then(Value::as_str) == Some(run_id)
4359 && body.get("task_id").and_then(Value::as_str) == Some(task_id)
4360 && body.get("workflow_task_attempt").and_then(Value::as_u64) == Some(workflow_task_attempt)
4361}
4362
4363fn protocol_failure(status: reqwest::StatusCode, raw_body: &str) -> Option<ProtocolFailure> {
4364 let body: Value = serde_json::from_str(raw_body).ok()?;
4365 let reason = body.get("reason")?.as_str()?;
4366 if !matches!(
4367 reason,
4368 "missing_protocol_version"
4369 | "unsupported_protocol_version"
4370 | "missing_control_plane_version"
4371 | "unsupported_control_plane_version"
4372 ) {
4373 return None;
4374 }
4375
4376 Some(ProtocolFailure {
4377 status: status.as_u16(),
4378 reason: reason.to_string(),
4379 message: body
4380 .get("message")
4381 .or_else(|| body.get("error"))
4382 .and_then(Value::as_str)
4383 .unwrap_or("protocol version rejected")
4384 .to_string(),
4385 supported_version: body
4386 .get("supported_version")
4387 .and_then(Value::as_str)
4388 .map(str::to_string),
4389 requested_version: body
4390 .get("requested_version")
4391 .and_then(Value::as_str)
4392 .map(str::to_string),
4393 body,
4394 })
4395}
4396
4397fn long_poll_timeout_seconds(timeout: Duration) -> u64 {
4398 timeout
4399 .as_secs()
4400 .saturating_add(u64::from(timeout.subsec_nanos() > 0))
4401 .min(MAX_LONG_POLL_TIMEOUT_SECONDS)
4402}
4403
4404fn worker_operation_is_retryable(error: &Error) -> bool {
4405 match error {
4406 Error::Transport(error) => {
4407 error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()
4408 }
4409 Error::Http { status, .. } => {
4410 matches!(
4411 *status,
4412 reqwest::StatusCode::REQUEST_TIMEOUT | reqwest::StatusCode::TOO_MANY_REQUESTS
4413 ) || status.is_server_error()
4414 }
4415 _ => false,
4416 }
4417}
4418
4419fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration {
4420 let exponent = retry.saturating_sub(1).min(31) as u32;
4421 policy
4422 .initial_backoff
4423 .saturating_mul(1_u32 << exponent)
4424 .min(policy.max_backoff)
4425}
4426
4427#[derive(Debug)]
4428pub struct ClientBuilder {
4429 base_url: String,
4430 token: Option<String>,
4431 control_token: Option<String>,
4432 worker_token: Option<String>,
4433 namespace: String,
4434 timeout: Duration,
4435}
4436
4437impl ClientBuilder {
4438 pub fn token(mut self, token: Option<String>) -> Self {
4439 self.token = token;
4440 self
4441 }
4442
4443 pub fn control_token(mut self, token: Option<String>) -> Self {
4444 self.control_token = token;
4445 self
4446 }
4447
4448 pub fn worker_token(mut self, token: Option<String>) -> Self {
4449 self.worker_token = token;
4450 self
4451 }
4452
4453 pub fn namespace(mut self, namespace: impl Into<String>) -> Self {
4454 self.namespace = namespace.into();
4455 self
4456 }
4457
4458 pub fn timeout(mut self, timeout: Duration) -> Self {
4459 self.timeout = timeout;
4460 self
4461 }
4462
4463 pub fn build(self) -> Result<Client> {
4464 let base_url = self.base_url.trim_end_matches('/').to_string();
4465 let has_sdk_api_suffix = reqwest::Url::parse(&base_url)
4466 .map(|url| url.path().trim_end_matches('/').ends_with("/api"))
4467 .unwrap_or_else(|_| base_url.ends_with("/api"));
4468
4469 if has_sdk_api_suffix {
4470 return Err(Error::InvalidBaseUrl);
4471 }
4472
4473 Ok(Client {
4474 http: reqwest::Client::builder().timeout(self.timeout).build()?,
4475 base_url,
4476 token: self.token,
4477 control_token: self.control_token,
4478 worker_token: self.worker_token,
4479 namespace: self.namespace,
4480 })
4481 }
4482}
4483
4484#[derive(Clone, Debug)]
4485pub struct WorkflowHandle {
4486 client: Client,
4487 pub workflow_id: String,
4488 pub run_id: Option<String>,
4489 pub workflow_type: String,
4490}
4491
4492impl WorkflowHandle {
4493 pub async fn describe(&self) -> Result<WorkflowDescription> {
4495 self.client.describe_workflow(&self.workflow_id).await
4496 }
4497
4498 pub async fn describe_selected_run(&self) -> Result<WorkflowDescription> {
4500 let run_id = self.run_id.as_deref().ok_or_else(|| {
4501 Error::Codec("run_id is required for selected-run description".to_string())
4502 })?;
4503 self.client
4504 .describe_workflow_run(&self.workflow_id, run_id)
4505 .await
4506 }
4507
4508 pub async fn signal<T: Serialize>(&self, signal_name: &str, input: T) -> Result<Value> {
4509 self.client
4510 .signal_workflow(&self.workflow_id, signal_name, input)
4511 .await
4512 }
4513
4514 pub async fn append_message<T: Serialize>(
4515 &self,
4516 stream_name: &str,
4517 message_id: &str,
4518 input: T,
4519 ) -> Result<Value> {
4520 self.client
4521 .append_message_stream(&self.workflow_id, stream_name, message_id, input)
4522 .await
4523 }
4524
4525 pub async fn signal_selected_run<T: Serialize>(
4527 &self,
4528 signal_name: &str,
4529 input: T,
4530 ) -> Result<Value> {
4531 let run_id = self.run_id.as_deref().ok_or_else(|| {
4532 Error::Codec("run_id is required for selected-run signaling".to_string())
4533 })?;
4534 self.client
4535 .signal_workflow_run(&self.workflow_id, run_id, signal_name, input)
4536 .await
4537 }
4538
4539 pub async fn cancel(&self, options: WorkflowCommandOptions) -> Result<WorkflowCommandResult> {
4541 self.client
4542 .cancel_workflow(&self.workflow_id, options)
4543 .await
4544 }
4545
4546 pub async fn cancel_selected_run(
4548 &self,
4549 options: WorkflowCommandOptions,
4550 ) -> Result<WorkflowCommandResult> {
4551 let run_id = self.run_id.as_deref().ok_or_else(|| {
4552 Error::Codec("run_id is required for selected-run cancellation".to_string())
4553 })?;
4554 self.client
4555 .cancel_workflow_run(&self.workflow_id, run_id, options)
4556 .await
4557 }
4558
4559 pub async fn terminate(
4561 &self,
4562 options: WorkflowCommandOptions,
4563 ) -> Result<WorkflowCommandResult> {
4564 self.client
4565 .terminate_workflow(&self.workflow_id, options)
4566 .await
4567 }
4568
4569 pub async fn terminate_selected_run(
4571 &self,
4572 options: WorkflowCommandOptions,
4573 ) -> Result<WorkflowCommandResult> {
4574 let run_id = self.run_id.as_deref().ok_or_else(|| {
4575 Error::Codec("run_id is required for selected-run termination".to_string())
4576 })?;
4577 self.client
4578 .terminate_workflow_run(&self.workflow_id, run_id, options)
4579 .await
4580 }
4581
4582 pub async fn query<T: Serialize>(&self, query_name: &str, input: T) -> Result<Value> {
4584 self.client
4585 .query_workflow(&self.workflow_id, query_name, input)
4586 .await
4587 }
4588
4589 pub async fn query_avro_value<T: Serialize>(
4590 &self,
4591 query_name: &str,
4592 input: T,
4593 ) -> Result<AvroValue> {
4594 self.client
4595 .query_workflow_avro_value(&self.workflow_id, query_name, input)
4596 .await
4597 }
4598
4599 pub async fn update<T: Serialize>(
4600 &self,
4601 update_name: &str,
4602 input: T,
4603 request_id: Option<&str>,
4604 ) -> Result<Value> {
4605 self.client
4606 .update_workflow(&self.workflow_id, update_name, input, request_id)
4607 .await
4608 }
4609
4610 pub async fn update_avro_value<T: Serialize>(
4611 &self,
4612 update_name: &str,
4613 input: T,
4614 request_id: Option<&str>,
4615 ) -> Result<AvroValue> {
4616 self.client
4617 .update_workflow_avro_value(&self.workflow_id, update_name, input, request_id)
4618 .await
4619 }
4620
4621 pub async fn query_selected_run<T: Serialize>(
4623 &self,
4624 query_name: &str,
4625 input: T,
4626 ) -> Result<Value> {
4627 let run_id = self
4628 .run_id
4629 .as_deref()
4630 .ok_or_else(|| Error::Codec("run_id is required for selected-run query".to_string()))?;
4631 self.client
4632 .query_workflow_run(&self.workflow_id, run_id, query_name, input)
4633 .await
4634 }
4635
4636 pub async fn result(&self, options: WorkflowResultOptions) -> Result<Value> {
4638 self.result_target(options, None).await
4639 }
4640
4641 pub async fn result_avro_value(&self, options: WorkflowResultOptions) -> Result<AvroValue> {
4643 self.result_avro_value_target(options, None).await
4644 }
4645
4646 pub async fn result_typed<T: DeserializeOwned>(
4648 &self,
4649 options: WorkflowResultOptions,
4650 ) -> Result<T> {
4651 let result = self.result_avro_value(options).await?;
4652 decode_handler_result(result, HandlerKind::Workflow, &self.workflow_type)
4653 }
4654
4655 pub async fn result_selected_run(&self, options: WorkflowResultOptions) -> Result<Value> {
4657 let run_id = self.run_id.as_deref().ok_or_else(|| {
4658 Error::Codec("run_id is required for selected-run result".to_string())
4659 })?;
4660 self.result_target(options, Some(run_id)).await
4661 }
4662
4663 pub async fn result_selected_run_avro_value(
4665 &self,
4666 options: WorkflowResultOptions,
4667 ) -> Result<AvroValue> {
4668 let run_id = self.run_id.as_deref().ok_or_else(|| {
4669 Error::Codec("run_id is required for selected-run result".to_string())
4670 })?;
4671 self.result_avro_value_target(options, Some(run_id)).await
4672 }
4673
4674 pub async fn result_selected_run_typed<T: DeserializeOwned>(
4676 &self,
4677 options: WorkflowResultOptions,
4678 ) -> Result<T> {
4679 let result = self.result_selected_run_avro_value(options).await?;
4680 decode_handler_result(result, HandlerKind::Workflow, &self.workflow_type)
4681 }
4682
4683 async fn result_avro_value_target(
4684 &self,
4685 options: WorkflowResultOptions,
4686 selected_run_id: Option<&str>,
4687 ) -> Result<AvroValue> {
4688 let started = Instant::now();
4689
4690 loop {
4691 let description = match selected_run_id {
4692 Some(run_id) => {
4693 self.client
4694 .describe_workflow_run(&self.workflow_id, run_id)
4695 .await?
4696 }
4697 None => self.describe().await?,
4698 };
4699 if description.is_completed() {
4700 return description.output_avro_value.ok_or_else(|| {
4701 Error::Codec(
4702 "missing_payload_envelope: typed workflow result requires output_envelope"
4703 .to_string(),
4704 )
4705 });
4706 }
4707 if description.is_terminal() {
4708 let outcome =
4709 workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
4710 return Err(match outcome.kind {
4711 WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
4712 WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
4713 WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
4714 WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
4715 });
4716 }
4717 if started.elapsed() >= options.timeout {
4718 return Err(Error::Timeout);
4719 }
4720 tokio::time::sleep(options.poll_interval).await;
4721 }
4722 }
4723
4724 async fn result_target(
4725 &self,
4726 options: WorkflowResultOptions,
4727 selected_run_id: Option<&str>,
4728 ) -> Result<Value> {
4729 let started = Instant::now();
4730
4731 loop {
4732 let description = match selected_run_id {
4733 Some(run_id) => {
4734 self.client
4735 .describe_workflow_run(&self.workflow_id, run_id)
4736 .await?
4737 }
4738 None => self.describe().await?,
4739 };
4740 if description.is_completed() {
4741 return Ok(description.output.unwrap_or(Value::Null));
4742 }
4743
4744 if description.is_terminal() {
4745 let outcome =
4746 workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
4747 return Err(match outcome.kind {
4748 WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
4749 WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
4750 WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
4751 WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
4752 });
4753 }
4754
4755 if started.elapsed() >= options.timeout {
4756 return Err(Error::WorkflowTimedOut(WorkflowTerminalOutcome {
4757 kind: WorkflowTerminalKind::TimedOut,
4758 workflow_id: description
4759 .workflow_id
4760 .clone()
4761 .unwrap_or_else(|| self.workflow_id.clone()),
4762 run_id: description
4763 .run_id
4764 .clone()
4765 .or_else(|| selected_run_id.map(str::to_string)),
4766 reason: "result_wait_timeout".to_string(),
4767 failure_category: Some("client_timeout".to_string()),
4768 failure_id: None,
4769 exception_type: None,
4770 exception_class: None,
4771 non_retryable: None,
4772 message: Some(format!(
4773 "workflow result was not terminal within {:?}",
4774 options.timeout
4775 )),
4776 exception: None,
4777 raw: description.raw_value(),
4778 }));
4779 }
4780
4781 tokio::time::sleep(options.poll_interval).await;
4782 }
4783 }
4784}
4785
4786#[derive(Clone, Copy, Debug)]
4787pub struct WorkflowResultOptions {
4788 pub poll_interval: Duration,
4789 pub timeout: Duration,
4790}
4791
4792impl Default for WorkflowResultOptions {
4793 fn default() -> Self {
4794 Self {
4795 poll_interval: Duration::from_millis(500),
4796 timeout: Duration::from_secs(30),
4797 }
4798 }
4799}
4800
4801#[derive(Clone, Debug, Deserialize)]
4802pub struct WorkflowDescription {
4803 pub workflow_id: Option<String>,
4804 pub run_id: Option<String>,
4805 pub workflow_type: Option<String>,
4806 pub status: Option<String>,
4807 #[serde(default)]
4808 pub closed_reason: Option<String>,
4809 #[serde(default)]
4810 pub error: Option<String>,
4811 #[serde(default)]
4812 pub failure: Option<Value>,
4813 #[serde(default)]
4814 pub exception: Option<Value>,
4815 #[serde(default)]
4816 pub failures: Vec<Value>,
4817 #[serde(default)]
4818 pub output: Option<Value>,
4819 #[serde(default)]
4820 pub output_envelope: Option<Value>,
4821 #[serde(skip)]
4822 pub output_avro_value: Option<AvroValue>,
4823 #[serde(flatten)]
4824 pub raw: HashMap<String, Value>,
4825}
4826
4827#[derive(Clone, Debug, Deserialize)]
4829pub struct WorkflowStreamDescription {
4830 pub stream_name: String,
4831 pub status: String,
4832 pub last_offset: i64,
4833 pub total_items: u64,
4834 pub pending_items: u64,
4835 #[serde(default)]
4836 pub opened_at: Option<String>,
4837 #[serde(default)]
4838 pub last_appended_at: Option<String>,
4839 #[serde(default)]
4840 pub closed_at: Option<String>,
4841 #[serde(default)]
4842 pub error_reason: Option<String>,
4843 #[serde(default)]
4844 pub retention_seconds: Option<u64>,
4845 #[serde(flatten)]
4846 pub raw: HashMap<String, Value>,
4847}
4848
4849impl WorkflowStreamDescription {
4850 pub fn is_terminal(&self) -> bool {
4851 matches!(self.status.as_str(), "closed" | "errored")
4852 }
4853}
4854
4855#[derive(Clone, Debug, Default)]
4857pub struct WorkflowStreamAppendItem {
4858 pub payload_envelope: Option<Value>,
4859 pub payload_reference: Option<String>,
4860 pub item_type: Option<String>,
4861 pub content_type: Option<String>,
4862 pub idempotency_key: Option<String>,
4863}
4864
4865impl WorkflowStreamAppendItem {
4866 pub fn new<T: Serialize>(payload: T) -> Result<Self> {
4868 let value = AvroValue::from_serialize(&payload)?;
4869 Ok(Self {
4870 payload_envelope: Some(encode_typed_envelope(&value, DEFAULT_CODEC)?),
4871 ..Self::default()
4872 })
4873 }
4874
4875 pub fn from_reference(reference: impl Into<String>) -> Self {
4877 Self {
4878 payload_reference: Some(reference.into()),
4879 ..Self::default()
4880 }
4881 }
4882
4883 pub fn item_type(mut self, item_type: impl Into<String>) -> Self {
4884 self.item_type = Some(item_type.into());
4885 self
4886 }
4887
4888 pub fn content_type(mut self, content_type: impl Into<String>) -> Self {
4889 self.content_type = Some(content_type.into());
4890 self
4891 }
4892
4893 pub fn idempotency_key(mut self, idempotency_key: impl Into<String>) -> Self {
4894 self.idempotency_key = Some(idempotency_key.into());
4895 self
4896 }
4897
4898 fn wire_value(&self, derived_idempotency_key: Option<String>) -> Value {
4899 let mut item = serde_json::Map::new();
4900 if let Some(payload) = &self.payload_envelope {
4901 item.insert("payload".to_string(), payload.clone());
4902 item.insert("payload_codec".to_string(), json!(DEFAULT_CODEC));
4903 }
4904 if let Some(reference) = &self.payload_reference {
4905 item.insert("payload_reference".to_string(), json!(reference));
4906 }
4907 if let Some(item_type) = &self.item_type {
4908 item.insert("item_type".to_string(), json!(item_type));
4909 }
4910 if let Some(content_type) = &self.content_type {
4911 item.insert("content_type".to_string(), json!(content_type));
4912 }
4913 if let Some(key) = derived_idempotency_key
4914 .as_ref()
4915 .or(self.idempotency_key.as_ref())
4916 {
4917 item.insert("idempotency_key".to_string(), json!(key));
4918 }
4919 Value::Object(item)
4920 }
4921}
4922
4923#[derive(Clone, Debug)]
4925pub struct WorkflowStreamItem {
4926 pub offset: u64,
4927 pub payload: Option<Value>,
4928 pub payload_envelope: Option<Value>,
4929 pub payload_reference: Option<String>,
4930 pub payload_codec: Option<String>,
4931 pub idempotency_key: Option<String>,
4932 pub item_type: Option<String>,
4933 pub content_type: Option<String>,
4934 pub origin: Option<String>,
4935 pub origin_reference: Option<String>,
4936 pub emitted_at: Option<String>,
4937 pub raw: Value,
4938}
4939
4940#[derive(Clone, Debug)]
4942pub struct WorkflowStreamPage {
4943 pub stream: WorkflowStreamDescription,
4944 pub items: Vec<WorkflowStreamItem>,
4945 pub next_offset: u64,
4946 pub terminal: bool,
4947}
4948
4949#[derive(Clone, Debug)]
4951pub struct WorkflowStreamAppendResult {
4952 pub stream: WorkflowStreamDescription,
4953 pub accepted_offsets: Vec<u64>,
4954 pub accepted: u64,
4955 pub deduped: u64,
4956}
4957
4958#[derive(Deserialize)]
4959struct WorkflowStreamListResponse {
4960 #[serde(default)]
4961 streams: Vec<WorkflowStreamDescription>,
4962}
4963
4964#[derive(Deserialize)]
4965struct WorkflowStreamDescriptionResponse {
4966 stream: WorkflowStreamDescription,
4967}
4968
4969#[derive(Deserialize)]
4970struct WorkflowStreamPageResponse {
4971 stream: WorkflowStreamDescription,
4972 #[serde(default)]
4973 items: Vec<Value>,
4974 next_offset: u64,
4975 terminal: bool,
4976}
4977
4978#[derive(Deserialize)]
4979struct WorkflowStreamAppendResponse {
4980 stream: WorkflowStreamDescription,
4981 #[serde(default)]
4982 accepted_offsets: Vec<u64>,
4983 accepted: u64,
4984 deduped: u64,
4985}
4986
4987impl WorkflowDescription {
4988 pub fn is_completed(&self) -> bool {
4989 matches!(self.status.as_deref(), Some("completed" | "Completed"))
4990 }
4991
4992 pub fn is_terminal(&self) -> bool {
4993 matches!(
4994 self.status.as_deref(),
4995 Some(
4996 "completed"
4997 | "Completed"
4998 | "failed"
4999 | "Failed"
5000 | "cancelled"
5001 | "Cancelled"
5002 | "terminated"
5003 | "Terminated"
5004 | "timed_out"
5005 | "TimedOut",
5006 )
5007 )
5008 }
5009
5010 fn decode_payloads(&mut self) -> Result<()> {
5011 if let Some(envelope) = &self.output_envelope {
5012 let value = decode_wire_avro_value(envelope, DEFAULT_CODEC)?;
5013 self.output = Some(value.clone().into_json()?);
5014 self.output_avro_value = Some(value);
5015 }
5016
5017 Ok(())
5018 }
5019
5020 fn raw_value(&self) -> Value {
5021 let mut data = self.raw.clone();
5022 data.insert(
5023 "workflow_id".to_string(),
5024 self.workflow_id
5025 .clone()
5026 .map(Value::String)
5027 .unwrap_or(Value::Null),
5028 );
5029 data.insert(
5030 "run_id".to_string(),
5031 self.run_id
5032 .clone()
5033 .map(Value::String)
5034 .unwrap_or(Value::Null),
5035 );
5036 data.insert(
5037 "workflow_type".to_string(),
5038 self.workflow_type
5039 .clone()
5040 .map(Value::String)
5041 .unwrap_or(Value::Null),
5042 );
5043 data.insert(
5044 "status".to_string(),
5045 self.status
5046 .clone()
5047 .map(Value::String)
5048 .unwrap_or(Value::Null),
5049 );
5050 data.insert(
5051 "closed_reason".to_string(),
5052 self.closed_reason
5053 .clone()
5054 .map(Value::String)
5055 .unwrap_or(Value::Null),
5056 );
5057 if let Some(failure) = &self.failure {
5058 data.insert("failure".to_string(), failure.clone());
5059 }
5060 if let Some(exception) = &self.exception {
5061 data.insert("exception".to_string(), exception.clone());
5062 }
5063 Value::Object(data.into_iter().collect())
5064 }
5065}
5066
5067fn workflow_terminal_outcome(
5068 description: &WorkflowDescription,
5069 workflow_id: &str,
5070 run_id: Option<&str>,
5071) -> WorkflowTerminalOutcome {
5072 let terminal_kind = description
5073 .closed_reason
5074 .as_deref()
5075 .or(description.status.as_deref())
5076 .unwrap_or("failed")
5077 .to_ascii_lowercase();
5078 let kind = match terminal_kind.as_str() {
5079 "cancelled" | "canceled" => WorkflowTerminalKind::Cancelled,
5080 "terminated" => WorkflowTerminalKind::Terminated,
5081 "timed_out" | "timedout" => WorkflowTerminalKind::TimedOut,
5082 _ => WorkflowTerminalKind::Failed,
5083 };
5084 let default_reason = match kind {
5085 WorkflowTerminalKind::Failed => "workflow_failed",
5086 WorkflowTerminalKind::Cancelled => "cancelled",
5087 WorkflowTerminalKind::Terminated => "terminated",
5088 WorkflowTerminalKind::TimedOut => "timed_out",
5089 };
5090 let failure = description
5091 .failure
5092 .as_ref()
5093 .filter(|value| value.is_object());
5094 let nested_failure = failure
5095 .and_then(|value| value.get("failures"))
5096 .and_then(Value::as_array)
5097 .and_then(|failures| failures.last())
5098 .or_else(|| description.failures.last());
5099 let exception = description
5100 .exception
5101 .clone()
5102 .or_else(|| failure.and_then(|value| value.get("exception")).cloned())
5103 .or_else(|| {
5104 nested_failure
5105 .and_then(|value| value.get("exception_payload"))
5106 .cloned()
5107 });
5108 let string_field = |name: &str| {
5109 failure
5110 .and_then(|value| value.get(name))
5111 .and_then(Value::as_str)
5112 .or_else(|| {
5113 nested_failure
5114 .and_then(|value| value.get(name))
5115 .and_then(Value::as_str)
5116 })
5117 .map(str::to_string)
5118 };
5119 let exception_field = |name: &str| {
5120 exception
5121 .as_ref()
5122 .and_then(|value| value.get(name))
5123 .and_then(Value::as_str)
5124 .map(str::to_string)
5125 };
5126 let message = description
5127 .error
5128 .clone()
5129 .or_else(|| string_field("message"))
5130 .or_else(|| exception_field("message"));
5131 let reason = description
5132 .raw
5133 .get("reason")
5134 .and_then(Value::as_str)
5135 .map(str::to_string)
5136 .or_else(|| {
5137 failure
5138 .and_then(|value| value.get("reason"))
5139 .and_then(Value::as_str)
5140 .map(str::to_string)
5141 })
5142 .or_else(|| description.closed_reason.clone())
5143 .unwrap_or_else(|| default_reason.to_string());
5144 let failure_id = string_field("failure_id").or_else(|| {
5145 nested_failure
5146 .and_then(|value| value.get("id"))
5147 .and_then(Value::as_str)
5148 .map(str::to_string)
5149 });
5150
5151 WorkflowTerminalOutcome {
5152 kind,
5153 workflow_id: description
5154 .workflow_id
5155 .clone()
5156 .unwrap_or_else(|| workflow_id.to_string()),
5157 run_id: description
5158 .run_id
5159 .clone()
5160 .or_else(|| run_id.map(str::to_string)),
5161 reason,
5162 failure_category: string_field("failure_category")
5163 .or_else(|| Some(default_reason.to_string())),
5164 failure_id,
5165 exception_type: string_field("exception_type").or_else(|| exception_field("type")),
5166 exception_class: string_field("exception_class").or_else(|| exception_field("class")),
5167 non_retryable: failure
5168 .and_then(|value| value.get("non_retryable"))
5169 .and_then(Value::as_bool)
5170 .or_else(|| {
5171 nested_failure
5172 .and_then(|value| value.get("non_retryable"))
5173 .and_then(Value::as_bool)
5174 }),
5175 message,
5176 exception,
5177 raw: description.raw_value(),
5178 }
5179}
5180
5181#[derive(Clone, Debug, Deserialize)]
5182pub struct RegisterWorkerResponse {
5183 pub worker_id: String,
5184 pub registered: bool,
5185 #[serde(default)]
5186 pub heartbeat_interval_seconds: Option<u64>,
5187 #[serde(default)]
5188 pub protocol_version: Option<String>,
5189 #[serde(default)]
5190 pub server_capabilities: Option<Value>,
5191}
5192
5193#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
5195pub struct WorkerDeregistrationEnvelope {
5196 pub worker_id: String,
5197 pub outcome: String,
5198 pub recovered_workflow_task_count: u64,
5199}
5200
5201#[derive(Clone, Debug, Deserialize)]
5202pub struct PollWorkflowTaskResponse {
5203 #[serde(default)]
5204 pub task: Option<WorkflowTask>,
5205 #[serde(default)]
5206 pub poll_status: Option<String>,
5207 #[serde(default)]
5208 pub reason: Option<String>,
5209 #[serde(default)]
5210 pub protocol_version: Option<String>,
5211 #[serde(default)]
5212 pub server_capabilities: Option<Value>,
5213}
5214
5215impl PollWorkflowTaskResponse {
5216 pub fn outcome(&self) -> WorkerPollOutcome {
5218 worker_poll_outcome(
5219 self.task.is_some(),
5220 self.poll_status.as_deref(),
5221 self.reason.as_deref(),
5222 )
5223 }
5224}
5225
5226fn runtime_supports_workflow_memo_updates(capabilities: Option<&Value>) -> bool {
5227 let Some(capabilities) = capabilities.and_then(Value::as_object) else {
5228 return false;
5229 };
5230 let supported = capabilities
5231 .get("workflow_memo_updates")
5232 .and_then(Value::as_object)
5233 .and_then(|memo| memo.get("supported"))
5234 .and_then(Value::as_bool)
5235 == Some(true);
5236 let command_advertised = capabilities
5237 .get("supported_workflow_task_commands")
5238 .and_then(Value::as_array)
5239 .is_some_and(|commands| {
5240 commands
5241 .iter()
5242 .any(|command| command.as_str() == Some("upsert_memo"))
5243 });
5244 supported && command_advertised
5245}
5246
5247fn commands_use_workflow_memo_updates(commands: &[Value]) -> bool {
5248 commands
5249 .iter()
5250 .any(|command| command.get("type").and_then(Value::as_str) == Some("upsert_memo"))
5251}
5252
5253#[derive(Clone, Debug, Deserialize)]
5254pub struct PollActivityTaskResponse {
5255 #[serde(default)]
5256 pub task: Option<ActivityTask>,
5257 #[serde(default)]
5258 pub poll_status: Option<String>,
5259 #[serde(default)]
5260 pub reason: Option<String>,
5261}
5262
5263impl PollActivityTaskResponse {
5264 pub fn outcome(&self) -> WorkerPollOutcome {
5266 worker_poll_outcome(
5267 self.task.is_some(),
5268 self.poll_status.as_deref(),
5269 self.reason.as_deref(),
5270 )
5271 }
5272}
5273
5274#[derive(Clone, Debug, Deserialize)]
5275pub struct PollQueryTaskResponse {
5276 #[serde(default)]
5277 pub task: Option<QueryTask>,
5278 #[serde(default)]
5279 pub poll_status: Option<String>,
5280 #[serde(default)]
5281 pub reason: Option<String>,
5282}
5283
5284impl PollQueryTaskResponse {
5285 pub fn outcome(&self) -> WorkerPollOutcome {
5287 worker_poll_outcome(
5288 self.task.is_some(),
5289 self.poll_status.as_deref(),
5290 self.reason.as_deref(),
5291 )
5292 }
5293}
5294
5295#[derive(Clone, Debug, PartialEq, Eq)]
5297pub enum WorkerPollOutcome {
5298 Task,
5300 Idle {
5302 poll_status: Option<String>,
5303 reason: Option<String>,
5304 },
5305 Stop {
5307 poll_status: Option<String>,
5308 reason: Option<String>,
5309 },
5310}
5311
5312impl WorkerPollOutcome {
5313 pub fn should_stop(&self) -> bool {
5314 matches!(self, Self::Stop { .. })
5315 }
5316}
5317
5318fn worker_poll_outcome(
5319 has_task: bool,
5320 poll_status: Option<&str>,
5321 reason: Option<&str>,
5322) -> WorkerPollOutcome {
5323 if worker_poll_is_stop(poll_status, reason) {
5324 return WorkerPollOutcome::Stop {
5325 poll_status: poll_status.map(str::to_string),
5326 reason: reason.map(str::to_string),
5327 };
5328 }
5329
5330 if has_task {
5331 WorkerPollOutcome::Task
5332 } else {
5333 WorkerPollOutcome::Idle {
5334 poll_status: poll_status.map(str::to_string),
5335 reason: reason.map(str::to_string),
5336 }
5337 }
5338}
5339
5340#[derive(Clone, Debug, Deserialize)]
5342pub struct QueryTask {
5343 pub query_task_id: String,
5344 #[serde(default = "default_workflow_task_attempt")]
5345 pub query_task_attempt: u64,
5346 #[serde(default)]
5347 pub lease_owner: Option<String>,
5348 #[serde(default)]
5349 pub workflow_id: Option<String>,
5350 #[serde(default)]
5351 pub run_id: Option<String>,
5352 pub workflow_type: String,
5353 pub query_name: String,
5354 #[serde(
5355 default = "missing_task_payload_codec",
5356 deserialize_with = "deserialize_task_payload_codec"
5357 )]
5358 pub payload_codec: String,
5359 #[serde(default)]
5360 pub workflow_arguments: Option<Value>,
5361 #[serde(default)]
5362 pub query_arguments: Option<Value>,
5363 #[serde(default)]
5364 pub history_events: Vec<HistoryEvent>,
5365 #[serde(default)]
5366 pub history_export: Option<Value>,
5367 #[serde(default)]
5368 pub run_status: Option<String>,
5369}
5370
5371#[derive(Clone, Debug, Deserialize)]
5372pub struct WorkflowTask {
5373 pub task_id: String,
5374 #[serde(default)]
5375 pub workflow_command_id: Option<String>,
5376 #[serde(default)]
5377 pub workflow_id: Option<String>,
5378 #[serde(default)]
5379 pub run_id: Option<String>,
5380 pub workflow_type: String,
5381 #[serde(default)]
5382 pub cancel_requested: bool,
5383 #[serde(
5384 default = "missing_task_payload_codec",
5385 deserialize_with = "deserialize_task_payload_codec"
5386 )]
5387 pub payload_codec: String,
5388 #[serde(default)]
5389 pub arguments: Option<Value>,
5390 #[serde(default)]
5391 pub history_events: Vec<HistoryEvent>,
5392 #[serde(default)]
5393 pub total_history_events: Option<u64>,
5394 #[serde(default)]
5395 pub history_size_bytes: Option<u64>,
5396 #[serde(default)]
5397 pub continue_as_new_recommended: Option<bool>,
5398 #[serde(default)]
5399 pub history_budget_pressure: Option<String>,
5400 #[serde(default)]
5401 pub next_history_page_token: Option<String>,
5402 #[serde(default = "default_workflow_task_attempt")]
5403 pub workflow_task_attempt: u64,
5404 #[serde(default)]
5405 pub workflow_signal_id: Option<String>,
5406 #[serde(default)]
5407 pub signal_name: Option<String>,
5408 #[serde(default)]
5409 pub signal_arguments: Option<Value>,
5410 #[serde(default)]
5411 pub workflow_update_id: Option<String>,
5412 #[serde(default)]
5413 pub update_name: Option<String>,
5414 #[serde(default)]
5415 pub lease_owner: Option<String>,
5416}
5417
5418impl WorkflowTask {
5419 fn append_history_page(&mut self, page: WorkflowTaskHistoryPage) {
5420 self.history_events.extend(page.history_events);
5421
5422 if page.total_history_events.is_some() {
5423 self.total_history_events = page.total_history_events;
5424 }
5425
5426 self.next_history_page_token = page
5427 .next_history_page_token
5428 .filter(|token| !token.is_empty());
5429 }
5430}
5431
5432#[derive(Clone, Debug, Deserialize)]
5433struct WorkflowTaskHistoryPage {
5434 #[serde(default)]
5435 history_events: Vec<HistoryEvent>,
5436 #[serde(default)]
5437 total_history_events: Option<u64>,
5438 #[serde(default)]
5439 next_history_page_token: Option<String>,
5440}
5441
5442#[derive(Clone, Debug, Deserialize)]
5443pub struct ActivityTask {
5444 pub task_id: String,
5445 #[serde(default)]
5446 pub activity_attempt_id: Option<String>,
5447 #[serde(default)]
5448 pub attempt_id: Option<String>,
5449 pub activity_type: String,
5450 #[serde(
5451 default = "missing_task_payload_codec",
5452 deserialize_with = "deserialize_task_payload_codec"
5453 )]
5454 pub payload_codec: String,
5455 #[serde(default)]
5456 pub arguments: Option<Value>,
5457 #[serde(default = "default_attempt_number")]
5458 pub attempt_number: u64,
5459 #[serde(default)]
5460 pub lease_owner: Option<String>,
5461}
5462
5463#[derive(Clone, Debug, Deserialize)]
5464pub struct HistoryEvent {
5465 #[serde(alias = "type")]
5466 pub event_type: String,
5467 #[serde(default)]
5468 pub payload: Value,
5469 #[serde(flatten)]
5470 pub raw: HashMap<String, Value>,
5471}
5472
5473#[derive(Clone, Debug, PartialEq)]
5475pub struct QuerySignal {
5476 pub id: Option<String>,
5477 pub name: String,
5478 pub arguments: Vec<Value>,
5479 avro_arguments: Vec<AvroValue>,
5480 pub workflow_sequence: Option<u64>,
5481}
5482
5483impl QuerySignal {
5484 pub fn arguments_avro_value(&self) -> &[AvroValue] {
5486 &self.avro_arguments
5487 }
5488}
5489
5490#[derive(Clone, Debug)]
5496pub struct QueryContext {
5497 pub workflow_id: Option<String>,
5498 pub run_id: Option<String>,
5499 pub workflow_type: String,
5500 pub run_status: Option<String>,
5501 workflow_input: Value,
5502 workflow_input_avro_value: AvroValue,
5503 history_events: Arc<Vec<HistoryEvent>>,
5504 signal_events: Arc<Vec<QuerySignal>>,
5505}
5506
5507impl QueryContext {
5508 pub fn workflow_input(&self) -> &Value {
5510 &self.workflow_input
5511 }
5512
5513 pub fn workflow_input_avro_value(&self) -> &AvroValue {
5515 &self.workflow_input_avro_value
5516 }
5517
5518 pub fn history_events(&self) -> &[HistoryEvent] {
5520 self.history_events.as_slice()
5521 }
5522
5523 pub fn signal_events(&self) -> &[QuerySignal] {
5525 self.signal_events.as_slice()
5526 }
5527
5528 pub fn signals(&self, signal_name: &str) -> Vec<Vec<Value>> {
5530 self.signal_events
5531 .iter()
5532 .filter(|signal| signal.name == signal_name)
5533 .map(|signal| signal.arguments.clone())
5534 .collect()
5535 }
5536
5537 pub fn signals_avro_value(&self, signal_name: &str) -> Vec<Vec<AvroValue>> {
5539 self.signal_events
5540 .iter()
5541 .filter(|signal| signal.name == signal_name)
5542 .map(|signal| signal.avro_arguments.clone())
5543 .collect()
5544 }
5545}
5546
5547#[derive(Clone, Debug, Deserialize)]
5548pub struct ActivityHeartbeatResponse {
5549 #[serde(default)]
5550 pub cancel_requested: bool,
5551 #[serde(default)]
5552 pub heartbeat_recorded: bool,
5553 #[serde(default)]
5554 pub can_continue: Option<bool>,
5555 #[serde(default)]
5556 pub reason: Option<String>,
5557 #[serde(default)]
5558 pub run_closed_reason: Option<String>,
5559 #[serde(default)]
5560 pub run_closed_at: Option<String>,
5561 #[serde(default)]
5562 pub lease_expires_at: Option<String>,
5563 #[serde(default)]
5564 pub last_heartbeat_at: Option<String>,
5565}
5566
5567impl ActivityHeartbeatResponse {
5568 pub fn should_stop(&self) -> bool {
5570 self.cancel_requested || self.can_continue == Some(false)
5571 }
5572}
5573
5574fn missing_task_payload_codec() -> String {
5575 MISSING_TASK_PAYLOAD_CODEC.to_string()
5576}
5577
5578fn deserialize_task_payload_codec<'de, D>(deserializer: D) -> std::result::Result<String, D::Error>
5579where
5580 D: Deserializer<'de>,
5581{
5582 Ok(match Value::deserialize(deserializer)? {
5583 Value::String(codec) => codec,
5584 Value::Null => NULL_TASK_PAYLOAD_CODEC.to_string(),
5585 _ => NON_STRING_TASK_PAYLOAD_CODEC.to_string(),
5586 })
5587}
5588
5589fn default_workflow_task_attempt() -> u64 {
5590 1
5591}
5592
5593fn default_attempt_number() -> u64 {
5594 1
5595}
5596
5597type WorkflowFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5598type WorkflowHandler = Arc<dyn Fn(WorkflowContext, AvroValue) -> WorkflowFuture + Send + Sync>;
5599type ErasedWorkflowState = Arc<dyn Any + Send + Sync>;
5600type WorkflowStateSnapshot = Arc<dyn Fn() -> Result<ErasedWorkflowState> + Send + Sync>;
5601type ReplayedWorkflowHandler =
5602 Arc<dyn Fn(WorkflowContext, AvroValue) -> ReplayedWorkflowInvocation + Send + Sync>;
5603type ActivityFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5604type ActivityHandler = Arc<dyn Fn(ActivityContext, AvroValue) -> ActivityFuture + Send + Sync>;
5605type QueryFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5606type QueryHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
5607type UpdateHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
5608type ReplayedQueryHandler = Arc<
5609 dyn Fn(QueryContext, ErasedWorkflowState, AvroValue) -> std::result::Result<QueryFuture, String>
5610 + Send
5611 + Sync,
5612>;
5613type WorkerHeartbeatObserver = Arc<dyn Fn(&WorkerHeartbeatObservation) + Send + Sync>;
5614
5615struct ReplayedWorkflowInvocation {
5616 future: WorkflowFuture,
5617 snapshot: WorkflowStateSnapshot,
5618}
5619
5620#[derive(Clone)]
5621struct RegisteredWorkflow {
5622 execute: WorkflowHandler,
5623 replay: Option<ReplayedWorkflowHandler>,
5624 state_type: Option<TypeId>,
5625}
5626
5627#[derive(Debug)]
5628struct WorkflowTaskDecision {
5629 commands: Vec<Value>,
5630 message_stream_cursors: Vec<Value>,
5631 message_stream_waits: Vec<Value>,
5632}
5633
5634impl WorkflowTaskDecision {
5635 fn without_message_streams(commands: Vec<Value>) -> Self {
5636 Self {
5637 commands,
5638 message_stream_cursors: Vec::new(),
5639 message_stream_waits: Vec::new(),
5640 }
5641 }
5642}
5643
5644#[derive(Clone)]
5645enum RegisteredQuery {
5646 Snapshot(QueryHandler),
5647 Replayed {
5648 state_type: TypeId,
5649 handler: ReplayedQueryHandler,
5650 },
5651}
5652
5653#[derive(Clone, Debug)]
5654pub struct WorkerHeartbeatObservation {
5655 pub worker_id: String,
5656 pub task_queue: String,
5657 pub acknowledged_at_unix_millis: u64,
5658 pub acknowledgement: Value,
5659}
5660
5661#[derive(Clone, Copy, Debug)]
5668pub struct WorkerRetryPolicy {
5669 pub max_retries: usize,
5671 pub initial_backoff: Duration,
5673 pub max_backoff: Duration,
5675}
5676
5677impl Default for WorkerRetryPolicy {
5678 fn default() -> Self {
5679 Self {
5680 max_retries: 5,
5681 initial_backoff: Duration::from_millis(100),
5682 max_backoff: Duration::from_secs(5),
5683 }
5684 }
5685}
5686
5687#[derive(Clone, Copy, Debug, PartialEq, Eq)]
5688enum ManagedPollOutcome {
5689 Idle,
5690 Handled,
5691 Stop,
5692}
5693
5694#[derive(Clone)]
5695pub struct Worker {
5696 client: Client,
5697 worker_id: String,
5698 task_queue: String,
5699 workflows: HashMap<String, RegisteredWorkflow>,
5700 activities: HashMap<String, ActivityHandler>,
5701 queries: HashMap<String, HashMap<String, RegisteredQuery>>,
5702 updates: HashMap<String, HashMap<String, UpdateHandler>>,
5703 max_concurrent_workflow_tasks: usize,
5704 max_concurrent_activity_tasks: usize,
5705 poll_timeout: Duration,
5706 heartbeat_interval: Duration,
5707 retry_policy: WorkerRetryPolicy,
5708 heartbeat_observer: Option<WorkerHeartbeatObserver>,
5709}
5710
5711impl Worker {
5712 pub fn new(client: Client, task_queue: impl Into<String>) -> Self {
5713 Self {
5714 client,
5715 worker_id: default_worker_id(),
5716 task_queue: task_queue.into(),
5717 workflows: HashMap::new(),
5718 activities: HashMap::new(),
5719 queries: HashMap::new(),
5720 updates: HashMap::new(),
5721 max_concurrent_workflow_tasks: 10,
5722 max_concurrent_activity_tasks: 10,
5723 poll_timeout: Duration::from_secs(30),
5724 heartbeat_interval: Duration::from_secs(60),
5725 retry_policy: WorkerRetryPolicy::default(),
5726 heartbeat_observer: None,
5727 }
5728 }
5729
5730 pub fn worker_id(mut self, worker_id: impl Into<String>) -> Self {
5731 self.worker_id = worker_id.into();
5732 self
5733 }
5734
5735 pub fn poll_timeout(mut self, timeout: Duration) -> Self {
5736 self.poll_timeout = timeout;
5737 self
5738 }
5739
5740 pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
5741 self.heartbeat_interval = interval;
5742 self
5743 }
5744
5745 pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self {
5747 self.retry_policy = policy;
5748 self
5749 }
5750
5751 pub fn on_worker_heartbeat<F>(mut self, observer: F) -> Self
5752 where
5753 F: Fn(&WorkerHeartbeatObservation) + Send + Sync + 'static,
5754 {
5755 self.heartbeat_observer = Some(Arc::new(observer));
5756 self
5757 }
5758
5759 pub fn max_concurrent_workflow_tasks(mut self, count: usize) -> Self {
5760 self.max_concurrent_workflow_tasks = count.max(1);
5761 self
5762 }
5763
5764 pub fn max_concurrent_activity_tasks(mut self, count: usize) -> Self {
5765 self.max_concurrent_activity_tasks = count.max(1);
5766 self
5767 }
5768
5769 pub fn register_workflow<F, Fut>(&mut self, workflow_type: impl Into<String>, handler: F)
5776 where
5777 F: Fn(WorkflowContext, Value) -> Fut + Send + Sync + 'static,
5778 Fut: Future<Output = Result<Value>> + Send + 'static,
5779 {
5780 let handler = Arc::new(handler);
5781 self.workflows.insert(
5782 workflow_type.into(),
5783 RegisteredWorkflow {
5784 execute: Arc::new(move |ctx, input| {
5785 let handler = Arc::clone(&handler);
5786 Box::pin(async move {
5787 let result = handler(ctx, input.into_json()?).await?;
5788 AvroValue::from_serialize(&result)
5789 })
5790 }),
5791 replay: None,
5792 state_type: None,
5793 },
5794 );
5795 }
5796
5797 pub fn register_typed_workflow<I, O, F, Fut>(
5808 &mut self,
5809 workflow_type: impl Into<String>,
5810 handler: F,
5811 ) where
5812 I: DeserializeOwned + Send + 'static,
5813 O: Serialize + Send + 'static,
5814 F: Fn(WorkflowContext, I) -> Fut + Send + Sync + 'static,
5815 Fut: Future<Output = Result<O>> + Send + 'static,
5816 {
5817 let workflow_type = workflow_type.into();
5818 let handler_name = workflow_type.clone();
5819 let handler = Arc::new(handler);
5820 self.workflows.insert(
5821 workflow_type,
5822 RegisteredWorkflow {
5823 execute: Arc::new(move |ctx, input| {
5824 let handler = Arc::clone(&handler);
5825 let handler_name = handler_name.clone();
5826 Box::pin(async move {
5827 let input =
5828 decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
5829 let result = handler(ctx, input).await?;
5830 encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
5831 })
5832 }),
5833 replay: None,
5834 state_type: None,
5835 },
5836 );
5837 }
5838
5839 pub fn register_workflow_avro_value<F, Fut>(
5841 &mut self,
5842 workflow_type: impl Into<String>,
5843 handler: F,
5844 ) where
5845 F: Fn(WorkflowContext, AvroValue) -> Fut + Send + Sync + 'static,
5846 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
5847 {
5848 self.workflows.insert(
5849 workflow_type.into(),
5850 RegisteredWorkflow {
5851 execute: Arc::new(move |ctx, input| Box::pin(handler(ctx, input))),
5852 replay: None,
5853 state_type: None,
5854 },
5855 );
5856 }
5857
5858 pub fn register_replayed_workflow<S, Factory, F, Fut>(
5866 &mut self,
5867 workflow_type: impl Into<String>,
5868 state_factory: Factory,
5869 handler: F,
5870 ) where
5871 S: Clone + Send + Sync + 'static,
5872 Factory: Fn() -> S + Send + Sync + 'static,
5873 F: Fn(WorkflowContext, Value, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
5874 Fut: Future<Output = Result<Value>> + Send + 'static,
5875 {
5876 let state_factory = Arc::new(state_factory);
5877 let handler = Arc::new(handler);
5878
5879 let execute_factory = Arc::clone(&state_factory);
5880 let execute_handler = Arc::clone(&handler);
5881 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
5882 let state = WorkflowInstance::new(execute_factory());
5883 let handler = Arc::clone(&execute_handler);
5884 Box::pin(async move {
5885 let result = handler(ctx, input.into_json()?, state).await?;
5886 AvroValue::from_serialize(&result)
5887 }) as WorkflowFuture
5888 });
5889
5890 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
5891 let state = WorkflowInstance::new(state_factory());
5892 let snapshot_state = state.clone();
5893 let snapshot: WorkflowStateSnapshot =
5894 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
5895 let replay_handler = Arc::clone(&handler);
5896 let future = async move {
5897 let result = replay_handler(ctx, input.into_json()?, state).await?;
5898 AvroValue::from_serialize(&result)
5899 };
5900 ReplayedWorkflowInvocation {
5901 future: Box::pin(future),
5902 snapshot,
5903 }
5904 });
5905
5906 self.workflows.insert(
5907 workflow_type.into(),
5908 RegisteredWorkflow {
5909 execute,
5910 replay: Some(replay),
5911 state_type: Some(TypeId::of::<S>()),
5912 },
5913 );
5914 }
5915
5916 pub fn register_typed_replayed_workflow<I, O, S, Factory, F, Fut>(
5922 &mut self,
5923 workflow_type: impl Into<String>,
5924 state_factory: Factory,
5925 handler: F,
5926 ) where
5927 I: DeserializeOwned + Send + 'static,
5928 O: Serialize + Send + 'static,
5929 S: Clone + Send + Sync + 'static,
5930 Factory: Fn() -> S + Send + Sync + 'static,
5931 F: Fn(WorkflowContext, I, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
5932 Fut: Future<Output = Result<O>> + Send + 'static,
5933 {
5934 let workflow_type = workflow_type.into();
5935 let state_factory = Arc::new(state_factory);
5936 let handler = Arc::new(handler);
5937
5938 let execute_name = workflow_type.clone();
5939 let execute_factory = Arc::clone(&state_factory);
5940 let execute_handler = Arc::clone(&handler);
5941 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
5942 let state = WorkflowInstance::new(execute_factory());
5943 let handler = Arc::clone(&execute_handler);
5944 let handler_name = execute_name.clone();
5945 Box::pin(async move {
5946 let input = decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
5947 let result = handler(ctx, input, state).await?;
5948 encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
5949 }) as WorkflowFuture
5950 });
5951
5952 let replay_name = workflow_type.clone();
5953 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
5954 let state = WorkflowInstance::new(state_factory());
5955 let snapshot_state = state.clone();
5956 let snapshot: WorkflowStateSnapshot =
5957 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
5958 let handler = Arc::clone(&handler);
5959 let handler_name = replay_name.clone();
5960 let future = async move {
5961 let input = decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
5962 let result = handler(ctx, input, state).await?;
5963 encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
5964 };
5965 ReplayedWorkflowInvocation {
5966 future: Box::pin(future),
5967 snapshot,
5968 }
5969 });
5970
5971 self.workflows.insert(
5972 workflow_type,
5973 RegisteredWorkflow {
5974 execute,
5975 replay: Some(replay),
5976 state_type: Some(TypeId::of::<S>()),
5977 },
5978 );
5979 }
5980
5981 pub fn register_replayed_workflow_avro_value<S, Factory, F, Fut>(
5983 &mut self,
5984 workflow_type: impl Into<String>,
5985 state_factory: Factory,
5986 handler: F,
5987 ) where
5988 S: Clone + Send + Sync + 'static,
5989 Factory: Fn() -> S + Send + Sync + 'static,
5990 F: Fn(WorkflowContext, AvroValue, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
5991 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
5992 {
5993 let state_factory = Arc::new(state_factory);
5994 let handler = Arc::new(handler);
5995
5996 let execute_factory = Arc::clone(&state_factory);
5997 let execute_handler = Arc::clone(&handler);
5998 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
5999 let state = WorkflowInstance::new(execute_factory());
6000 Box::pin(execute_handler(ctx, input, state)) as WorkflowFuture
6001 });
6002
6003 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6004 let state = WorkflowInstance::new(state_factory());
6005 let snapshot_state = state.clone();
6006 let snapshot: WorkflowStateSnapshot =
6007 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6008 ReplayedWorkflowInvocation {
6009 future: Box::pin(handler(ctx, input, state)),
6010 snapshot,
6011 }
6012 });
6013
6014 self.workflows.insert(
6015 workflow_type.into(),
6016 RegisteredWorkflow {
6017 execute,
6018 replay: Some(replay),
6019 state_type: Some(TypeId::of::<S>()),
6020 },
6021 );
6022 }
6023
6024 pub fn register_activity<F, Fut>(&mut self, activity_type: impl Into<String>, handler: F)
6025 where
6026 F: Fn(ActivityContext, Value) -> Fut + Send + Sync + 'static,
6027 Fut: Future<Output = Result<Value>> + Send + 'static,
6028 {
6029 let handler = Arc::new(handler);
6030 self.activities.insert(
6031 activity_type.into(),
6032 Arc::new(move |ctx, args| {
6033 let handler = Arc::clone(&handler);
6034 Box::pin(async move {
6035 let result = handler(ctx, args.into_json()?).await?;
6036 AvroValue::from_serialize(&result)
6037 })
6038 }),
6039 );
6040 }
6041
6042 pub fn register_typed_activity<I, O, F, Fut>(
6048 &mut self,
6049 activity_type: impl Into<String>,
6050 handler: F,
6051 ) where
6052 I: DeserializeOwned + Send + 'static,
6053 O: Serialize + Send + 'static,
6054 F: Fn(ActivityContext, I) -> Fut + Send + Sync + 'static,
6055 Fut: Future<Output = Result<O>> + Send + 'static,
6056 {
6057 let activity_type = activity_type.into();
6058 let handler_name = activity_type.clone();
6059 let handler = Arc::new(handler);
6060 self.activities.insert(
6061 activity_type,
6062 Arc::new(move |ctx, input| {
6063 let handler = Arc::clone(&handler);
6064 let handler_name = handler_name.clone();
6065 Box::pin(async move {
6066 let input =
6067 decode_handler_input::<I>(input, HandlerKind::Activity, &handler_name)?;
6068 let result = handler(ctx, input).await?;
6069 encode_handler_result(&result, HandlerKind::Activity, &handler_name)
6070 })
6071 }),
6072 );
6073 }
6074
6075 pub fn register_activity_avro_value<F, Fut>(
6077 &mut self,
6078 activity_type: impl Into<String>,
6079 handler: F,
6080 ) where
6081 F: Fn(ActivityContext, AvroValue) -> Fut + Send + Sync + 'static,
6082 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6083 {
6084 self.activities.insert(
6085 activity_type.into(),
6086 Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
6087 );
6088 }
6089
6090 pub fn register_query<F, Fut>(
6096 &mut self,
6097 workflow_type: impl Into<String>,
6098 query_name: impl Into<String>,
6099 handler: F,
6100 ) where
6101 F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
6102 Fut: Future<Output = Result<Value>> + Send + 'static,
6103 {
6104 let handler = Arc::new(handler);
6105 self.queries
6106 .entry(workflow_type.into())
6107 .or_default()
6108 .insert(
6109 query_name.into(),
6110 RegisteredQuery::Snapshot(Arc::new(move |ctx, args| {
6111 let handler = Arc::clone(&handler);
6112 Box::pin(async move {
6113 let result = handler(ctx, args.into_json()?).await?;
6114 AvroValue::from_serialize(&result)
6115 })
6116 })),
6117 );
6118 }
6119
6120 pub fn register_query_avro_value<F, Fut>(
6122 &mut self,
6123 workflow_type: impl Into<String>,
6124 query_name: impl Into<String>,
6125 handler: F,
6126 ) where
6127 F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
6128 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6129 {
6130 self.queries
6131 .entry(workflow_type.into())
6132 .or_default()
6133 .insert(
6134 query_name.into(),
6135 RegisteredQuery::Snapshot(Arc::new(move |ctx, args| Box::pin(handler(ctx, args)))),
6136 );
6137 }
6138
6139 pub fn register_replayed_query<S, F, Fut>(
6146 &mut self,
6147 workflow_type: impl Into<String>,
6148 query_name: impl Into<String>,
6149 handler: F,
6150 ) where
6151 S: Clone + Send + Sync + 'static,
6152 F: Fn(QueryContext, Arc<S>, Value) -> Fut + Send + Sync + 'static,
6153 Fut: Future<Output = Result<Value>> + Send + 'static,
6154 {
6155 let handler = Arc::new(handler);
6156 let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
6157 let state = state.downcast::<S>().map_err(|_| {
6158 "registered query state type does not match the replayed workflow state".to_string()
6159 })?;
6160 let handler = Arc::clone(&handler);
6161 Ok(Box::pin(async move {
6162 let result = handler(ctx, state, args.into_json()?).await?;
6163 AvroValue::from_serialize(&result)
6164 }))
6165 });
6166
6167 self.queries
6168 .entry(workflow_type.into())
6169 .or_default()
6170 .insert(
6171 query_name.into(),
6172 RegisteredQuery::Replayed {
6173 state_type: TypeId::of::<S>(),
6174 handler: erased_handler,
6175 },
6176 );
6177 }
6178
6179 pub fn register_replayed_query_avro_value<S, F, Fut>(
6181 &mut self,
6182 workflow_type: impl Into<String>,
6183 query_name: impl Into<String>,
6184 handler: F,
6185 ) where
6186 S: Clone + Send + Sync + 'static,
6187 F: Fn(QueryContext, Arc<S>, AvroValue) -> Fut + Send + Sync + 'static,
6188 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6189 {
6190 let handler = Arc::new(handler);
6191 let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
6192 let state = state.downcast::<S>().map_err(|_| {
6193 "registered query state type does not match the replayed workflow state".to_string()
6194 })?;
6195 Ok(Box::pin(handler(ctx, state, args)))
6196 });
6197
6198 self.queries
6199 .entry(workflow_type.into())
6200 .or_default()
6201 .insert(
6202 query_name.into(),
6203 RegisteredQuery::Replayed {
6204 state_type: TypeId::of::<S>(),
6205 handler: erased_handler,
6206 },
6207 );
6208 }
6209
6210 pub fn register_update<F, Fut>(
6212 &mut self,
6213 workflow_type: impl Into<String>,
6214 update_name: impl Into<String>,
6215 handler: F,
6216 ) where
6217 F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
6218 Fut: Future<Output = Result<Value>> + Send + 'static,
6219 {
6220 let handler = Arc::new(handler);
6221 self.updates
6222 .entry(workflow_type.into())
6223 .or_default()
6224 .insert(
6225 update_name.into(),
6226 Arc::new(move |ctx, args| {
6227 let handler = Arc::clone(&handler);
6228 Box::pin(async move {
6229 let result = handler(ctx, args.into_json()?).await?;
6230 AvroValue::from_serialize(&result)
6231 })
6232 }),
6233 );
6234 }
6235
6236 pub fn register_update_avro_value<F, Fut>(
6238 &mut self,
6239 workflow_type: impl Into<String>,
6240 update_name: impl Into<String>,
6241 handler: F,
6242 ) where
6243 F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
6244 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6245 {
6246 self.updates
6247 .entry(workflow_type.into())
6248 .or_default()
6249 .insert(
6250 update_name.into(),
6251 Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
6252 );
6253 }
6254
6255 pub async fn register(&self) -> Result<RegisterWorkerResponse> {
6256 let mut command_contracts = serde_json::Map::new();
6257 for workflow_type in self.workflows.keys() {
6258 let mut queries = self
6259 .queries
6260 .get(workflow_type)
6261 .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
6262 .unwrap_or_default();
6263 queries.sort();
6264 let mut updates = self
6265 .updates
6266 .get(workflow_type)
6267 .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
6268 .unwrap_or_default();
6269 updates.sort();
6270 command_contracts.insert(
6271 workflow_type.clone(),
6272 json!({
6273 "queries": queries,
6274 "query_contracts": [],
6275 "signals": [],
6276 "signal_contracts": [],
6277 "updates": updates,
6278 "update_contracts": [],
6279 "update_validators": [],
6280 }),
6281 );
6282 }
6283
6284 self.client
6285 .register_worker_with_command_contracts(
6286 &self.worker_id,
6287 &self.task_queue,
6288 self.workflows.keys().cloned().collect(),
6289 self.activities.keys().cloned().collect(),
6290 self.max_concurrent_workflow_tasks,
6291 self.max_concurrent_activity_tasks,
6292 [
6293 Some(CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY.to_string()),
6294 Some(DURABLE_SELECTION_CAPABILITY.to_string()),
6295 Some(MEMO_UPSERTS_CAPABILITY.to_string()),
6296 Some(TYPED_SEARCH_ATTRIBUTES_CAPABILITY.to_string()),
6297 (!self.queries.is_empty()).then(|| QUERY_TASKS_CAPABILITY.to_string()),
6298 (!self.updates.is_empty()).then(|| WORKFLOW_UPDATES_CAPABILITY.to_string()),
6299 worker_protocol_supports_message_streams(WORKER_PROTOCOL_VERSION)
6300 .then(|| MESSAGE_STREAMS_CAPABILITY.to_string()),
6301 ]
6302 .into_iter()
6303 .flatten()
6304 .collect(),
6305 Value::Object(command_contracts),
6306 )
6307 .await
6308 }
6309
6310 pub async fn run(&self) -> Result<()> {
6316 self.run_until(std::future::pending::<()>()).await
6317 }
6318
6319 pub async fn run_until<F>(&self, shutdown: F) -> Result<()>
6323 where
6324 F: Future<Output = ()>,
6325 {
6326 let registration = self.register().await?;
6327 if !registration.registered {
6328 return Err(Error::WorkerLoop(format!(
6329 "worker registration for {:?} was not accepted",
6330 self.worker_id
6331 )));
6332 }
6333 let registered_worker_id = registration.worker_id.clone();
6334 let primary = self.run_registered_until(shutdown, registration).await;
6335 let deregistration = self
6336 .client
6337 .deregister_worker_registration(®istered_worker_id)
6338 .await;
6339
6340 match (primary, deregistration) {
6341 (Ok(()), Ok(_)) => Ok(()),
6342 (Ok(()), Err(deregistration)) => Err(deregistration),
6343 (Err(primary), Ok(_)) => Err(primary),
6344 (Err(primary), Err(deregistration)) => Err(Error::WorkerShutdown {
6345 primary: Box::new(primary),
6346 deregistration: Box::new(deregistration),
6347 }),
6348 }
6349 }
6350
6351 async fn run_registered_until<F>(
6352 &self,
6353 shutdown: F,
6354 registration: RegisterWorkerResponse,
6355 ) -> Result<()>
6356 where
6357 F: Future<Output = ()>,
6358 {
6359 let heartbeat_interval = Duration::from_secs(
6360 registration
6361 .heartbeat_interval_seconds
6362 .unwrap_or(self.heartbeat_interval.as_secs().max(1)),
6363 );
6364 let heartbeat = tokio::time::sleep(Duration::ZERO);
6370 tokio::pin!(heartbeat);
6371 tokio::pin!(shutdown);
6372 let stop = Arc::new(AtomicBool::new(false));
6373 let mut workflow_poller = (!self.workflows.is_empty()).then(|| {
6377 let worker = self.clone();
6378 let stop = Arc::clone(&stop);
6379 tokio::spawn(async move { worker.poll_workflows_until_stopped(stop).await })
6380 });
6381 let mut activity_poller = (!self.activities.is_empty()).then(|| {
6382 let worker = self.clone();
6383 let stop = Arc::clone(&stop);
6384 tokio::spawn(async move { worker.poll_activities_until_stopped(stop).await })
6385 });
6386 let mut query_poller = (!self.queries.is_empty()).then(|| {
6387 let worker = self.clone();
6388 let stop = Arc::clone(&stop);
6389 tokio::spawn(async move { worker.poll_queries_until_stopped(stop).await })
6390 });
6391
6392 loop {
6393 tokio::select! {
6394 _ = &mut shutdown => {
6395 stop.store(true, Ordering::SeqCst);
6396 break;
6397 }
6398 _ = &mut heartbeat => {
6399 let result = self.retry_worker_operation(|| {
6400 self.client.heartbeat_worker(
6401 &self.worker_id,
6402 self.max_concurrent_workflow_tasks,
6403 self.max_concurrent_activity_tasks,
6404 )
6405 }).await;
6406 heartbeat
6407 .as_mut()
6408 .reset(tokio::time::Instant::now() + heartbeat_interval);
6409 match result {
6410 Ok(acknowledgement) => {
6411 if let Some(observer) = &self.heartbeat_observer {
6412 observer(&WorkerHeartbeatObservation {
6413 worker_id: self.worker_id.clone(),
6414 task_queue: self.task_queue.clone(),
6415 acknowledged_at_unix_millis: SystemTime::now()
6416 .duration_since(UNIX_EPOCH)
6417 .unwrap_or_default()
6418 .as_millis()
6419 .min(u64::MAX as u128)
6420 as u64,
6421 acknowledgement,
6422 });
6423 }
6424 }
6425 Err(error) => {
6426 stop.store(true, Ordering::SeqCst);
6427 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await?;
6428 return Err(error);
6429 }
6430 }
6431 }
6432 result = OptionFuture::from(workflow_poller.as_mut()), if workflow_poller.is_some() => {
6433 workflow_poller = None;
6434 let stopped_by_server = stop.load(Ordering::SeqCst);
6435 stop.store(true, Ordering::SeqCst);
6436 let poller_result = optional_poller_result("workflow", result);
6437 let join_result =
6438 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6439 poller_result?;
6440 join_result?;
6441 if stopped_by_server {
6442 return Ok(());
6443 }
6444 return Err(Error::WorkerLoop(
6445 "workflow poller stopped unexpectedly".to_string(),
6446 ));
6447 }
6448 result = OptionFuture::from(activity_poller.as_mut()), if activity_poller.is_some() => {
6449 activity_poller = None;
6450 let stopped_by_server = stop.load(Ordering::SeqCst);
6451 stop.store(true, Ordering::SeqCst);
6452 let poller_result = optional_poller_result("activity", result);
6453 let join_result =
6454 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6455 poller_result?;
6456 join_result?;
6457 if stopped_by_server {
6458 return Ok(());
6459 }
6460 return Err(Error::WorkerLoop(
6461 "activity poller stopped unexpectedly".to_string(),
6462 ));
6463 }
6464 result = OptionFuture::from(query_poller.as_mut()), if query_poller.is_some() => {
6465 query_poller = None;
6466 let stopped_by_server = stop.load(Ordering::SeqCst);
6467 stop.store(true, Ordering::SeqCst);
6468 let poller_result = optional_poller_result("query", result);
6469 let join_result =
6470 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6471 poller_result?;
6472 join_result?;
6473 if stopped_by_server {
6474 return Ok(());
6475 }
6476 return Err(Error::WorkerLoop(
6477 "query poller stopped unexpectedly".to_string(),
6478 ));
6479 }
6480 }
6481 }
6482
6483 join_pollers(
6484 workflow_poller.take(),
6485 activity_poller.take(),
6486 query_poller.take(),
6487 )
6488 .await
6489 }
6490
6491 pub async fn run_once(&self) -> Result<usize> {
6505 let mut handled = 0;
6506 match self.poll_workflow_once().await? {
6507 ManagedPollOutcome::Handled => handled += 1,
6508 ManagedPollOutcome::Stop => return Ok(handled),
6509 ManagedPollOutcome::Idle => {}
6510 }
6511 match self.poll_activity_once().await? {
6512 ManagedPollOutcome::Handled => handled += 1,
6513 ManagedPollOutcome::Stop => return Ok(handled),
6514 ManagedPollOutcome::Idle => {}
6515 }
6516 if !self.queries.is_empty() {
6517 match self.poll_query_once().await? {
6518 ManagedPollOutcome::Handled => handled += 1,
6519 ManagedPollOutcome::Stop => return Ok(handled),
6520 ManagedPollOutcome::Idle => {}
6521 }
6522 }
6523 Ok(handled)
6524 }
6525
6526 async fn poll_workflow_once(&self) -> Result<ManagedPollOutcome> {
6527 let poll_request_id = unique_request_id("rust-workflow-poll");
6528 let response = self
6529 .retry_worker_operation(|| {
6530 self.client.poll_workflow_task_response_with_request_id(
6531 &self.worker_id,
6532 &self.task_queue,
6533 self.poll_timeout,
6534 &poll_request_id,
6535 0,
6536 )
6537 })
6538 .await?;
6539 if response.outcome().should_stop() {
6540 return Ok(ManagedPollOutcome::Stop);
6541 }
6542 let memo_updates_supported =
6543 runtime_supports_workflow_memo_updates(response.server_capabilities.as_ref());
6544 let Some(task) = response.task else {
6545 return Ok(ManagedPollOutcome::Idle);
6546 };
6547
6548 let task_id = task.task_id.clone();
6549 let attempt = task.workflow_task_attempt;
6550 let run_id = task.run_id.clone();
6551 let lease_owner = task
6552 .lease_owner
6553 .clone()
6554 .unwrap_or_else(|| self.worker_id.clone());
6555
6556 match self.execute_workflow_task_decision(task) {
6557 Ok(decision)
6558 if commands_use_workflow_memo_updates(&decision.commands)
6559 && !memo_updates_supported =>
6560 {
6561 self.client
6562 .fail_workflow_task(
6563 &task_id,
6564 &lease_owner,
6565 attempt,
6566 Error::WorkflowMemoUpdatesUnavailable.to_string(),
6567 )
6568 .await?;
6569 }
6570 Ok(decision) if decision.commands.is_empty() => {
6571 self.client
6577 .fail_workflow_task_with_type(
6578 &task_id,
6579 &lease_owner,
6580 attempt,
6581 WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE,
6582 WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE,
6583 )
6584 .await?;
6585 }
6586 Ok(decision) => {
6587 let completion = self
6588 .client
6589 .complete_workflow_task_with_message_streams(
6590 &task_id,
6591 &lease_owner,
6592 attempt,
6593 decision.commands,
6594 decision.message_stream_cursors,
6595 decision.message_stream_waits,
6596 )
6597 .await;
6598 if let Err(error) = completion {
6599 if !workflow_task_completion_is_terminal_timeout(
6600 &error,
6601 &task_id,
6602 attempt,
6603 run_id.as_deref(),
6604 ) {
6605 return Err(error);
6606 }
6607 }
6608 }
6609 Err(error) => {
6610 self.client
6611 .fail_workflow_task(&task_id, &lease_owner, attempt, error.to_string())
6612 .await?;
6613 }
6614 }
6615
6616 Ok(ManagedPollOutcome::Handled)
6617 }
6618
6619 async fn poll_workflows_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
6620 while !stop.load(Ordering::SeqCst) {
6621 if self.poll_workflow_once().await? == ManagedPollOutcome::Stop {
6622 stop.store(true, Ordering::SeqCst);
6623 break;
6624 }
6625 }
6626
6627 Ok(())
6628 }
6629
6630 async fn poll_activity_once(&self) -> Result<ManagedPollOutcome> {
6631 let poll_request_id = unique_request_id("rust-activity-poll");
6632 let response = self
6633 .retry_worker_operation(|| {
6634 self.client.poll_activity_task_response_with_request_id(
6635 &self.worker_id,
6636 &self.task_queue,
6637 self.poll_timeout,
6638 &poll_request_id,
6639 0,
6640 )
6641 })
6642 .await?;
6643 if response.outcome().should_stop() {
6644 return Ok(ManagedPollOutcome::Stop);
6645 }
6646 let Some(task) = response.task else {
6647 return Ok(ManagedPollOutcome::Idle);
6648 };
6649
6650 let task_id = task.task_id.clone();
6651 let attempt_id = task
6652 .activity_attempt_id
6653 .clone()
6654 .or(task.attempt_id.clone())
6655 .unwrap_or_default();
6656 let lease_owner = task
6657 .lease_owner
6658 .clone()
6659 .unwrap_or_else(|| self.worker_id.clone());
6660 let codec = task.payload_codec.clone();
6661 let result = self.execute_activity_task(task).await;
6662 match result {
6663 Ok(value) => {
6664 let completion = self
6665 .client
6666 .complete_activity_task(&task_id, &attempt_id, &lease_owner, value, &codec)
6667 .await;
6668 if let Err(error) = completion {
6669 if !activity_task_rejection_is_final(&error) {
6670 return Err(error);
6671 }
6672 }
6673 }
6674 Err(error) => {
6675 let failure = self
6676 .client
6677 .fail_activity_task(
6678 &task_id,
6679 &attempt_id,
6680 &lease_owner,
6681 error.to_string(),
6682 false,
6683 )
6684 .await;
6685 if let Err(error) = failure {
6686 if !activity_task_rejection_is_final(&error) {
6687 return Err(error);
6688 }
6689 }
6690 }
6691 }
6692
6693 Ok(ManagedPollOutcome::Handled)
6694 }
6695
6696 async fn poll_activities_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
6697 while !stop.load(Ordering::SeqCst) {
6698 if self.poll_activity_once().await? == ManagedPollOutcome::Stop {
6699 stop.store(true, Ordering::SeqCst);
6700 break;
6701 }
6702 }
6703
6704 Ok(())
6705 }
6706
6707 async fn poll_query_once(&self) -> Result<ManagedPollOutcome> {
6708 let poll_request_id = unique_request_id("rust-query-poll");
6709 let response = self
6710 .retry_worker_operation(|| {
6711 self.client.poll_query_task_response_with_request_id(
6712 &self.worker_id,
6713 &self.task_queue,
6714 self.poll_timeout,
6715 &poll_request_id,
6716 0,
6717 )
6718 })
6719 .await?;
6720 if response.outcome().should_stop() {
6721 return Ok(ManagedPollOutcome::Stop);
6722 }
6723 let Some(task) = response.task else {
6724 return Ok(ManagedPollOutcome::Idle);
6725 };
6726
6727 let query_task_id = task.query_task_id.clone();
6728 let attempt = task.query_task_attempt;
6729 let lease_owner = task
6730 .lease_owner
6731 .clone()
6732 .unwrap_or_else(|| self.worker_id.clone());
6733 let codec = task.payload_codec.clone();
6734
6735 match self.execute_query_task(task).await {
6736 Ok(value) => {
6737 let result_envelope = match encode_typed_envelope(&value, &codec) {
6738 Ok(result_envelope) => result_envelope,
6739 Err(error) => {
6740 let failure = self
6741 .client
6742 .fail_query_task(
6743 &query_task_id,
6744 &lease_owner,
6745 attempt,
6746 error.to_string(),
6747 "query_result_encode_failed",
6748 "QueryResultEncodeFailed",
6749 )
6750 .await;
6751 if let Err(error) = failure {
6752 if !query_task_rejection_is_final(&error) {
6753 return Err(error);
6754 }
6755 }
6756 return Ok(ManagedPollOutcome::Handled);
6757 }
6758 };
6759
6760 if let Err(error) = self
6761 .client
6762 .complete_query_task_with_envelope(
6763 &query_task_id,
6764 &lease_owner,
6765 attempt,
6766 value.clone().into_json()?,
6767 result_envelope,
6768 )
6769 .await
6770 {
6771 if !query_task_rejection_is_final(&error) {
6772 return Err(error);
6773 }
6774 }
6775 }
6776 Err(failure) => {
6777 let result = self
6778 .client
6779 .fail_query_task(
6780 &query_task_id,
6781 &lease_owner,
6782 attempt,
6783 failure.message,
6784 failure.reason,
6785 failure.failure_type,
6786 )
6787 .await;
6788 if let Err(error) = result {
6789 if !query_task_rejection_is_final(&error) {
6790 return Err(error);
6791 }
6792 }
6793 }
6794 }
6795
6796 Ok(ManagedPollOutcome::Handled)
6797 }
6798
6799 async fn poll_queries_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
6800 while !stop.load(Ordering::SeqCst) {
6801 if self.poll_query_once().await? == ManagedPollOutcome::Stop {
6802 stop.store(true, Ordering::SeqCst);
6803 break;
6804 }
6805 }
6806
6807 Ok(())
6808 }
6809
6810 async fn retry_worker_operation<T, F, Fut>(&self, mut operation: F) -> Result<T>
6811 where
6812 F: FnMut() -> Fut,
6813 Fut: Future<Output = Result<T>>,
6814 {
6815 let mut retries = 0;
6816
6817 loop {
6818 match operation().await {
6819 Err(error)
6820 if worker_operation_is_retryable(&error)
6821 && retries < self.retry_policy.max_retries =>
6822 {
6823 retries += 1;
6824 tokio::time::sleep(worker_retry_delay(self.retry_policy, retries)).await;
6825 }
6826 result => return result,
6827 }
6828 }
6829 }
6830
6831 async fn execute_query_task(
6832 &self,
6833 mut task: QueryTask,
6834 ) -> std::result::Result<AvroValue, QueryTaskExecutionFailure> {
6835 validate_query_task_payloads(&task).map_err(|error| {
6836 QueryTaskExecutionFailure::new(
6837 "query_payload_decode_failed",
6838 error.to_string(),
6839 "QueryPayloadDecodeFailed",
6840 )
6841 })?;
6842
6843 if !self.workflows.contains_key(&task.workflow_type) {
6844 return Err(QueryTaskExecutionFailure::new(
6845 "query_workflow_type_not_registered",
6846 format!("no workflow registered for type {:?}", task.workflow_type),
6847 "WorkflowTypeNotRegistered",
6848 ));
6849 }
6850
6851 let Some(handlers) = self.queries.get(&task.workflow_type) else {
6852 return Err(QueryTaskExecutionFailure::new(
6853 "query_handler_unavailable",
6854 format!(
6855 "query handlers are unavailable for workflow type {:?}",
6856 task.workflow_type
6857 ),
6858 "QueryHandlerUnavailable",
6859 ));
6860 };
6861 let Some(query) = handlers.get(&task.query_name) else {
6862 return Err(QueryTaskExecutionFailure::new(
6863 "rejected_unknown_query",
6864 format!("unknown query {:?}", task.query_name),
6865 "QueryFailed",
6866 ));
6867 };
6868
6869 let args = decode_task_avro_arguments(task.query_arguments.as_ref(), &task.payload_codec)
6870 .map_err(|error| {
6871 QueryTaskExecutionFailure::new(
6872 "query_payload_decode_failed",
6873 format!("cannot decode query arguments: {error}"),
6874 "QueryPayloadDecodeFailed",
6875 )
6876 })?;
6877 let workflow_input_typed =
6878 decode_task_avro_arguments(task.workflow_arguments.as_ref(), &task.payload_codec)
6879 .map_err(|error| {
6880 QueryTaskExecutionFailure::new(
6881 "query_workflow_state_unavailable",
6882 format!("cannot decode workflow start input: {error}"),
6883 "QueryWorkflowStateUnavailable",
6884 )
6885 })?;
6886 let workflow_input = workflow_input_typed.clone().into_json().map_err(|error| {
6887 QueryTaskExecutionFailure::new(
6888 "query_workflow_state_unavailable",
6889 format!("cannot project workflow start input: {error}"),
6890 "QueryWorkflowStateUnavailable",
6891 )
6892 })?;
6893 hydrate_query_history_from_export(&mut task).map_err(|error| {
6894 QueryTaskExecutionFailure::new(
6895 "query_workflow_state_unavailable",
6896 format!("cannot restore query history snapshot: {error}"),
6897 "QueryWorkflowStateUnavailable",
6898 )
6899 })?;
6900 enrich_query_history_from_export(&mut task).map_err(|error| {
6901 QueryTaskExecutionFailure::new(
6902 "query_workflow_state_unavailable",
6903 format!("cannot restore compact query history payloads: {error}"),
6904 "QueryWorkflowStateUnavailable",
6905 )
6906 })?;
6907 let signal_events = query_signal_events(&task).map_err(|error| {
6908 QueryTaskExecutionFailure::new(
6909 "query_workflow_state_unavailable",
6910 format!("cannot decode committed workflow signals: {error}"),
6911 "QueryWorkflowStateUnavailable",
6912 )
6913 })?;
6914 let history_events = Arc::new(std::mem::take(&mut task.history_events));
6915 let context = QueryContext {
6916 workflow_id: task.workflow_id,
6917 run_id: task.run_id,
6918 workflow_type: task.workflow_type.clone(),
6919 run_status: task.run_status,
6920 workflow_input,
6921 workflow_input_avro_value: workflow_input_typed.clone(),
6922 history_events: Arc::clone(&history_events),
6923 signal_events: Arc::new(signal_events),
6924 };
6925
6926 let future = match query {
6927 RegisteredQuery::Snapshot(handler) => handler(context, args),
6928 RegisteredQuery::Replayed {
6929 state_type,
6930 handler,
6931 } => {
6932 let workflow = self
6933 .workflows
6934 .get(&task.workflow_type)
6935 .expect("workflow registration was checked above");
6936 if workflow.state_type != Some(*state_type) {
6937 return Err(QueryTaskExecutionFailure::new(
6938 "query_workflow_state_unavailable",
6939 "replayed query state type does not match its workflow registration",
6940 "QueryWorkflowStateUnavailable",
6941 ));
6942 }
6943 let replay = workflow.replay.as_ref().ok_or_else(|| {
6944 QueryTaskExecutionFailure::new(
6945 "query_workflow_state_unavailable",
6946 format!(
6947 "workflow type {:?} is not registered for instance-state replay",
6948 task.workflow_type
6949 ),
6950 "QueryWorkflowStateUnavailable",
6951 )
6952 })?;
6953 let workflow_state = Arc::new(Mutex::new(
6954 WorkflowState::new_with_identity(
6955 history_events.as_ref().clone(),
6956 context.workflow_id.clone(),
6957 context.run_id.clone(),
6958 self.task_queue.clone(),
6959 task.payload_codec,
6960 None,
6961 )
6962 .map_err(|error| {
6963 QueryTaskExecutionFailure::new(
6964 "query_workflow_state_unavailable",
6965 format!("workflow replay failed before query: {error}"),
6966 "QueryWorkflowStateUnavailable",
6967 )
6968 })?,
6969 ));
6970 let workflow_context = WorkflowContext {
6971 state: workflow_state,
6972 };
6973 let mut invocation = replay(workflow_context.clone(), workflow_input_typed.clone());
6974 let mut cx = TaskContext::from_waker(noop_waker_ref());
6975 match invocation.future.as_mut().poll(&mut cx) {
6976 Poll::Ready(Ok(_)) => {
6977 workflow_context
6978 .ensure_history_consumed()
6979 .map_err(|error| {
6980 QueryTaskExecutionFailure::new(
6981 "query_workflow_state_unavailable",
6982 format!("workflow replay failed before query: {error}"),
6983 "QueryWorkflowStateUnavailable",
6984 )
6985 })?;
6986 }
6987 Poll::Ready(Err(error)) => {
6988 return Err(QueryTaskExecutionFailure::new(
6989 "query_workflow_state_unavailable",
6990 format!("workflow replay failed before query: {error}"),
6991 "QueryWorkflowStateUnavailable",
6992 ));
6993 }
6994 Poll::Pending => {
6995 let commands = workflow_context.take_commands().map_err(|error| {
6996 QueryTaskExecutionFailure::new(
6997 "query_workflow_state_unavailable",
6998 format!("workflow replay failed before query: {error}"),
6999 "QueryWorkflowStateUnavailable",
7000 )
7001 })?;
7002 if commands.is_empty()
7003 && !workflow_context
7004 .matched_recorded_pending()
7005 .map_err(|error| {
7006 QueryTaskExecutionFailure::new(
7007 "query_workflow_state_unavailable",
7008 format!("workflow replay failed before query: {error}"),
7009 "QueryWorkflowStateUnavailable",
7010 )
7011 })?
7012 {
7013 return Err(QueryTaskExecutionFailure::new(
7014 "query_workflow_state_unavailable",
7015 "workflow replay yielded without a durable command",
7016 "QueryWorkflowStateUnavailable",
7017 ));
7018 }
7019 }
7020 }
7021 let state = (invocation.snapshot)().map_err(|error| {
7022 QueryTaskExecutionFailure::new(
7023 "query_workflow_state_unavailable",
7024 format!("cannot snapshot replayed workflow state: {error}"),
7025 "QueryWorkflowStateUnavailable",
7026 )
7027 })?;
7028 handler(context, state, args).map_err(|message| {
7029 QueryTaskExecutionFailure::new(
7030 "query_workflow_state_unavailable",
7031 message,
7032 "QueryWorkflowStateUnavailable",
7033 )
7034 })?
7035 }
7036 };
7037
7038 future.await.map_err(|error| {
7039 QueryTaskExecutionFailure::new("query_rejected", error.to_string(), "QueryFailed")
7040 })
7041 }
7042
7043 #[cfg(test)]
7044 fn execute_workflow_task(&self, task: WorkflowTask) -> Result<Vec<Value>> {
7045 Ok(self.execute_workflow_task_decision(task)?.commands)
7046 }
7047
7048 fn execute_workflow_task_decision(&self, task: WorkflowTask) -> Result<WorkflowTaskDecision> {
7049 validate_workflow_task_payloads(&task)?;
7050
7051 if let Some(update_id) = task
7052 .workflow_update_id
7053 .as_deref()
7054 .filter(|update_id| !update_id.is_empty())
7055 {
7056 return self
7057 .execute_update_task(&task, update_id)
7058 .map(WorkflowTaskDecision::without_message_streams);
7059 }
7060
7061 let workflow = self
7062 .workflows
7063 .get(&task.workflow_type)
7064 .ok_or_else(|| Error::WorkflowNotRegistered(task.workflow_type.clone()))?;
7065 let input = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
7066 let resume_signal = decode_resume_signal(&task)?;
7067 let history_budget = WorkflowHistoryBudget {
7068 event_count: task
7069 .total_history_events
7070 .unwrap_or_else(|| u64::try_from(task.history_events.len()).unwrap_or(u64::MAX)),
7071 size_bytes: task.history_size_bytes,
7072 continue_as_new_recommended: task.continue_as_new_recommended.unwrap_or(false),
7073 pressure: task.history_budget_pressure.clone(),
7074 };
7075 let workflow_command_identity = task
7076 .workflow_command_id
7077 .clone()
7078 .filter(|identity| !identity.is_empty())
7079 .unwrap_or_default();
7080 let mut workflow_state = WorkflowState::new_with_identity(
7081 task.history_events,
7082 task.workflow_id,
7083 task.run_id,
7084 self.task_queue.clone(),
7085 task.payload_codec.clone(),
7086 resume_signal,
7087 )?;
7088 workflow_state.history_budget = history_budget;
7089 workflow_state.workflow_command_identity = workflow_command_identity;
7090 workflow_state.cancel_requested = task.cancel_requested;
7091 let state = Arc::new(Mutex::new(workflow_state));
7092 let ctx = WorkflowContext { state };
7093 let mut future = (workflow.execute)(ctx.clone(), input);
7094 let mut cx = TaskContext::from_waker(noop_waker_ref());
7095
7096 match future.as_mut().poll(&mut cx) {
7097 Poll::Ready(Ok(result)) => {
7098 ctx.ensure_history_consumed()?;
7099 let result = encode_typed_envelope(&result, &task.payload_codec)?;
7100 let mut commands = ctx.take_commands()?;
7101 commands.push(json!({
7102 "type": "complete_workflow",
7103 "result": result
7104 }));
7105 self.message_stream_decision(&ctx, commands)
7106 }
7107 Poll::Ready(Err(error)) => {
7108 if let Error::ContinueAsNew(request) = error {
7109 let mut commands = ctx.take_commands()?;
7110 if let Some(command) = ctx.continue_as_new_command(request)? {
7111 commands.push(command);
7112 }
7113 ctx.ensure_history_consumed()?;
7114 return self.message_stream_decision(&ctx, commands);
7115 }
7116 if workflow_task_integrity_error(&error) {
7117 return Err(error);
7122 }
7123 ctx.ensure_history_consumed()?;
7126 let mut commands = ctx.take_commands()?;
7127 commands.push(workflow_failure_command(&error));
7128 self.message_stream_decision(&ctx, commands)
7129 }
7130 Poll::Pending => {
7131 let commands = ctx.take_commands()?;
7132 if commands.is_empty() && !ctx.matched_recorded_pending()? {
7133 Err(Error::WorkflowYieldedWithoutCommand)
7134 } else {
7135 self.message_stream_decision(&ctx, commands)
7136 }
7137 }
7138 }
7139 }
7140
7141 fn message_stream_decision(
7142 &self,
7143 ctx: &WorkflowContext,
7144 commands: Vec<Value>,
7145 ) -> Result<WorkflowTaskDecision> {
7146 let (message_stream_cursors, message_stream_waits) = ctx.message_stream_metadata()?;
7147 Ok(WorkflowTaskDecision {
7148 commands,
7149 message_stream_cursors,
7150 message_stream_waits,
7151 })
7152 }
7153
7154 fn execute_update_task(&self, task: &WorkflowTask, update_id: &str) -> Result<Vec<Value>> {
7155 if !self.workflows.contains_key(&task.workflow_type) {
7156 return Err(Error::WorkflowNotRegistered(task.workflow_type.clone()));
7157 }
7158
7159 let accepted = task.history_events.iter().rev().find_map(|event| {
7160 (event.event_type == "UpdateAccepted"
7161 && event.payload.get("update_id").and_then(Value::as_str) == Some(update_id))
7162 .then_some(&event.payload)
7163 });
7164 let update_name = accepted
7165 .and_then(|payload| payload.get("update_name"))
7166 .and_then(Value::as_str)
7167 .or(task.update_name.as_deref())
7168 .unwrap_or_default();
7169 let Some(handler) = self
7170 .updates
7171 .get(&task.workflow_type)
7172 .and_then(|handlers| handlers.get(update_name))
7173 else {
7174 return Ok(vec![json!({
7175 "type": "fail_update",
7176 "update_id": update_id,
7177 "message": format!(
7178 "no update handler is registered for {}.{update_name}",
7179 task.workflow_type
7180 ),
7181 "exception_type": "UnknownUpdate",
7182 "non_retryable": true,
7183 })]);
7184 };
7185 let arguments = accepted
7186 .and_then(|payload| payload.get("arguments"))
7187 .or(task.arguments.as_ref());
7188 let arguments = decode_task_avro_arguments(arguments, &task.payload_codec)?;
7189 let context = QueryContext {
7190 workflow_id: task.workflow_id.clone(),
7191 run_id: task.run_id.clone(),
7192 workflow_type: task.workflow_type.clone(),
7193 run_status: Some("running".to_string()),
7194 workflow_input: Value::Null,
7195 workflow_input_avro_value: AvroValue::Null,
7196 history_events: Arc::new(task.history_events.clone()),
7197 signal_events: Arc::new(Vec::new()),
7198 };
7199 let mut future = handler(context, arguments);
7200 let mut cx = TaskContext::from_waker(noop_waker_ref());
7201
7202 match future.as_mut().poll(&mut cx) {
7203 Poll::Ready(Ok(result)) => Ok(vec![json!({
7204 "type": "complete_update",
7205 "update_id": update_id,
7206 "result": encode_typed_envelope(&result, &task.payload_codec)?,
7207 })]),
7208 Poll::Ready(Err(error)) => Ok(vec![json!({
7209 "type": "fail_update",
7210 "update_id": update_id,
7211 "message": error.to_string(),
7212 "exception_type": "UpdateFailed",
7213 "non_retryable": true,
7214 })]),
7215 Poll::Pending => Err(Error::WorkflowYieldedWithoutCommand),
7216 }
7217 }
7218
7219 async fn execute_activity_task(&self, task: ActivityTask) -> Result<AvroValue> {
7220 validate_activity_task_payloads(&task)?;
7221
7222 let handler = self
7223 .activities
7224 .get(&task.activity_type)
7225 .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?;
7226 let args = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
7227 let attempt_id = task
7228 .activity_attempt_id
7229 .clone()
7230 .or(task.attempt_id.clone())
7231 .unwrap_or_default();
7232 let lease_owner = task
7233 .lease_owner
7234 .clone()
7235 .unwrap_or_else(|| self.worker_id.clone());
7236 let ctx = ActivityContext {
7237 client: self.client.clone(),
7238 task_id: task.task_id,
7239 activity_attempt_id: attempt_id,
7240 lease_owner,
7241 activity_type: task.activity_type,
7242 attempt_number: task.attempt_number,
7243 task_queue: self.task_queue.clone(),
7244 worker_id: self.worker_id.clone(),
7245 };
7246
7247 handler(ctx, args).await
7248 }
7249}
7250
7251fn poller_result(
7252 kind: &str,
7253 result: std::result::Result<Result<()>, tokio::task::JoinError>,
7254) -> Result<()> {
7255 match result {
7256 Ok(result) => result,
7257 Err(error) => Err(Error::WorkerLoop(format!(
7258 "{kind} poller join error: {error}"
7259 ))),
7260 }
7261}
7262
7263fn optional_poller_result(
7264 kind: &str,
7265 result: Option<std::result::Result<Result<()>, tokio::task::JoinError>>,
7266) -> Result<()> {
7267 match result {
7268 Some(result) => poller_result(kind, result),
7269 None => Ok(()),
7270 }
7271}
7272
7273async fn join_pollers(
7274 workflow_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7275 activity_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7276 query_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7277) -> Result<()> {
7278 let mut first_error = None;
7279
7280 if let Some(handle) = workflow_poller {
7281 if let Err(error) = poller_result("workflow", handle.await) {
7282 first_error.get_or_insert(error);
7283 }
7284 }
7285
7286 if let Some(handle) = activity_poller {
7287 if let Err(error) = poller_result("activity", handle.await) {
7288 first_error.get_or_insert(error);
7289 }
7290 }
7291
7292 if let Some(handle) = query_poller {
7293 if let Err(error) = poller_result("query", handle.await) {
7294 first_error.get_or_insert(error);
7295 }
7296 }
7297
7298 if let Some(error) = first_error {
7299 Err(error)
7300 } else {
7301 Ok(())
7302 }
7303}
7304
7305fn default_worker_id() -> String {
7306 let millis = SystemTime::now()
7307 .duration_since(UNIX_EPOCH)
7308 .unwrap_or_default()
7309 .as_millis();
7310 format!("rust-worker-{}-{millis}", std::process::id())
7311}
7312
7313fn percent_encode_path_segment(segment: &str) -> String {
7314 const HEX: &[u8; 16] = b"0123456789ABCDEF";
7315 let mut encoded = String::with_capacity(segment.len());
7316
7317 for byte in segment.bytes() {
7318 if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') {
7319 encoded.push(char::from(byte));
7320 } else {
7321 encoded.push('%');
7322 encoded.push(char::from(HEX[(byte >> 4) as usize]));
7323 encoded.push(char::from(HEX[(byte & 0x0f) as usize]));
7324 }
7325 }
7326
7327 encoded
7328}
7329
7330fn unique_request_id(prefix: &str) -> String {
7331 let nanos = SystemTime::now()
7332 .duration_since(UNIX_EPOCH)
7333 .unwrap_or_default()
7334 .as_nanos();
7335 format!("{prefix}-{}-{nanos}", std::process::id())
7336}
7337
7338#[derive(Debug)]
7339struct QueryTaskExecutionFailure {
7340 reason: String,
7341 message: String,
7342 failure_type: String,
7343}
7344
7345impl QueryTaskExecutionFailure {
7346 fn new(
7347 reason: impl Into<String>,
7348 message: impl Into<String>,
7349 failure_type: impl Into<String>,
7350 ) -> Self {
7351 Self {
7352 reason: reason.into(),
7353 message: message.into(),
7354 failure_type: failure_type.into(),
7355 }
7356 }
7357}
7358
7359#[derive(Clone, Debug)]
7365pub struct WorkflowInstance<S> {
7366 state: Arc<Mutex<S>>,
7367}
7368
7369impl<S> WorkflowInstance<S> {
7370 fn new(state: S) -> Self {
7371 Self {
7372 state: Arc::new(Mutex::new(state)),
7373 }
7374 }
7375
7376 pub fn read<R>(&self, reader: impl FnOnce(&S) -> R) -> Result<R> {
7378 let state = self
7379 .state
7380 .lock()
7381 .map_err(|_| Error::WorkflowStatePoisoned)?;
7382 Ok(reader(&state))
7383 }
7384
7385 pub fn update<R>(&self, transition: impl FnOnce(&mut S) -> R) -> Result<R> {
7387 let mut state = self
7388 .state
7389 .lock()
7390 .map_err(|_| Error::WorkflowStatePoisoned)?;
7391 Ok(transition(&mut state))
7392 }
7393}
7394
7395impl<S: Clone> WorkflowInstance<S> {
7396 fn snapshot(&self) -> Result<S> {
7397 self.read(Clone::clone)
7398 }
7399}
7400
7401#[derive(Clone, Debug, PartialEq)]
7402pub struct MessageStreamMessage {
7403 pub stream_name: String,
7404 pub message_id: String,
7405 pub position: u64,
7406 pub arguments: Vec<AvroValue>,
7407}
7408
7409#[derive(Clone, Debug)]
7410pub struct MessageStream {
7411 ctx: WorkflowContext,
7412 name: String,
7413}
7414
7415impl MessageStream {
7416 pub async fn receive(&self, max_items: usize) -> Result<Vec<MessageStreamMessage>> {
7418 if !(1..=MESSAGE_STREAM_MAX_BATCH).contains(&max_items) {
7419 return Err(Error::Codec(format!(
7420 "message stream max_items must be between 1 and {MESSAGE_STREAM_MAX_BATCH}"
7421 )));
7422 }
7423 loop {
7424 if let Some(batch) = self.ctx.take_message_stream_batch(&self.name, max_items)? {
7425 return Ok(batch);
7426 }
7427
7428 self.ctx.record_message_stream_wait(&self.name)?;
7429 let replay_wait_sequence = self.ctx.next_message_stream_wait_sequence()?;
7430 let arguments = self.ctx.wait_runtime_signal(MESSAGE_STREAM_SIGNAL).await?;
7431 self.ctx.buffer_message_stream_delivery(arguments)?;
7432 if let Some(sequence) = replay_wait_sequence {
7433 self.ctx.buffer_message_stream_history_for_wait(sequence)?;
7434 }
7435 }
7436 }
7437
7438 pub async fn receive_one(&self) -> Result<MessageStreamMessage> {
7439 self.receive(1)
7440 .await?
7441 .into_iter()
7442 .next()
7443 .ok_or_else(|| Error::Codec("message stream resumed without a message".to_string()))
7444 }
7445}
7446
7447#[derive(Clone, Debug)]
7448pub struct WorkflowContext {
7449 state: Arc<Mutex<WorkflowState>>,
7450}
7451
7452fn valid_memo_key(key: &str) -> bool {
7453 let numeric_candidate = key.strip_prefix('-').unwrap_or(key);
7454
7455 !key.is_empty()
7456 && key.len() <= 64
7457 && (numeric_candidate.is_empty()
7458 || !numeric_candidate.bytes().all(|byte| byte.is_ascii_digit()))
7459 && key
7460 .bytes()
7461 .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b':' | b'-'))
7462}
7463
7464fn avro_encoded_size(value: &AvroValue) -> Result<usize> {
7465 BASE64
7466 .decode(encode_avro_value(value)?.blob)
7467 .map(|bytes| bytes.len())
7468 .map_err(|error| Error::Codec(format!("memo Avro encoding was not strict base64: {error}")))
7469}
7470
7471fn canonical_memo_entries(value: AvroValue, require_entries: bool) -> Result<AvroValue> {
7472 let AvroValue::Map(entries) = value else {
7473 return Err(Error::InvalidMemoUpdate(
7474 "entries must serialize to an Avro string-keyed map".to_string(),
7475 ));
7476 };
7477 if require_entries && entries.is_empty() {
7478 return Err(Error::InvalidMemoUpdate(
7479 "at least one entry is required".to_string(),
7480 ));
7481 }
7482 if entries.len() > MAX_MEMO_ENTRIES {
7483 return Err(Error::InvalidMemoUpdate(format!(
7484 "at most {MAX_MEMO_ENTRIES} entries are allowed"
7485 )));
7486 }
7487
7488 for (key, value) in &entries {
7489 if !valid_memo_key(&key) {
7490 return Err(Error::InvalidMemoUpdate(
7491 "keys must match ^(?!-?[0-9]+$)[A-Za-z0-9_.:-]{1,64}$".to_string(),
7492 ));
7493 }
7494 if avro_encoded_size(value)? > MAX_MEMO_VALUE_SIZE_BYTES {
7495 return Err(Error::InvalidMemoUpdate(format!(
7496 "value {key:?} exceeds the {MAX_MEMO_VALUE_SIZE_BYTES}-byte limit"
7497 )));
7498 }
7499 }
7500
7501 let value = AvroValue::Map(entries);
7502 if avro_encoded_size(&value)? > MAX_MEMO_TOTAL_SIZE_BYTES {
7503 return Err(Error::InvalidMemoUpdate(format!(
7504 "update exceeds the {MAX_MEMO_TOTAL_SIZE_BYTES}-byte total limit"
7505 )));
7506 }
7507 Ok(value)
7508}
7509
7510fn decode_memo_history_map(envelope: &Value, require_entries: bool) -> Result<AvroValue> {
7511 let object = envelope.as_object().ok_or_else(|| {
7512 Error::InvalidMemoUpdate(
7513 "history field must use the public {codec, blob} payload envelope".to_string(),
7514 )
7515 })?;
7516 if object.len() != 2 || !object.contains_key("codec") || !object.contains_key("blob") {
7517 return Err(Error::InvalidMemoUpdate(
7518 "history field must use exactly the public {codec, blob} payload envelope".to_string(),
7519 ));
7520 }
7521
7522 canonical_memo_entries(
7523 decode_wire_avro_value(envelope, DEFAULT_CODEC)?,
7524 require_entries,
7525 )
7526}
7527
7528impl WorkflowContext {
7529 pub fn message_stream(&self, name: impl Into<String>) -> Result<MessageStream> {
7530 let name = name.into();
7531 if name.is_empty()
7532 || name.len() > 128
7533 || !name.bytes().all(|byte| {
7534 byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b':' | b'-')
7535 })
7536 {
7537 return Err(Error::Codec(
7538 "message stream names must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens"
7539 .to_string(),
7540 ));
7541 }
7542 Ok(MessageStream {
7543 ctx: self.clone(),
7544 name,
7545 })
7546 }
7547
7548 fn record_message_stream_wait(&self, name: &str) -> Result<()> {
7549 let mut state = self
7550 .state
7551 .lock()
7552 .map_err(|_| Error::WorkflowStatePoisoned)?;
7553 let position = state.message_stream_cursors.get(name).copied().unwrap_or(0);
7554 state
7555 .message_stream_waits
7556 .insert(name.to_string(), position);
7557 Ok(())
7558 }
7559
7560 fn buffer_message_stream(&self, message: MessageStreamMessage) -> Result<()> {
7561 let mut state = self
7562 .state
7563 .lock()
7564 .map_err(|_| Error::WorkflowStatePoisoned)?;
7565 let cursor = state
7566 .message_stream_cursors
7567 .get(&message.stream_name)
7568 .copied()
7569 .unwrap_or(0);
7570 if message.position <= cursor {
7571 return Ok(());
7572 }
7573 let pending = state
7574 .message_stream_messages
7575 .entry(message.stream_name.clone())
7576 .or_default();
7577 if pending.iter().any(|candidate| {
7578 candidate.position == message.position || candidate.message_id == message.message_id
7579 }) {
7580 return Ok(());
7581 }
7582 pending.push(message);
7583 pending.sort_by_key(|candidate| candidate.position);
7584 Ok(())
7585 }
7586
7587 fn buffer_message_stream_delivery(&self, arguments: Vec<Value>) -> Result<Option<String>> {
7588 if let Some(delivery) = decode_message_stream_delivery(arguments)? {
7589 match delivery {
7590 MessageStreamDelivery::Message(message) => {
7591 let stream_name = message.stream_name.clone();
7592 self.buffer_message_stream(message)?;
7593 return Ok(Some(stream_name));
7594 }
7595 MessageStreamDelivery::Cursor {
7596 stream_name,
7597 through_position,
7598 } => self.apply_message_stream_cursor(&stream_name, through_position)?,
7599 }
7600 }
7601 Ok(None)
7602 }
7603
7604 fn next_message_stream_wait_sequence(&self) -> Result<Option<u64>> {
7605 let state = self
7606 .state
7607 .lock()
7608 .map_err(|_| Error::WorkflowStatePoisoned)?;
7609 Ok(match state.recorded_commands.get(state.command_cursor) {
7610 Some(RecordedCommand::SignalWait {
7611 sequence,
7612 signal_name,
7613 ..
7614 }) if signal_name == MESSAGE_STREAM_SIGNAL => Some(*sequence),
7615 _ => None,
7616 })
7617 }
7618
7619 fn buffer_message_stream_history_for_wait(&self, wait_sequence: u64) -> Result<()> {
7620 let (history, payload_codec) = {
7621 let state = self
7622 .state
7623 .lock()
7624 .map_err(|_| Error::WorkflowStatePoisoned)?;
7625 (
7626 Arc::clone(&state.history_events),
7627 state.payload_codec.clone(),
7628 )
7629 };
7630
7631 let Some(opened_index) = history.iter().position(|event| {
7632 event.event_type == "SignalWaitOpened"
7633 && durable_event_sequence(event) == Some(wait_sequence)
7634 && event.payload.get("signal_name").and_then(Value::as_str)
7635 == Some(MESSAGE_STREAM_SIGNAL)
7636 }) else {
7637 return Ok(());
7638 };
7639 let boundary_index = history
7640 .iter()
7641 .enumerate()
7642 .skip(opened_index + 1)
7643 .find_map(|(index, event)| {
7644 (durable_event_sequence(event).is_some_and(|sequence| sequence > wait_sequence)
7645 && is_authored_command_open_event(event))
7646 .then_some(index)
7647 })
7648 .unwrap_or(history.len());
7649
7650 for event in history[opened_index + 1..boundary_index]
7651 .iter()
7652 .filter(|event| {
7653 event.event_type == "SignalReceived"
7654 && event.payload.get("signal_name").and_then(Value::as_str)
7655 == Some(MESSAGE_STREAM_SIGNAL)
7656 })
7657 {
7658 let arguments = decode_signal_event_arguments(event, &payload_codec)?
7659 .into_iter()
7660 .map(AvroValue::into_json)
7661 .collect::<Result<Vec<_>>>()?;
7662 self.buffer_message_stream_delivery(arguments)?;
7663 }
7664 Ok(())
7665 }
7666
7667 fn apply_message_stream_cursor(&self, name: &str, through_position: u64) -> Result<()> {
7668 let mut state = self
7669 .state
7670 .lock()
7671 .map_err(|_| Error::WorkflowStatePoisoned)?;
7672 let cursor = state
7673 .message_stream_cursors
7674 .entry(name.to_string())
7675 .or_default();
7676 *cursor = (*cursor).max(through_position);
7677 if let Some(pending) = state.message_stream_messages.get_mut(name) {
7678 pending.retain(|message| message.position > through_position);
7679 }
7680 Ok(())
7681 }
7682
7683 fn take_message_stream_batch(
7684 &self,
7685 name: &str,
7686 max_items: usize,
7687 ) -> Result<Option<Vec<MessageStreamMessage>>> {
7688 let mut state = self
7689 .state
7690 .lock()
7691 .map_err(|_| Error::WorkflowStatePoisoned)?;
7692 let cursor = state.message_stream_cursors.get(name).copied().unwrap_or(0);
7693 let pending = state
7694 .message_stream_messages
7695 .entry(name.to_string())
7696 .or_default();
7697 let count = contiguous_message_stream_count(pending, cursor, max_items);
7698 if count == 0 {
7699 return Ok(None);
7700 }
7701 let batch = pending.drain(..count).collect::<Vec<_>>();
7702 let position = batch.last().map(|message| message.position).unwrap_or(0);
7703 state
7704 .message_stream_cursors
7705 .insert(name.to_string(), position);
7706 state.message_stream_waits.remove(name);
7707 Ok(Some(batch))
7708 }
7709
7710 fn message_stream_metadata(&self) -> Result<(Vec<Value>, Vec<Value>)> {
7711 let state = self
7712 .state
7713 .lock()
7714 .map_err(|_| Error::WorkflowStatePoisoned)?;
7715 let mut cursors = state.message_stream_cursors.iter().collect::<Vec<_>>();
7716 cursors.sort_by_key(|(name, _)| *name);
7717 let mut waits = state.message_stream_waits.iter().collect::<Vec<_>>();
7718 waits.sort_by_key(|(name, _)| *name);
7719 Ok((
7720 cursors
7721 .into_iter()
7722 .map(|(name, position)| json!({"stream_name": name, "through_position": position}))
7723 .collect(),
7724 waits
7725 .into_iter()
7726 .map(|(name, position)| json!({"stream_name": name, "after_position": position}))
7727 .collect(),
7728 ))
7729 }
7730 pub fn workflow_identity(&self) -> Result<WorkflowIdentity> {
7732 let state = self
7733 .state
7734 .lock()
7735 .map_err(|_| Error::WorkflowStatePoisoned)?;
7736 Ok(WorkflowIdentity {
7737 workflow_id: state.workflow_id.clone(),
7738 run_id: state.run_id.clone(),
7739 })
7740 }
7741
7742 pub fn history_budget(&self) -> Result<WorkflowHistoryBudget> {
7744 let state = self
7745 .state
7746 .lock()
7747 .map_err(|_| Error::WorkflowStatePoisoned)?;
7748 Ok(state.history_budget.clone())
7749 }
7750
7751 pub fn continue_as_new<T: Serialize>(&self, args: T) -> Result<Value> {
7757 self.continue_as_new_with_options(ContinueAsNewOptions::new(), args)
7758 }
7759
7760 pub fn continue_as_new_with_options<T: Serialize>(
7762 &self,
7763 options: ContinueAsNewOptions,
7764 args: T,
7765 ) -> Result<Value> {
7766 options.validate()?;
7767 Err(Error::ContinueAsNew(ContinueAsNewRequest {
7768 arguments: normalize_avro_arguments(AvroValue::from_serialize(&args)?),
7769 options,
7770 }))
7771 }
7772
7773 pub fn activity<T: Serialize>(
7774 &self,
7775 activity_type: impl Into<String>,
7776 args: T,
7777 ) -> ActivityCall {
7778 self.activity_with_options(activity_type, ActivityOptions::new(), args)
7779 }
7780
7781 pub fn activity_on_queue<T, Q>(
7782 &self,
7783 activity_type: impl Into<String>,
7784 task_queue: Option<Q>,
7785 args: T,
7786 ) -> ActivityCall
7787 where
7788 T: Serialize,
7789 Q: Into<String>,
7790 {
7791 let mut options = ActivityOptions::new();
7792 options.task_queue = task_queue.map(Into::into);
7793 self.activity_with_options(activity_type, options, args)
7794 }
7795
7796 pub fn activity_with_options<T: Serialize>(
7834 &self,
7835 activity_type: impl Into<String>,
7836 options: ActivityOptions,
7837 args: T,
7838 ) -> ActivityCall {
7839 ActivityCall {
7840 ctx: self.clone(),
7841 activity_type: activity_type.into(),
7842 options,
7843 args: Some(AvroValue::from_serialize(&args)),
7844 scheduled: false,
7845 parallel_group_path: Vec::new(),
7846 }
7847 }
7848
7849 pub async fn activity_avro_value<T: Serialize>(
7850 &self,
7851 activity_type: impl Into<String>,
7852 args: T,
7853 ) -> Result<AvroValue> {
7854 let mut call = self.activity(activity_type, args);
7855 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
7856 }
7857
7858 pub async fn activity_avro_value_with_options<T: Serialize>(
7859 &self,
7860 activity_type: impl Into<String>,
7861 options: ActivityOptions,
7862 args: T,
7863 ) -> Result<AvroValue> {
7864 let mut call = self.activity_with_options(activity_type, options, args);
7865 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
7866 }
7867
7868 pub async fn activity_typed<I, O>(&self, activity_type: impl Into<String>, args: I) -> Result<O>
7870 where
7871 I: Serialize,
7872 O: DeserializeOwned,
7873 {
7874 self.activity_typed_with_options(activity_type, ActivityOptions::new(), args)
7875 .await
7876 }
7877
7878 pub async fn activity_typed_with_options<I, O>(
7884 &self,
7885 activity_type: impl Into<String>,
7886 options: ActivityOptions,
7887 args: I,
7888 ) -> Result<O>
7889 where
7890 I: Serialize,
7891 O: DeserializeOwned,
7892 {
7893 let activity_type = activity_type.into();
7894 let encoded = AvroValue::from_serialize(&args).map_err(|error| {
7895 handler_type_error::<I>(
7896 HandlerKind::Activity,
7897 &activity_type,
7898 HandlerValueKind::Input,
7899 error.to_string(),
7900 )
7901 });
7902 let mut call = ActivityCall {
7903 ctx: self.clone(),
7904 activity_type: activity_type.clone(),
7905 options,
7906 args: Some(encoded),
7907 scheduled: false,
7908 parallel_group_path: Vec::new(),
7909 };
7910 let result = std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await?;
7911 decode_handler_result(result, HandlerKind::Activity, &activity_type)
7912 }
7913
7914 pub fn parallel(&self, operations: Vec<ParallelOperation>) -> ParallelCall {
7921 ParallelCall::new(self.clone(), operations)
7922 }
7923
7924 pub fn join(&self, operations: Vec<ParallelOperation>) -> ParallelCall {
7926 self.parallel(operations)
7927 }
7928
7929 pub async fn parallel_avro_value(
7931 &self,
7932 operations: Vec<ParallelOperation>,
7933 ) -> Result<Vec<ParallelAvroResult>> {
7934 let mut call = self.parallel(operations);
7935 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
7936 }
7937
7938 pub fn select(&self, operations: Vec<ParallelOperation>) -> SelectCall {
7941 let operations = operations
7942 .into_iter()
7943 .enumerate()
7944 .map(|(index, operation)| (SelectionKey::Index(index), operation))
7945 .collect();
7946 SelectCall::new(self.clone(), operations)
7947 }
7948
7949 pub fn select_keyed<K>(&self, operations: Vec<(K, ParallelOperation)>) -> SelectCall
7951 where
7952 K: Into<SelectionKey>,
7953 {
7954 SelectCall::new(
7955 self.clone(),
7956 operations
7957 .into_iter()
7958 .map(|(key, operation)| (key.into(), operation))
7959 .collect(),
7960 )
7961 }
7962
7963 pub fn saga(&self) -> Saga {
7965 Saga::new(self.clone())
7966 }
7967
7968 pub fn is_cancellation_requested(&self) -> Result<bool> {
7970 let state = self
7971 .state
7972 .lock()
7973 .map_err(|_| Error::WorkflowStatePoisoned)?;
7974 Ok(state.cancel_requested)
7975 }
7976
7977 pub fn throw_if_cancellation_requested(&self) -> Result<()> {
7982 if self.is_cancellation_requested()? {
7983 return Err(Error::WorkflowCancellationRequested(
7984 WorkflowCancellationRequested,
7985 ));
7986 }
7987 Ok(())
7988 }
7989
7990 pub fn wait_signal(&self, signal_name: impl Into<String>) -> SignalCall {
7991 SignalCall {
7992 ctx: self.clone(),
7993 signal_name: signal_name.into(),
7994 runtime_reserved_allowed: false,
7995 opened_wait: false,
7996 matched_pending: false,
7997 parallel_group_path: Vec::new(),
7998 }
7999 }
8000
8001 fn wait_runtime_signal(&self, signal_name: impl Into<String>) -> SignalCall {
8002 SignalCall {
8003 ctx: self.clone(),
8004 signal_name: signal_name.into(),
8005 runtime_reserved_allowed: true,
8006 opened_wait: false,
8007 matched_pending: false,
8008 parallel_group_path: Vec::new(),
8009 }
8010 }
8011
8012 pub async fn wait_signal_avro_value(
8013 &self,
8014 signal_name: impl Into<String>,
8015 ) -> Result<Vec<AvroValue>> {
8016 let mut call = self.wait_signal(signal_name);
8017 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8018 }
8019
8020 pub fn signals(&self, signal_name: &str) -> Result<Vec<Vec<Value>>> {
8026 self.signals_avro_value(signal_name)?
8027 .into_iter()
8028 .map(|arguments| {
8029 arguments
8030 .into_iter()
8031 .map(AvroValue::into_json)
8032 .collect::<Result<Vec<_>>>()
8033 })
8034 .collect()
8035 }
8036
8037 pub fn signals_avro_value(&self, signal_name: &str) -> Result<Vec<Vec<AvroValue>>> {
8039 let state = self
8040 .state
8041 .lock()
8042 .map_err(|_| Error::WorkflowStatePoisoned)?;
8043 state
8044 .history_events
8045 .iter()
8046 .filter(|event| {
8047 event.event_type == "SignalReceived"
8048 && event.payload.get("signal_name").and_then(Value::as_str) == Some(signal_name)
8049 })
8050 .map(|event| decode_signal_event_arguments(event, &state.payload_codec))
8051 .collect()
8052 }
8053
8054 pub fn updates(&self, update_name: &str) -> Result<Vec<Vec<Value>>> {
8060 self.updates_avro_value(update_name)?
8061 .into_iter()
8062 .map(|arguments| {
8063 arguments
8064 .into_iter()
8065 .map(AvroValue::into_json)
8066 .collect::<Result<Vec<_>>>()
8067 })
8068 .collect()
8069 }
8070
8071 pub fn updates_avro_value(&self, update_name: &str) -> Result<Vec<Vec<AvroValue>>> {
8073 let state = self
8074 .state
8075 .lock()
8076 .map_err(|_| Error::WorkflowStatePoisoned)?;
8077 let mut seen = Vec::new();
8078 let mut updates = Vec::new();
8079 for event in state.history_events.iter() {
8080 if !matches!(
8081 event.event_type.as_str(),
8082 "UpdateAccepted" | "UpdateApplied"
8083 ) || event.payload.get("update_name").and_then(Value::as_str) != Some(update_name)
8084 || event.payload.get("arguments").is_none()
8085 {
8086 continue;
8087 }
8088 if let Some(update_id) = event.payload.get("update_id").and_then(Value::as_str) {
8089 if seen.iter().any(|recorded| recorded == update_id) {
8090 continue;
8091 }
8092 seen.push(update_id.to_string());
8093 }
8094 updates.push(decode_update_event_arguments(event, &state.payload_codec)?);
8095 }
8096 Ok(updates)
8097 }
8098
8099 pub fn wait_condition<F>(
8107 &self,
8108 options: ConditionWaitOptions,
8109 predicate: F,
8110 ) -> ConditionWaitCall
8111 where
8112 F: Fn() -> Result<bool> + Send + 'static,
8113 {
8114 ConditionWaitCall {
8115 ctx: self.clone(),
8116 options,
8117 predicate: Box::new(predicate),
8118 occurrence_id: None,
8119 opened_wait: false,
8120 parallel_group_path: Vec::new(),
8121 }
8122 }
8123
8124 pub fn sleep(&self, duration: Duration) -> TimerCall {
8145 let delay_seconds = duration
8146 .as_secs()
8147 .checked_add(u64::from(duration.subsec_nanos() > 0));
8148 TimerCall {
8149 ctx: self.clone(),
8150 delay_seconds,
8151 scheduled: false,
8152 matched_pending: false,
8153 parallel_group_path: Vec::new(),
8154 }
8155 }
8156
8157 pub fn start_timer(&self, duration: Duration) -> TimerCall {
8159 self.sleep(duration)
8160 }
8161
8162 pub fn side_effect<T, F>(&self, callback: F) -> Result<T>
8169 where
8170 T: Serialize + DeserializeOwned,
8171 F: FnOnce() -> T,
8172 {
8173 {
8174 let mut state = self
8175 .state
8176 .lock()
8177 .map_err(|_| Error::WorkflowStatePoisoned)?;
8178 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8179 return match recorded {
8180 RecordedCommand::SideEffect { sequence, value } => {
8181 state.command_cursor += 1;
8182 value.deserialize().map_err(|error| {
8183 Error::NonDeterministicReplay(ReplayFailure::new(
8184 "side_effect_type_mismatch",
8185 Some(sequence),
8186 Some(std::any::type_name::<T>().to_string()),
8187 Some(error.to_string()),
8188 "recorded side-effect value is incompatible with the requested Rust type",
8189 ))
8190 })
8191 }
8192 other => Err(command_mismatch(&other, "side effect")),
8193 };
8194 }
8195 }
8196
8197 let value = callback();
8198 let avro_value = AvroValue::from_serialize(&value)?;
8199 let mut state = self
8200 .state
8201 .lock()
8202 .map_err(|_| Error::WorkflowStatePoisoned)?;
8203 let result = encode_typed_envelope(&avro_value, &state.payload_codec)?;
8204 state.commands.push(json!({
8205 "type": "record_side_effect",
8206 "result": result,
8207 }));
8208 Ok(value)
8209 }
8210
8211 pub fn side_effect_avro_value<F>(&self, callback: F) -> Result<AvroValue>
8213 where
8214 F: FnOnce() -> AvroValue,
8215 {
8216 {
8217 let mut state = self
8218 .state
8219 .lock()
8220 .map_err(|_| Error::WorkflowStatePoisoned)?;
8221 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8222 return match recorded {
8223 RecordedCommand::SideEffect { value, .. } => {
8224 state.command_cursor += 1;
8225 Ok(value)
8226 }
8227 other => Err(command_mismatch(&other, "side effect")),
8228 };
8229 }
8230 }
8231
8232 let value = callback();
8233 let mut state = self
8234 .state
8235 .lock()
8236 .map_err(|_| Error::WorkflowStatePoisoned)?;
8237 let result = encode_typed_envelope(&value, &state.payload_codec)?;
8238 state.commands.push(json!({
8239 "type": "record_side_effect",
8240 "result": result,
8241 }));
8242 Ok(value)
8243 }
8244
8245 pub fn append_workflow_stream(
8251 &self,
8252 stream_name: impl Into<String>,
8253 items: &[WorkflowStreamAppendItem],
8254 max_pending_items: Option<u64>,
8255 ) -> Result<()> {
8256 if items.is_empty() {
8257 return Err(Error::Codec(
8258 "workflow_stream_items_empty: append requires at least one item".to_string(),
8259 ));
8260 }
8261 if max_pending_items == Some(0) {
8262 return Err(Error::Codec(
8263 "workflow_stream_pending_limit_invalid: max_pending_items must be positive"
8264 .to_string(),
8265 ));
8266 }
8267 let stream_name = stream_name.into();
8268 if stream_name.is_empty() {
8269 return Err(Error::Codec(
8270 "workflow_stream_name_invalid: stream name must not be empty".to_string(),
8271 ));
8272 }
8273
8274 let mut state = self
8275 .state
8276 .lock()
8277 .map_err(|_| Error::WorkflowStatePoisoned)?;
8278 let command_ordinal = state.workflow_stream_command_counter;
8279 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8280 state.workflow_stream_command_counter += 1;
8281 return match recorded {
8282 RecordedCommand::SideEffect { .. } => {
8283 state.command_cursor += 1;
8284 Ok(())
8285 }
8286 other => Err(command_mismatch(&other, "workflow stream append")),
8287 };
8288 }
8289
8290 let identity = Self::workflow_stream_command_identity(&state)?.to_string();
8291 state.workflow_stream_command_counter += 1;
8292 let wire_items = items
8293 .iter()
8294 .enumerate()
8295 .map(|(item_index, item)| {
8296 item.wire_value(Some(format!(
8297 "dw-stream:{identity}:{command_ordinal}:{item_index}"
8298 )))
8299 })
8300 .collect::<Vec<_>>();
8301 let mut directive = json!({
8302 "operation": "append",
8303 "stream_name": stream_name,
8304 "command_identity": identity,
8305 "command_ordinal": command_ordinal,
8306 "items": wire_items,
8307 });
8308 if let Some(max_pending_items) = max_pending_items {
8309 directive["max_pending_items"] = json!(max_pending_items);
8310 }
8311 let result = encode_typed_envelope(&AvroValue::Null, &state.payload_codec)?;
8312 state.commands.push(json!({
8313 "type": "record_side_effect",
8314 "result": result,
8315 "workflow_stream": directive,
8316 }));
8317 Ok(())
8318 }
8319
8320 pub fn close_workflow_stream(
8322 &self,
8323 stream_name: impl Into<String>,
8324 retention_seconds: Option<u64>,
8325 ) -> Result<()> {
8326 self.finish_workflow_stream(stream_name.into(), None, retention_seconds)
8327 }
8328
8329 pub fn error_workflow_stream(
8331 &self,
8332 stream_name: impl Into<String>,
8333 error_reason: impl Into<String>,
8334 retention_seconds: Option<u64>,
8335 ) -> Result<()> {
8336 let error_reason = error_reason.into();
8337 if error_reason.is_empty() {
8338 return Err(Error::Codec(
8339 "workflow_stream_error_invalid: error reason must not be empty".to_string(),
8340 ));
8341 }
8342 self.finish_workflow_stream(stream_name.into(), Some(error_reason), retention_seconds)
8343 }
8344
8345 fn finish_workflow_stream(
8346 &self,
8347 stream_name: String,
8348 error_reason: Option<String>,
8349 retention_seconds: Option<u64>,
8350 ) -> Result<()> {
8351 if stream_name.is_empty() {
8352 return Err(Error::Codec(
8353 "workflow_stream_name_invalid: stream name must not be empty".to_string(),
8354 ));
8355 }
8356 if retention_seconds == Some(0) {
8357 return Err(Error::Codec(
8358 "workflow_stream_retention_invalid: retention_seconds must be positive".to_string(),
8359 ));
8360 }
8361 let mut state = self
8362 .state
8363 .lock()
8364 .map_err(|_| Error::WorkflowStatePoisoned)?;
8365 let command_ordinal = state.workflow_stream_command_counter;
8366 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8367 state.workflow_stream_command_counter += 1;
8368 return match recorded {
8369 RecordedCommand::SideEffect { .. } => {
8370 state.command_cursor += 1;
8371 Ok(())
8372 }
8373 other => Err(command_mismatch(&other, "workflow stream close")),
8374 };
8375 }
8376 let identity = Self::workflow_stream_command_identity(&state)?.to_string();
8377 state.workflow_stream_command_counter += 1;
8378 let mut directive = json!({
8379 "operation": if error_reason.is_some() { "error" } else { "close" },
8380 "stream_name": stream_name,
8381 "command_identity": identity,
8382 "command_ordinal": command_ordinal,
8383 });
8384 if let Some(error_reason) = error_reason {
8385 directive["error_reason"] = json!(error_reason);
8386 }
8387 if let Some(retention_seconds) = retention_seconds {
8388 directive["retention_seconds"] = json!(retention_seconds);
8389 }
8390 let result = encode_typed_envelope(&AvroValue::Null, &state.payload_codec)?;
8391 state.commands.push(json!({
8392 "type": "record_side_effect",
8393 "result": result,
8394 "workflow_stream": directive,
8395 }));
8396 Ok(())
8397 }
8398
8399 fn workflow_stream_command_identity(state: &WorkflowState) -> Result<&str> {
8400 let identity = state.workflow_command_identity.as_str();
8401 if identity.is_empty() {
8402 return Err(Error::MissingWorkflowCommandIdentity);
8403 }
8404 Ok(identity)
8405 }
8406
8407 pub fn upsert_search_attributes(&self, update: SearchAttributeUpdate) -> Result<()> {
8413 update.validate()?;
8414 let (attributes, attribute_types) = update.into_wire_parts();
8415 let mut state = self
8416 .state
8417 .lock()
8418 .map_err(|_| Error::WorkflowStatePoisoned)?;
8419
8420 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8421 return match recorded {
8422 RecordedCommand::SearchAttributes {
8423 sequence,
8424 attributes: recorded_attributes,
8425 attribute_types: recorded_attribute_types,
8426 } => {
8427 if recorded_attributes != attributes {
8428 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8429 "search_attribute_value_mismatch",
8430 Some(sequence),
8431 Some(recorded_attributes.to_string()),
8432 Some(attributes.to_string()),
8433 "search-attribute values differ from the recorded durable command",
8434 )));
8435 }
8436 if let RecordedSnapshotValue::Known(recorded_types) = recorded_attribute_types {
8437 if recorded_types != attribute_types {
8438 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8439 "search_attribute_type_mismatch",
8440 Some(sequence),
8441 Some(json!(recorded_types).to_string()),
8442 Some(json!(attribute_types).to_string()),
8443 "search-attribute declared types differ from the recorded durable command",
8444 )));
8445 }
8446 }
8447 state.command_cursor += 1;
8448 Ok(())
8449 }
8450 other => Err(command_mismatch(&other, "search-attribute update")),
8451 };
8452 }
8453
8454 let mut command = serde_json::Map::from_iter([
8455 ("type".to_string(), json!("upsert_search_attributes")),
8456 ("attributes".to_string(), attributes),
8457 ]);
8458 if !attribute_types.is_empty() {
8459 command.insert("attribute_types".to_string(), json!(attribute_types));
8460 }
8461 state.commands.push(Value::Object(command));
8462 Ok(())
8463 }
8464
8465 pub fn uuid_v4(&self) -> Result<Uuid> {
8467 self.side_effect(Uuid::new_v4)
8468 }
8469
8470 pub fn get_version(
8473 &self,
8474 change_id: impl Into<String>,
8475 min_supported: i32,
8476 max_supported: i32,
8477 ) -> Result<i32> {
8478 let change_id = change_id.into();
8479 if change_id.trim().is_empty() {
8480 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8481 "version_change_id_invalid",
8482 None,
8483 Some("non-empty change ID".to_string()),
8484 Some(change_id),
8485 "version markers require a stable non-empty change ID",
8486 )));
8487 }
8488 if min_supported > max_supported {
8489 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8490 "version_range_invalid",
8491 None,
8492 Some("min_supported <= max_supported".to_string()),
8493 Some(format!("{min_supported}..={max_supported}")),
8494 "version marker supported range is invalid",
8495 )));
8496 }
8497
8498 let mut state = self
8499 .state
8500 .lock()
8501 .map_err(|_| Error::WorkflowStatePoisoned)?;
8502 if let Some((version, sequence)) = state.version_markers.get(&change_id).copied() {
8503 ensure_version_supported(&change_id, version, min_supported, max_supported, sequence)?;
8504 return Ok(version);
8505 }
8506
8507 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8508 return match recorded {
8509 RecordedCommand::VersionMarker {
8510 sequence,
8511 change_id: recorded_change_id,
8512 version,
8513 ..
8514 } => {
8515 if recorded_change_id != change_id {
8516 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8517 "version_change_id_mismatch",
8518 Some(sequence),
8519 Some(recorded_change_id),
8520 Some(change_id),
8521 "recorded version marker change ID differs from current workflow code",
8522 )));
8523 }
8524 ensure_version_supported(
8525 &change_id,
8526 version,
8527 min_supported,
8528 max_supported,
8529 sequence,
8530 )?;
8531 state.command_cursor += 1;
8532 state.version_markers.insert(change_id, (version, sequence));
8533 Ok(version)
8534 }
8535 other => Err(command_mismatch(
8536 &other,
8537 format!("version marker:{change_id}"),
8538 )),
8539 };
8540 }
8541
8542 let version = max_supported;
8543 state.commands.push(json!({
8544 "type": "record_version_marker",
8545 "change_id": change_id,
8546 "version": version,
8547 "min_supported": min_supported,
8548 "max_supported": max_supported,
8549 }));
8550 state.version_markers.insert(change_id, (version, 0));
8553 Ok(version)
8554 }
8555
8556 pub fn patched(&self, change_id: impl Into<String>) -> Result<bool> {
8558 Ok(self.get_version(change_id, -1, 1)? == 1)
8559 }
8560
8561 pub fn deprecate_patch(&self, change_id: impl Into<String>) -> Result<()> {
8563 self.get_version(change_id, -1, 1).map(|_| ())
8564 }
8565
8566 pub fn upsert_memo<T: Serialize>(&self, entries: T) -> Result<()> {
8571 let entries = canonical_memo_entries(AvroValue::from_serialize(&entries)?, true)?;
8572 let mut state = self
8573 .state
8574 .lock()
8575 .map_err(|_| Error::WorkflowStatePoisoned)?;
8576
8577 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8578 return match recorded {
8579 RecordedCommand::Memo {
8580 sequence,
8581 entries: recorded_entries,
8582 } => {
8583 if recorded_entries != entries {
8584 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8585 "memo_update_mismatch",
8586 Some(sequence),
8587 Some(format!("{recorded_entries:?}")),
8588 Some(format!("{entries:?}")),
8589 "recorded memo entries differ from the current workflow update",
8590 )));
8591 }
8592 state.command_cursor += 1;
8593 Ok(())
8594 }
8595 other => Err(command_mismatch(&other, "memo upsert")),
8596 };
8597 }
8598
8599 let entries_envelope = encode_typed_envelope(&entries, DEFAULT_CODEC)?;
8600 state.commands.push(json!({
8601 "type": "upsert_memo",
8602 "entries": entries_envelope,
8603 }));
8604 Ok(())
8605 }
8606
8607 pub fn start_child_workflow<T: Serialize>(
8634 &self,
8635 workflow_type: impl Into<String>,
8636 options: ChildWorkflowOptions,
8637 args: T,
8638 ) -> ChildWorkflowCall {
8639 ChildWorkflowCall {
8640 ctx: self.clone(),
8641 workflow_type: workflow_type.into(),
8642 options,
8643 args: Some(AvroValue::from_serialize(&args)),
8644 scheduled: false,
8645 matched_pending: false,
8646 parallel_group_path: Vec::new(),
8647 }
8648 }
8649
8650 pub async fn start_child_workflow_avro_value<T: Serialize>(
8651 &self,
8652 workflow_type: impl Into<String>,
8653 options: ChildWorkflowOptions,
8654 args: T,
8655 ) -> Result<ChildWorkflowAvroResult> {
8656 let mut call = self.start_child_workflow(workflow_type, options, args);
8657 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8658 }
8659
8660 fn take_commands(&self) -> Result<Vec<Value>> {
8661 let mut state = self
8662 .state
8663 .lock()
8664 .map_err(|_| Error::WorkflowStatePoisoned)?;
8665 Ok(std::mem::take(&mut state.commands))
8666 }
8667
8668 fn continue_as_new_command(&self, request: ContinueAsNewRequest) -> Result<Option<Value>> {
8669 let mut state = self
8670 .state
8671 .lock()
8672 .map_err(|_| Error::WorkflowStatePoisoned)?;
8673
8674 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8675 return Err(command_mismatch(&recorded, "continue as new"));
8676 }
8677 if state.recorded_continue_as_new_sequence.is_some() {
8678 state.continue_as_new_consumed = true;
8679 return Ok(None);
8680 }
8681
8682 let arguments = encode_typed_envelope(&request.arguments, &state.payload_codec)?;
8683 let mut command = serde_json::Map::from_iter([
8684 ("type".to_string(), json!("continue_as_new")),
8685 ("arguments".to_string(), arguments),
8686 ("queue".to_string(), json!(state.task_queue.clone())),
8687 ]);
8688 if let Some(workflow_type) = request.options.workflow_type {
8689 command.insert("workflow_type".to_string(), json!(workflow_type));
8690 }
8691 if let Some(task_queue) = request.options.task_queue {
8692 command.insert("queue".to_string(), json!(task_queue));
8693 }
8694 Ok(Some(Value::Object(command)))
8695 }
8696
8697 fn matched_recorded_pending(&self) -> Result<bool> {
8698 let state = self
8699 .state
8700 .lock()
8701 .map_err(|_| Error::WorkflowStatePoisoned)?;
8702 Ok(state.matched_recorded_pending)
8703 }
8704
8705 fn ensure_history_consumed(&self) -> Result<()> {
8706 let state = self
8707 .state
8708 .lock()
8709 .map_err(|_| Error::WorkflowStatePoisoned)?;
8710 if let Some(command) = state.recorded_commands.get(state.command_cursor) {
8711 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8712 "recorded_commands_unconsumed",
8713 Some(command.sequence()),
8714 Some(command.shape().to_string()),
8715 Some("workflow completion".to_string()),
8716 "workflow completed before consuming all recorded durable commands",
8717 )));
8718 }
8719 if let Some(sequence) = state
8720 .recorded_continue_as_new_sequence
8721 .filter(|_| !state.continue_as_new_consumed)
8722 {
8723 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8724 "recorded_continue_as_new_unconsumed",
8725 Some(sequence),
8726 Some("continue as new".to_string()),
8727 Some("workflow completion".to_string()),
8728 "workflow completed without consuming its recorded continue-as-new transition",
8729 )));
8730 }
8731 Ok(())
8732 }
8733}
8734
8735fn contiguous_message_stream_count(
8736 pending: &[MessageStreamMessage],
8737 cursor: u64,
8738 max_items: usize,
8739) -> usize {
8740 pending
8741 .iter()
8742 .take(max_items)
8743 .enumerate()
8744 .take_while(|(offset, message)| {
8745 u64::try_from(*offset)
8746 .ok()
8747 .and_then(|offset| cursor.checked_add(offset + 1))
8748 == Some(message.position)
8749 })
8750 .count()
8751}
8752
8753fn is_authored_command_open_event(event: &HistoryEvent) -> bool {
8754 matches!(
8755 event.event_type.as_str(),
8756 "ActivityScheduled"
8757 | "TimerScheduled"
8758 | "ChildWorkflowScheduled"
8759 | "SignalWaitOpened"
8760 | "ConditionWaitOpened"
8761 | "SearchAttributesUpserted"
8762 | "SideEffectRecorded"
8763 | "VersionMarkerRecorded"
8764 | "MemoUpserted"
8765 | "WorkflowContinuedAsNew"
8766 )
8767}
8768
8769#[derive(Debug)]
8770struct WorkflowState {
8771 workflow_id: Option<String>,
8772 run_id: Option<String>,
8773 task_queue: String,
8774 payload_codec: String,
8775 history_events: Arc<Vec<HistoryEvent>>,
8776 history_budget: WorkflowHistoryBudget,
8777 cancel_requested: bool,
8778 resume_signal: Option<ResumeSignal>,
8779 recorded_commands: Vec<RecordedCommand>,
8780 selection_markers: Vec<SelectionMarker>,
8781 selection_marker_cursor: usize,
8782 cancelled_selection_members: Vec<SelectionCancellation>,
8783 recorded_continue_as_new_sequence: Option<u64>,
8784 continue_as_new_consumed: bool,
8785 command_cursor: usize,
8786 condition_wait_occurrence_counter: u64,
8787 matched_recorded_pending: bool,
8788 version_markers: HashMap<String, (i32, u64)>,
8789 workflow_command_identity: String,
8790 workflow_stream_command_counter: u64,
8791 commands: Vec<Value>,
8792 message_stream_messages: HashMap<String, Vec<MessageStreamMessage>>,
8793 message_stream_cursors: HashMap<String, u64>,
8794 message_stream_waits: HashMap<String, u64>,
8795}
8796
8797impl WorkflowState {
8798 #[cfg(test)]
8799 fn new(
8800 history: Vec<HistoryEvent>,
8801 task_queue: String,
8802 payload_codec: String,
8803 resume_signal: Option<ResumeSignal>,
8804 ) -> Result<Self> {
8805 Self::new_with_identity(
8806 history,
8807 None,
8808 None,
8809 task_queue,
8810 payload_codec,
8811 resume_signal,
8812 )
8813 }
8814
8815 fn new_with_identity(
8816 history: Vec<HistoryEvent>,
8817 workflow_id: Option<String>,
8818 run_id: Option<String>,
8819 task_queue: String,
8820 payload_codec: String,
8821 resume_signal: Option<ResumeSignal>,
8822 ) -> Result<Self> {
8823 let recorded_commands = recorded_commands(
8824 &history,
8825 &payload_codec,
8826 WorkflowIdentity {
8827 workflow_id: workflow_id.clone(),
8828 run_id: run_id.clone(),
8829 },
8830 )?;
8831 let selection_markers = recorded_selection_markers(&history)?;
8832 let cancelled_selection_members = recorded_selection_cancellations(&history)?;
8833 let recorded_continue_as_new = history
8834 .iter()
8835 .filter(|event| event.event_type == "WorkflowContinuedAsNew")
8836 .collect::<Vec<_>>();
8837 if recorded_continue_as_new.len() > 1 {
8838 return Err(invalid_recorded_history(
8839 "duplicate_continue_as_new_transition",
8840 recorded_continue_as_new
8841 .last()
8842 .and_then(|event| durable_event_sequence(event))
8843 .unwrap_or(0),
8844 "one WorkflowContinuedAsNew event",
8845 &format!(
8846 "{} WorkflowContinuedAsNew events",
8847 recorded_continue_as_new.len()
8848 ),
8849 "workflow history records one continue-as-new transition more than once",
8850 ));
8851 }
8852 let recorded_continue_as_new_sequence = recorded_continue_as_new
8853 .first()
8854 .map(|event| {
8855 durable_event_sequence(event).ok_or_else(|| {
8856 Error::NonDeterministicReplay(ReplayFailure::new(
8857 "continue_as_new_sequence_missing",
8858 None,
8859 Some("recorded transition sequence".to_string()),
8860 Some("missing sequence".to_string()),
8861 "WorkflowContinuedAsNew history is missing its recorded sequence",
8862 ))
8863 })
8864 })
8865 .transpose()?;
8866 let mut message_stream_cursors = HashMap::new();
8867 for event in &history {
8868 if !matches!(
8869 event.event_type.as_str(),
8870 "SignalReceived" | "SignalApplied"
8871 ) || event.payload.get("signal_name").and_then(Value::as_str)
8872 != Some(MESSAGE_STREAM_SIGNAL)
8873 {
8874 continue;
8875 }
8876 let arguments = decode_signal_event_arguments(event, &payload_codec)?;
8877 if arguments.len() != 1 {
8878 continue;
8879 }
8880 let envelope = arguments[0].clone().into_json()?;
8881 let Some(envelope) = envelope.as_object() else {
8882 continue;
8883 };
8884 if envelope.get("schema").and_then(Value::as_str) != Some(MESSAGE_STREAM_CURSOR_SCHEMA)
8885 {
8886 continue;
8887 }
8888 let Some(stream_name) = envelope.get("stream_name").and_then(Value::as_str) else {
8889 continue;
8890 };
8891 let Some(through_position) = envelope.get("through_position").and_then(Value::as_u64)
8892 else {
8893 continue;
8894 };
8895 let cursor = message_stream_cursors
8896 .entry(stream_name.to_string())
8897 .or_insert(0);
8898 *cursor = (*cursor).max(through_position);
8899 }
8900 let event_count = u64::try_from(history.len()).unwrap_or(u64::MAX);
8901 let cancel_requested = history.iter().any(|event| {
8902 matches!(
8903 event.event_type.as_str(),
8904 "WorkflowCancellationRequested" | "WorkflowCancelRequested"
8905 )
8906 });
8907 Ok(Self {
8908 workflow_command_identity: String::new(),
8909 workflow_stream_command_counter: 0,
8910 workflow_id,
8911 run_id,
8912 task_queue,
8913 payload_codec,
8914 history_events: Arc::new(history),
8915 history_budget: WorkflowHistoryBudget {
8916 event_count,
8917 ..WorkflowHistoryBudget::default()
8918 },
8919 cancel_requested,
8920 resume_signal,
8921 recorded_commands,
8922 selection_markers,
8923 selection_marker_cursor: 0,
8924 cancelled_selection_members,
8925 recorded_continue_as_new_sequence,
8926 continue_as_new_consumed: false,
8927 command_cursor: 0,
8928 condition_wait_occurrence_counter: 0,
8929 matched_recorded_pending: false,
8930 version_markers: HashMap::new(),
8931 commands: Vec::new(),
8932 message_stream_messages: HashMap::new(),
8933 message_stream_cursors,
8934 message_stream_waits: HashMap::new(),
8935 })
8936 }
8937}
8938
8939enum MessageStreamDelivery {
8940 Message(MessageStreamMessage),
8941 Cursor {
8942 stream_name: String,
8943 through_position: u64,
8944 },
8945}
8946
8947fn decode_message_stream_delivery(arguments: Vec<Value>) -> Result<Option<MessageStreamDelivery>> {
8948 if arguments.len() != 1 {
8949 return Ok(None);
8950 }
8951 let envelope = arguments
8952 .into_iter()
8953 .next()
8954 .expect("one argument was checked");
8955 let Some(envelope) = envelope.as_object() else {
8956 return Ok(None);
8957 };
8958 let Some(stream_name) = envelope.get("stream_name").and_then(Value::as_str) else {
8959 return Ok(None);
8960 };
8961 if envelope.get("schema").and_then(Value::as_str) == Some(MESSAGE_STREAM_CURSOR_SCHEMA) {
8962 let Some(through_position) = envelope.get("through_position").and_then(Value::as_u64)
8963 else {
8964 return Ok(None);
8965 };
8966 return Ok(Some(MessageStreamDelivery::Cursor {
8967 stream_name: stream_name.to_string(),
8968 through_position,
8969 }));
8970 }
8971 if envelope.get("schema").and_then(Value::as_str) != Some(MESSAGE_STREAM_SCHEMA) {
8972 return Ok(None);
8973 }
8974 let Some(message_id) = envelope.get("message_id").and_then(Value::as_str) else {
8975 return Ok(None);
8976 };
8977 let Some(position) = envelope
8978 .get("position")
8979 .and_then(Value::as_u64)
8980 .filter(|value| *value > 0)
8981 else {
8982 return Ok(None);
8983 };
8984 let Some(payload_envelope) = envelope.get("payload_envelope") else {
8985 return Ok(None);
8986 };
8987 let Ok(payload_envelope) = serde_json::from_value::<PayloadEnvelope>(payload_envelope.clone())
8988 else {
8989 return Ok(None);
8990 };
8991 let decoded = decode_avro_value(&payload_envelope)?;
8992 let AvroValue::Array(values) = decoded else {
8993 return Ok(None);
8994 };
8995 Ok(Some(MessageStreamDelivery::Message(MessageStreamMessage {
8996 stream_name: stream_name.to_string(),
8997 message_id: message_id.to_string(),
8998 position,
8999 arguments: values,
9000 })))
9001}
9002
9003#[derive(Clone, Debug)]
9004enum RecordedCommand {
9005 Activity {
9006 sequence: u64,
9007 activity_type: Option<String>,
9008 options: Option<RecordedActivityOptions>,
9009 outcome: Option<ActivityOutcome>,
9010 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9011 },
9012 Timer {
9013 sequence: u64,
9014 delay_seconds: u64,
9015 fired: bool,
9016 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9017 },
9018 ChildWorkflow {
9019 sequence: u64,
9020 workflow_type: Option<String>,
9021 outcome: Option<ChildWorkflowOutcome>,
9022 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9023 },
9024 SignalWait {
9025 sequence: u64,
9026 signal_name: String,
9027 value: Option<Vec<AvroValue>>,
9028 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9029 },
9030 ConditionWait {
9031 sequence: u64,
9032 occurrence_id: String,
9033 condition_key: Option<String>,
9034 predicate_identity: String,
9035 timeout_seconds: Option<u64>,
9036 result: Option<ConditionWaitResult>,
9037 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9038 },
9039 SearchAttributes {
9040 sequence: u64,
9041 attributes: Value,
9042 attribute_types: RecordedSnapshotValue<BTreeMap<String, String>>,
9043 },
9044 SideEffect {
9045 sequence: u64,
9046 value: AvroValue,
9047 },
9048 VersionMarker {
9049 sequence: u64,
9050 change_id: String,
9051 version: i32,
9052 },
9053 Memo {
9054 sequence: u64,
9055 entries: AvroValue,
9056 },
9057}
9058
9059#[derive(Clone, Debug, PartialEq, Eq)]
9060struct SelectionMarker {
9061 selection_group_id: String,
9062 selection_group_base_sequence: u64,
9063 selection_group_size: usize,
9064 member_key: SelectionKey,
9065 member_index: usize,
9066 member_base_sequence: u64,
9067 member_size: usize,
9068 operation_kind: String,
9069 operation_identity: String,
9070 outcome: String,
9071 resolution_event_id: String,
9072 resolution_event_type: String,
9073}
9074
9075#[derive(Clone, Debug, PartialEq, Eq)]
9076struct SelectionCancellation {
9077 selection_group_id: String,
9078 member_key: SelectionKey,
9079 member_index: usize,
9080 member_base_sequence: u64,
9081 member_size: usize,
9082 operation_kind: String,
9083 operation_identity: String,
9084}
9085
9086fn recorded_selection_markers(events: &[HistoryEvent]) -> Result<Vec<SelectionMarker>> {
9087 let mut markers: Vec<SelectionMarker> = Vec::new();
9088 for event in events
9089 .iter()
9090 .filter(|event| event.event_type == "SelectionResolved")
9091 {
9092 let payload = &event.payload;
9093 let base_sequence = required_selection_u64(payload, "selection_group_base_sequence")?;
9094 let group_size = required_selection_usize(payload, "selection_group_size")?;
9095 let member_base_sequence = required_selection_u64(payload, "member_base_sequence")?;
9096 let member_size = required_selection_usize(payload, "member_size")?;
9097 let member_index = required_selection_usize_allow_zero(payload, "member_index")?;
9098 let group_id = payload_string(payload, "selection_group_id").ok_or_else(|| {
9099 invalid_recorded_history(
9100 "selection_marker_invalid",
9101 base_sequence,
9102 "non-empty selection_group_id",
9103 &payload.to_string(),
9104 "selection winner history is missing its durable group identity",
9105 )
9106 })?;
9107 let expected_group_id = format!("select-calls:{base_sequence}:{group_size}");
9108 if group_id != expected_group_id {
9109 return Err(invalid_recorded_history(
9110 "selection_marker_invalid",
9111 base_sequence,
9112 &expected_group_id,
9113 &group_id,
9114 "selection winner history contains an incompatible group identity",
9115 ));
9116 }
9117 let group_end = base_sequence
9118 .checked_add(u64::try_from(group_size).unwrap_or(u64::MAX))
9119 .unwrap_or(u64::MAX);
9120 let member_end = member_base_sequence
9121 .checked_add(u64::try_from(member_size).unwrap_or(u64::MAX))
9122 .unwrap_or(u64::MAX);
9123 if member_index >= group_size
9124 || member_base_sequence < base_sequence
9125 || member_end > group_end
9126 {
9127 return Err(invalid_recorded_history(
9128 "selection_marker_invalid",
9129 base_sequence,
9130 "winner member within selection group bounds",
9131 &payload.to_string(),
9132 "selection winner history contains an invalid member range",
9133 ));
9134 }
9135 let operation_kind = payload_string(payload, "operation_kind").ok_or_else(|| {
9136 invalid_recorded_history(
9137 "selection_marker_invalid",
9138 base_sequence,
9139 "selection operation kind",
9140 &payload.to_string(),
9141 "selection winner history is missing its operation kind",
9142 )
9143 })?;
9144 if !matches!(
9145 operation_kind.as_str(),
9146 "activity" | "child" | "timer" | "signal" | "condition" | "group"
9147 ) {
9148 return Err(invalid_recorded_history(
9149 "selection_marker_invalid",
9150 base_sequence,
9151 "activity, child, timer, signal, condition, or group",
9152 &operation_kind,
9153 "selection winner history contains an unsupported operation kind",
9154 ));
9155 }
9156 let operation_identity =
9157 payload_string(payload, "operation_identity").ok_or_else(|| {
9158 invalid_recorded_history(
9159 "selection_marker_invalid",
9160 base_sequence,
9161 "non-empty operation identity",
9162 &payload.to_string(),
9163 "selection winner history is missing its durable operation identity",
9164 )
9165 })?;
9166 let outcome = payload_string(payload, "outcome").ok_or_else(|| {
9167 invalid_recorded_history(
9168 "selection_marker_invalid",
9169 base_sequence,
9170 "completed or failed selection outcome",
9171 &payload.to_string(),
9172 "selection winner history is missing its outcome",
9173 )
9174 })?;
9175 if !matches!(outcome.as_str(), "completed" | "failed") {
9176 return Err(invalid_recorded_history(
9177 "selection_marker_invalid",
9178 base_sequence,
9179 "completed or failed selection outcome",
9180 &outcome,
9181 "selection winner history contains an unsupported outcome",
9182 ));
9183 }
9184 let marker = SelectionMarker {
9185 selection_group_id: group_id,
9186 selection_group_base_sequence: base_sequence,
9187 selection_group_size: group_size,
9188 member_key: selection_key_from_value(payload.get("member_key"), base_sequence)?,
9189 member_index,
9190 member_base_sequence,
9191 member_size,
9192 operation_kind,
9193 operation_identity,
9194 outcome,
9195 resolution_event_id: payload_string(payload, "resolution_event_id").ok_or_else(
9196 || {
9197 invalid_recorded_history(
9198 "selection_marker_invalid",
9199 base_sequence,
9200 "durable resolution_event_id",
9201 &payload.to_string(),
9202 "selection winner history is missing its terminal event identity",
9203 )
9204 },
9205 )?,
9206 resolution_event_type: payload_string(payload, "resolution_event_type").ok_or_else(
9207 || {
9208 invalid_recorded_history(
9209 "selection_marker_invalid",
9210 base_sequence,
9211 "durable resolution_event_type",
9212 &payload.to_string(),
9213 "selection winner history is missing its terminal event type",
9214 )
9215 },
9216 )?,
9217 };
9218 if let Some(existing) = markers
9219 .iter()
9220 .find(|existing| existing.selection_group_id == marker.selection_group_id)
9221 {
9222 if existing != &marker {
9223 return Err(invalid_recorded_history(
9224 "selection_marker_conflict",
9225 base_sequence,
9226 &format!("one winner for {}", marker.selection_group_id),
9227 &payload.to_string(),
9228 "selection history records conflicting winners for one durable group",
9229 ));
9230 }
9231 continue;
9232 }
9233 markers.push(marker);
9234 }
9235 Ok(markers)
9236}
9237
9238fn recorded_selection_cancellations(events: &[HistoryEvent]) -> Result<Vec<SelectionCancellation>> {
9239 let mut cancelled: Vec<SelectionCancellation> = Vec::new();
9240 for event in events
9241 .iter()
9242 .filter(|event| event.event_type == "SelectionOperationCancelled")
9243 {
9244 let group_id = payload_string(&event.payload, "selection_group_id").ok_or_else(|| {
9245 invalid_recorded_history(
9246 "selection_cancellation_invalid",
9247 0,
9248 "non-empty selection_group_id",
9249 &event.payload.to_string(),
9250 "selection cancellation history is missing its group identity",
9251 )
9252 })?;
9253 let member_base_sequence = required_selection_u64(&event.payload, "member_base_sequence")?;
9254 let marker = SelectionCancellation {
9255 selection_group_id: group_id,
9256 member_key: selection_key_from_value(
9257 event.payload.get("member_key"),
9258 member_base_sequence,
9259 )?,
9260 member_index: required_selection_usize_allow_zero(&event.payload, "member_index")?,
9261 member_base_sequence,
9262 member_size: required_selection_usize(&event.payload, "member_size")?,
9263 operation_kind: payload_string(&event.payload, "operation_kind").ok_or_else(|| {
9264 invalid_recorded_history(
9265 "selection_cancellation_invalid",
9266 member_base_sequence,
9267 "selection operation kind",
9268 &event.payload.to_string(),
9269 "selection cancellation is missing its operation kind",
9270 )
9271 })?,
9272 operation_identity: payload_string(&event.payload, "operation_identity").ok_or_else(
9273 || {
9274 invalid_recorded_history(
9275 "selection_cancellation_invalid",
9276 member_base_sequence,
9277 "selection operation identity",
9278 &event.payload.to_string(),
9279 "selection cancellation is missing its operation identity",
9280 )
9281 },
9282 )?,
9283 };
9284 if let Some(existing) = cancelled.iter().find(|recorded| {
9285 recorded.selection_group_id == marker.selection_group_id
9286 && recorded.member_base_sequence == marker.member_base_sequence
9287 }) {
9288 if existing != &marker {
9289 return Err(invalid_recorded_history(
9290 "selection_cancellation_conflict",
9291 member_base_sequence,
9292 "one stable SelectionOperationCancelled marker",
9293 &event.payload.to_string(),
9294 "selection cancellation history contains conflicting member metadata",
9295 ));
9296 }
9297 } else {
9298 cancelled.push(marker);
9299 }
9300 }
9301 Ok(cancelled)
9302}
9303
9304fn required_selection_u64(payload: &Value, field: &str) -> Result<u64> {
9305 payload
9306 .get(field)
9307 .and_then(value_as_u64)
9308 .filter(|value| *value > 0)
9309 .ok_or_else(|| {
9310 invalid_recorded_history(
9311 "selection_marker_invalid",
9312 0,
9313 &format!("positive integer {field}"),
9314 &payload.to_string(),
9315 "selection history contains invalid durable identity metadata",
9316 )
9317 })
9318}
9319
9320fn required_selection_usize(payload: &Value, field: &str) -> Result<usize> {
9321 required_selection_usize_allow_zero(payload, field).and_then(|value| {
9322 if value > 0 {
9323 Ok(value)
9324 } else {
9325 Err(invalid_recorded_history(
9326 "selection_marker_invalid",
9327 0,
9328 &format!("positive integer {field}"),
9329 &payload.to_string(),
9330 "selection history contains invalid durable identity metadata",
9331 ))
9332 }
9333 })
9334}
9335
9336fn required_selection_usize_allow_zero(payload: &Value, field: &str) -> Result<usize> {
9337 payload
9338 .get(field)
9339 .and_then(value_as_u64)
9340 .and_then(|value| usize::try_from(value).ok())
9341 .ok_or_else(|| {
9342 invalid_recorded_history(
9343 "selection_marker_invalid",
9344 0,
9345 &format!("non-negative integer {field}"),
9346 &payload.to_string(),
9347 "selection history contains invalid durable identity metadata",
9348 )
9349 })
9350}
9351
9352#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9353struct RecordedActivityOptions {
9354 task_queue: RecordedSnapshotValue<Option<String>>,
9355 execution_mode: RecordedSnapshotValue<Option<String>>,
9356 retry_policy: ActivityRetrySnapshot,
9357}
9358
9359#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9360enum RecordedSnapshotValue<T> {
9361 Unknown,
9363 Known(T),
9364}
9365
9366impl<T: PartialEq> RecordedSnapshotValue<T> {
9367 fn matches_current(&self, current: &Self) -> bool {
9368 match self {
9369 Self::Unknown => true,
9370 Self::Known(recorded) => matches!(current, Self::Known(value) if value == recorded),
9371 }
9372 }
9373}
9374
9375#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9376struct ActivityRetrySnapshot {
9377 snapshot_version: RecordedSnapshotValue<Option<u64>>,
9378 max_attempts: RecordedSnapshotValue<Option<u64>>,
9379 backoff_seconds: RecordedSnapshotValue<Vec<u64>>,
9380 start_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
9381 schedule_to_start_timeout: RecordedSnapshotValue<Option<u64>>,
9382 schedule_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
9383 heartbeat_timeout: RecordedSnapshotValue<Option<u64>>,
9384 non_retryable_error_types: RecordedSnapshotValue<Vec<String>>,
9385}
9386
9387impl ActivityRetrySnapshot {
9388 fn matches_current(&self, current: &Self) -> bool {
9389 self.snapshot_version
9390 .matches_current(¤t.snapshot_version)
9391 && self.max_attempts.matches_current(¤t.max_attempts)
9392 && self
9393 .backoff_seconds
9394 .matches_current(¤t.backoff_seconds)
9395 && self
9396 .start_to_close_timeout
9397 .matches_current(¤t.start_to_close_timeout)
9398 && self
9399 .schedule_to_start_timeout
9400 .matches_current(¤t.schedule_to_start_timeout)
9401 && self
9402 .schedule_to_close_timeout
9403 .matches_current(¤t.schedule_to_close_timeout)
9404 && self
9405 .heartbeat_timeout
9406 .matches_current(¤t.heartbeat_timeout)
9407 && self
9408 .non_retryable_error_types
9409 .matches_current(¤t.non_retryable_error_types)
9410 }
9411}
9412
9413fn recorded_optional_u64(
9414 object: Option<&serde_json::Map<String, Value>>,
9415 field: &str,
9416) -> RecordedSnapshotValue<Option<u64>> {
9417 match object.and_then(|object| object.get(field)) {
9418 None => RecordedSnapshotValue::Unknown,
9419 Some(Value::Null) => RecordedSnapshotValue::Known(None),
9420 Some(value) => RecordedSnapshotValue::Known(value_as_u64(value)),
9421 }
9422}
9423
9424fn recorded_optional_string(
9425 object: &serde_json::Map<String, Value>,
9426 field: &str,
9427) -> RecordedSnapshotValue<Option<String>> {
9428 match object.get(field) {
9429 None => RecordedSnapshotValue::Unknown,
9430 Some(Value::Null) => RecordedSnapshotValue::Known(None),
9431 Some(value) => RecordedSnapshotValue::Known(value.as_str().map(str::to_string)),
9432 }
9433}
9434
9435fn recorded_activity_retry_snapshot(policy: Option<&Value>) -> ActivityRetrySnapshot {
9436 let policy = policy.and_then(Value::as_object);
9437 let backoff_seconds = policy
9438 .and_then(|policy| policy.get("backoff_seconds"))
9439 .and_then(Value::as_array)
9440 .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
9441 .map_or(RecordedSnapshotValue::Unknown, RecordedSnapshotValue::Known);
9442 let mut non_retryable_error_types = Vec::new();
9443 for error_type in policy
9444 .and_then(|policy| policy.get("non_retryable_error_types"))
9445 .and_then(Value::as_array)
9446 .into_iter()
9447 .flatten()
9448 .filter_map(Value::as_str)
9449 .map(str::trim)
9450 .filter(|error_type| !error_type.is_empty())
9451 {
9452 if !non_retryable_error_types
9453 .iter()
9454 .any(|recorded| recorded == error_type)
9455 {
9456 non_retryable_error_types.push(error_type.to_string());
9457 }
9458 }
9459
9460 ActivityRetrySnapshot {
9461 snapshot_version: recorded_optional_u64(policy, "snapshot_version"),
9462 max_attempts: recorded_optional_u64(policy, "max_attempts"),
9463 backoff_seconds,
9464 start_to_close_timeout: recorded_optional_u64(policy, "start_to_close_timeout"),
9465 schedule_to_start_timeout: recorded_optional_u64(policy, "schedule_to_start_timeout"),
9466 schedule_to_close_timeout: recorded_optional_u64(policy, "schedule_to_close_timeout"),
9467 heartbeat_timeout: recorded_optional_u64(policy, "heartbeat_timeout"),
9468 non_retryable_error_types: if policy
9469 .is_some_and(|policy| policy.contains_key("non_retryable_error_types"))
9470 {
9471 RecordedSnapshotValue::Known(non_retryable_error_types)
9472 } else {
9473 RecordedSnapshotValue::Unknown
9474 },
9475 }
9476}
9477
9478fn current_activity_retry_snapshot(options: &ValidatedActivityOptions) -> ActivityRetrySnapshot {
9479 let policy = options.retry_policy.as_ref();
9480 let max_attempts = match policy.and_then(|policy| policy.get("max_attempts")) {
9481 Some(Value::Null) => None,
9482 Some(value) => value_as_u64(value),
9483 None => Some(1),
9484 };
9485 let backoff_seconds = policy
9486 .and_then(|policy| policy.get("backoff_seconds"))
9487 .and_then(Value::as_array)
9488 .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
9489 .unwrap_or_default();
9490 let non_retryable_error_types = policy
9491 .and_then(|policy| policy.get("non_retryable_error_types"))
9492 .and_then(Value::as_array)
9493 .into_iter()
9494 .flatten()
9495 .filter_map(Value::as_str)
9496 .map(str::to_string)
9497 .collect();
9498
9499 ActivityRetrySnapshot {
9500 snapshot_version: RecordedSnapshotValue::Known(Some(1)),
9501 max_attempts: RecordedSnapshotValue::Known(max_attempts),
9502 backoff_seconds: RecordedSnapshotValue::Known(backoff_seconds),
9503 start_to_close_timeout: RecordedSnapshotValue::Known(options.start_to_close_timeout),
9504 schedule_to_start_timeout: RecordedSnapshotValue::Known(options.schedule_to_start_timeout),
9505 schedule_to_close_timeout: RecordedSnapshotValue::Known(options.schedule_to_close_timeout),
9506 heartbeat_timeout: RecordedSnapshotValue::Known(options.heartbeat_timeout),
9507 non_retryable_error_types: RecordedSnapshotValue::Known(non_retryable_error_types),
9508 }
9509}
9510
9511fn activity_options_description(options: &RecordedActivityOptions) -> String {
9512 serde_json::to_string(options).unwrap_or_else(|_| format!("{options:?}"))
9513}
9514
9515impl RecordedCommand {
9516 fn sequence(&self) -> u64 {
9517 match self {
9518 Self::Activity { sequence, .. }
9519 | Self::Timer { sequence, .. }
9520 | Self::ChildWorkflow { sequence, .. }
9521 | Self::SignalWait { sequence, .. }
9522 | Self::ConditionWait { sequence, .. }
9523 | Self::SearchAttributes { sequence, .. }
9524 | Self::SideEffect { sequence, .. }
9525 | Self::VersionMarker { sequence, .. }
9526 | Self::Memo { sequence, .. } => *sequence,
9527 }
9528 }
9529
9530 fn shape(&self) -> &'static str {
9531 match self {
9532 Self::Activity { .. } => "activity",
9533 Self::Timer { .. } => "timer",
9534 Self::ChildWorkflow { .. } => "child workflow",
9535 Self::SignalWait { .. } => "signal wait",
9536 Self::ConditionWait { .. } => "condition wait",
9537 Self::SearchAttributes { .. } => "search-attribute update",
9538 Self::SideEffect { .. } => "side effect",
9539 Self::VersionMarker { .. } => "version marker",
9540 Self::Memo { .. } => "memo upsert",
9541 }
9542 }
9543}
9544
9545fn ensure_version_supported(
9546 change_id: &str,
9547 version: i32,
9548 min_supported: i32,
9549 max_supported: i32,
9550 sequence: u64,
9551) -> Result<()> {
9552 if (min_supported..=max_supported).contains(&version) {
9553 return Ok(());
9554 }
9555 Err(Error::NonDeterministicReplay(ReplayFailure::new(
9556 "version_marker_incompatible_range",
9557 (sequence != 0).then_some(sequence),
9558 Some(format!("{min_supported}..={max_supported}")),
9559 Some(format!("{change_id}:{version}")),
9560 "recorded workflow version is outside the range supported by current code",
9561 )))
9562}
9563
9564#[derive(Clone, Debug)]
9565struct ResumeSignal {
9566 signal_name: String,
9567 arguments: Vec<AvroValue>,
9568}
9569
9570const MAX_PARALLEL_OPERATIONS: usize = 1000;
9571
9572fn parallel_group_prefix(kind: &str) -> &'static str {
9573 match kind {
9574 "activity" => "parallel-activities",
9575 "child" => "parallel-children",
9576 "timer" => "parallel-timers",
9577 _ => "parallel-calls",
9578 }
9579}
9580
9581fn parallel_group_entry(
9582 base_sequence: u64,
9583 size: usize,
9584 index: usize,
9585 kind: &str,
9586) -> ParallelGroupMetadata {
9587 ParallelGroupMetadata {
9588 parallel_group_id: format!("{}:{base_sequence}:{size}", parallel_group_prefix(kind)),
9589 parallel_group_kind: kind.to_string(),
9590 parallel_group_base_sequence: base_sequence,
9591 parallel_group_size: size,
9592 parallel_group_index: index,
9593 parallel_group_mode: None,
9594 selection_member_key: None,
9595 selection_member_index: None,
9596 selection_member_base_sequence: None,
9597 selection_member_size: None,
9598 selection_member_kind: None,
9599 }
9600}
9601
9602struct SelectionMemberMetadata {
9603 key: SelectionKey,
9604 index: usize,
9605 base_sequence: u64,
9606 size: usize,
9607 kind: String,
9608}
9609
9610fn selection_group_entry(
9611 base_sequence: u64,
9612 size: usize,
9613 index: usize,
9614 kind: &str,
9615 member: &SelectionMemberMetadata,
9616) -> ParallelGroupMetadata {
9617 ParallelGroupMetadata {
9618 parallel_group_id: format!("select-calls:{base_sequence}:{size}"),
9619 parallel_group_kind: kind.to_string(),
9620 parallel_group_base_sequence: base_sequence,
9621 parallel_group_size: size,
9622 parallel_group_index: index,
9623 parallel_group_mode: Some("select".to_string()),
9624 selection_member_key: Some(member.key.clone()),
9625 selection_member_index: Some(member.index),
9626 selection_member_base_sequence: Some(member.base_sequence),
9627 selection_member_size: Some(member.size),
9628 selection_member_kind: Some(member.kind.clone()),
9629 }
9630}
9631
9632fn apply_parallel_group_path(
9633 command: &mut serde_json::Map<String, Value>,
9634 path: &[ParallelGroupMetadata],
9635) {
9636 let Some(inner) = path.last() else {
9637 return;
9638 };
9639 command.insert(
9640 "parallel_group_id".to_string(),
9641 json!(inner.parallel_group_id),
9642 );
9643 command.insert(
9644 "parallel_group_kind".to_string(),
9645 json!(inner.parallel_group_kind),
9646 );
9647 command.insert(
9648 "parallel_group_base_sequence".to_string(),
9649 json!(inner.parallel_group_base_sequence),
9650 );
9651 command.insert(
9652 "parallel_group_size".to_string(),
9653 json!(inner.parallel_group_size),
9654 );
9655 command.insert(
9656 "parallel_group_index".to_string(),
9657 json!(inner.parallel_group_index),
9658 );
9659 if let Some(mode) = &inner.parallel_group_mode {
9660 command.insert("parallel_group_mode".to_string(), json!(mode));
9661 }
9662 if let Some(key) = &inner.selection_member_key {
9663 command.insert("selection_member_key".to_string(), json!(key));
9664 }
9665 if let Some(index) = inner.selection_member_index {
9666 command.insert("selection_member_index".to_string(), json!(index));
9667 }
9668 if let Some(base_sequence) = inner.selection_member_base_sequence {
9669 command.insert(
9670 "selection_member_base_sequence".to_string(),
9671 json!(base_sequence),
9672 );
9673 }
9674 if let Some(size) = inner.selection_member_size {
9675 command.insert("selection_member_size".to_string(), json!(size));
9676 }
9677 if let Some(kind) = &inner.selection_member_kind {
9678 command.insert("selection_member_kind".to_string(), json!(kind));
9679 }
9680 command.insert("parallel_group_path".to_string(), json!(path));
9681}
9682
9683fn ensure_parallel_path_matches(
9684 sequence: u64,
9685 recorded: Option<&[ParallelGroupMetadata]>,
9686 expected: &[ParallelGroupMetadata],
9687) -> Result<()> {
9688 match (recorded, expected.is_empty()) {
9689 (None, true) => Ok(()),
9690 (Some(recorded), false) if recorded == expected => Ok(()),
9691 (None, false) => Err(invalid_recorded_history(
9692 "parallel_group_metadata_missing",
9693 sequence,
9694 &serde_json::to_string(expected).unwrap_or_default(),
9695 "<missing>",
9696 "recorded parallel member is missing its durable group path",
9697 )),
9698 (Some(recorded), true) => Err(invalid_recorded_history(
9699 "parallel_group_shape_mismatch",
9700 sequence,
9701 "sequential command",
9702 &serde_json::to_string(recorded).unwrap_or_default(),
9703 "recorded command belonged to a parallel group but current code schedules it sequentially",
9704 )),
9705 (Some(recorded), false) => Err(invalid_recorded_history(
9706 "parallel_group_shape_mismatch",
9707 sequence,
9708 &serde_json::to_string(recorded).unwrap_or_default(),
9709 &serde_json::to_string(expected).unwrap_or_default(),
9710 "recorded parallel-group identity or path changed during replay",
9711 )),
9712 }
9713}
9714
9715#[derive(Clone, Debug)]
9716enum ParallelShape {
9717 Leaf,
9718 Group(Vec<ParallelShape>),
9719}
9720
9721struct ParallelDescriptor {
9722 operation: ParallelOperation,
9723 offset: usize,
9724 member_path: Vec<usize>,
9725 group_path: Vec<ParallelGroupMetadata>,
9726}
9727
9728fn parallel_leaf_count(operations: &[ParallelOperation]) -> usize {
9729 operations
9730 .iter()
9731 .map(|operation| match operation {
9732 ParallelOperation::Group(children) => parallel_leaf_count(children),
9733 _ => 1,
9734 })
9735 .sum()
9736}
9737
9738fn parallel_operation_kind(operation: &ParallelOperation) -> Option<&'static str> {
9739 match operation {
9740 ParallelOperation::Activity { .. } => Some("activity"),
9741 ParallelOperation::ChildWorkflow { .. } => Some("child"),
9742 ParallelOperation::Timer(_) => Some("timer"),
9743 ParallelOperation::Signal(_) => Some("signal"),
9744 ParallelOperation::Condition { .. } => Some("condition"),
9745 ParallelOperation::Group(children) => parallel_group_kind(children),
9746 }
9747}
9748
9749fn parallel_group_kind(operations: &[ParallelOperation]) -> Option<&'static str> {
9750 let mut kind = None;
9751 for operation in operations {
9752 let Some(operation_kind) = parallel_operation_kind(operation) else {
9753 continue;
9754 };
9755 match kind {
9756 None => kind = Some(operation_kind),
9757 Some(current) if current == operation_kind => {}
9758 Some(_) => return Some("mixed"),
9759 }
9760 }
9761 kind
9762}
9763
9764fn validate_parallel_operations(
9765 operations: &[ParallelOperation],
9766 member_path: &mut Vec<usize>,
9767 root: bool,
9768) -> Result<()> {
9769 let leaves = parallel_leaf_count(operations);
9770 if leaves > MAX_PARALLEL_OPERATIONS {
9771 return Err(Error::InvalidParallelGroup(ParallelGroupError {
9772 reason: "fan_out_limit_exceeded",
9773 member_path: member_path.clone(),
9774 message: format!(
9775 "group contains {leaves} durable leaves; the limit is {MAX_PARALLEL_OPERATIONS}"
9776 ),
9777 }));
9778 }
9779 if !root && operations.is_empty() {
9780 return Err(Error::InvalidParallelGroup(ParallelGroupError {
9781 reason: "nested_group_empty",
9782 member_path: member_path.clone(),
9783 message: "a nested group must contain at least one durable leaf".to_string(),
9784 }));
9785 }
9786
9787 for (index, operation) in operations.iter().enumerate() {
9788 member_path.push(index);
9789 match operation {
9790 ParallelOperation::Activity {
9791 options, arguments, ..
9792 } => {
9793 options
9794 .validate()
9795 .map_err(|error| Error::InvalidActivityOptions(error))?;
9796 if let Err(error) = arguments {
9797 return Err(Error::InvalidParallelGroup(ParallelGroupError {
9798 reason: "arguments_invalid",
9799 member_path: member_path.clone(),
9800 message: error.to_string(),
9801 }));
9802 }
9803 }
9804 ParallelOperation::ChildWorkflow {
9805 options, arguments, ..
9806 } => {
9807 validate_parallel_child_options(options)?;
9808 if let Err(error) = arguments {
9809 return Err(Error::InvalidParallelGroup(ParallelGroupError {
9810 reason: "arguments_invalid",
9811 member_path: member_path.clone(),
9812 message: error.to_string(),
9813 }));
9814 }
9815 }
9816 ParallelOperation::Timer(duration)
9817 if duration.as_secs() == u64::MAX && duration.subsec_nanos() > 0 =>
9818 {
9819 return Err(Error::TimerDurationOverflow);
9820 }
9821 ParallelOperation::Timer(_) => {}
9822 ParallelOperation::Signal(signal_name) => {
9823 validate_user_signal_name(signal_name)?;
9824 if signal_name.trim().is_empty() {
9825 return Err(Error::InvalidParallelGroup(ParallelGroupError {
9826 reason: "signal_name_empty",
9827 member_path: member_path.clone(),
9828 message: "signal wait name must not be empty".to_string(),
9829 }));
9830 }
9831 }
9832 ParallelOperation::Condition { options, .. } => {
9833 options.validate()?;
9834 }
9835 ParallelOperation::Group(children) => {
9836 validate_parallel_operations(children, member_path, false)?;
9837 }
9838 }
9839 member_path.pop();
9840 }
9841 Ok(())
9842}
9843
9844fn validate_parallel_child_options(options: &ChildWorkflowOptions) -> Result<()> {
9845 if options.task_queue.trim().is_empty() {
9846 return Err(Error::InvalidChildWorkflowOptions(
9847 "task_queue must not be empty".to_string(),
9848 ));
9849 }
9850 for (name, value) in [
9851 (
9852 "execution_timeout_seconds",
9853 options.execution_timeout_seconds,
9854 ),
9855 ("run_timeout_seconds", options.run_timeout_seconds),
9856 ] {
9857 if value == Some(0) {
9858 return Err(Error::InvalidChildWorkflowOptions(format!(
9859 "{name} must be at least 1"
9860 )));
9861 }
9862 }
9863 if options
9864 .retry_policy
9865 .as_ref()
9866 .is_some_and(|policy| policy.max_attempts == Some(0))
9867 {
9868 return Err(Error::InvalidChildWorkflowOptions(
9869 "retry_policy.max_attempts must be at least 1".to_string(),
9870 ));
9871 }
9872 Ok(())
9873}
9874
9875fn parallel_shape(operations: &[ParallelOperation]) -> ParallelShape {
9876 ParallelShape::Group(
9877 operations
9878 .iter()
9879 .map(|operation| match operation {
9880 ParallelOperation::Group(children) => parallel_shape(children),
9881 _ => ParallelShape::Leaf,
9882 })
9883 .collect(),
9884 )
9885}
9886
9887fn parallel_descriptors(
9888 operations: Vec<ParallelOperation>,
9889 base_sequence: u64,
9890) -> Result<Vec<ParallelDescriptor>> {
9891 let size = parallel_leaf_count(&operations);
9892 let kind = parallel_group_kind(&operations).unwrap_or("activity");
9893 let mut descriptors = Vec::with_capacity(size);
9894 let mut cursor = 0;
9895
9896 for (index, operation) in operations.into_iter().enumerate() {
9897 match operation {
9898 ParallelOperation::Group(children) => {
9899 let child_base = base_sequence
9900 .checked_add(u64::try_from(cursor).unwrap_or(u64::MAX))
9901 .ok_or(Error::TimerDurationOverflow)?;
9902 for mut descriptor in parallel_descriptors(children, child_base)? {
9903 let outer_index = cursor + descriptor.offset;
9904 descriptor.group_path.insert(
9905 0,
9906 parallel_group_entry(base_sequence, size, outer_index, kind),
9907 );
9908 descriptor.member_path.insert(0, index);
9909 descriptor.offset = outer_index;
9910 descriptors.push(descriptor);
9911 }
9912 cursor = descriptors.len();
9913 }
9914 operation => {
9915 descriptors.push(ParallelDescriptor {
9916 operation,
9917 offset: cursor,
9918 member_path: vec![index],
9919 group_path: vec![parallel_group_entry(base_sequence, size, cursor, kind)],
9920 });
9921 cursor += 1;
9922 }
9923 }
9924 }
9925 Ok(descriptors)
9926}
9927
9928enum ParallelLeafCall {
9929 Activity(ActivityCall),
9930 ChildWorkflow(ChildWorkflowCall),
9931 Timer(TimerCall),
9932 Signal(SignalCall),
9933 Condition(ConditionWaitCall),
9934}
9935
9936fn parallel_leaf_call(
9937 ctx: &WorkflowContext,
9938 operation: ParallelOperation,
9939 parallel_group_path: Vec<ParallelGroupMetadata>,
9940) -> ParallelLeafCall {
9941 match operation {
9942 ParallelOperation::Activity {
9943 activity_type,
9944 options,
9945 arguments,
9946 } => ParallelLeafCall::Activity(ActivityCall {
9947 ctx: ctx.clone(),
9948 activity_type,
9949 options,
9950 args: Some(arguments),
9951 scheduled: false,
9952 parallel_group_path,
9953 }),
9954 ParallelOperation::ChildWorkflow {
9955 workflow_type,
9956 options,
9957 arguments,
9958 } => ParallelLeafCall::ChildWorkflow(ChildWorkflowCall {
9959 ctx: ctx.clone(),
9960 workflow_type,
9961 options,
9962 args: Some(arguments),
9963 scheduled: false,
9964 matched_pending: false,
9965 parallel_group_path,
9966 }),
9967 ParallelOperation::Timer(duration) => {
9968 let delay_seconds = duration
9969 .as_secs()
9970 .checked_add(u64::from(duration.subsec_nanos() > 0));
9971 ParallelLeafCall::Timer(TimerCall {
9972 ctx: ctx.clone(),
9973 delay_seconds,
9974 scheduled: false,
9975 matched_pending: false,
9976 parallel_group_path,
9977 })
9978 }
9979 ParallelOperation::Signal(signal_name) => ParallelLeafCall::Signal(SignalCall {
9980 ctx: ctx.clone(),
9981 signal_name,
9982 runtime_reserved_allowed: false,
9983 opened_wait: false,
9984 matched_pending: false,
9985 parallel_group_path,
9986 }),
9987 ParallelOperation::Condition { options, predicate } => {
9988 ParallelLeafCall::Condition(ConditionWaitCall {
9989 ctx: ctx.clone(),
9990 options,
9991 predicate,
9992 occurrence_id: None,
9993 opened_wait: false,
9994 parallel_group_path,
9995 })
9996 }
9997 ParallelOperation::Group(_) => {
9998 unreachable!("parallel descriptors contain only durable leaves")
9999 }
10000 }
10001}
10002
10003impl ParallelLeafCall {
10004 fn poll_avro_value(&mut self, cx: &mut TaskContext<'_>) -> Poll<Result<ParallelAvroResult>> {
10005 match self {
10006 Self::Activity(call) => Pin::new(call)
10007 .poll_avro_value(cx)
10008 .map_ok(ParallelAvroResult::Activity),
10009 Self::ChildWorkflow(call) => Pin::new(call)
10010 .poll_avro_value(cx)
10011 .map_ok(ParallelAvroResult::ChildWorkflow),
10012 Self::Timer(call) => Pin::new(call)
10013 .poll(cx)
10014 .map_ok(|()| ParallelAvroResult::Timer),
10015 Self::Signal(call) => Pin::new(call)
10016 .poll_avro_value(cx)
10017 .map_ok(ParallelAvroResult::Signal),
10018 Self::Condition(call) => Pin::new(call)
10019 .poll(cx)
10020 .map_ok(ParallelAvroResult::Condition),
10021 }
10022 }
10023}
10024
10025struct ParallelLeaf {
10026 call: ParallelLeafCall,
10027 member_path: Vec<usize>,
10028 group_path: Vec<ParallelGroupMetadata>,
10029 result: Option<ParallelAvroResult>,
10030}
10031
10032pub struct ParallelCall {
10034 ctx: WorkflowContext,
10035 operations: Option<Vec<ParallelOperation>>,
10036 shape: Option<ParallelShape>,
10037 leaves: Vec<ParallelLeaf>,
10038}
10039
10040impl ParallelCall {
10041 fn new(ctx: WorkflowContext, operations: Vec<ParallelOperation>) -> Self {
10042 Self {
10043 ctx,
10044 operations: Some(operations),
10045 shape: None,
10046 leaves: Vec::new(),
10047 }
10048 }
10049
10050 fn initialize(&mut self) -> Result<()> {
10051 let operations = self.operations.take().unwrap_or_default();
10052 validate_parallel_operations(&operations, &mut Vec::new(), true)?;
10053 self.shape = Some(parallel_shape(&operations));
10054 if operations.is_empty() {
10055 return Ok(());
10056 }
10057
10058 let base_sequence = {
10059 let state = self
10060 .ctx
10061 .state
10062 .lock()
10063 .map_err(|_| Error::WorkflowStatePoisoned)?;
10064 if let Some(recorded) = state.recorded_commands.get(state.command_cursor) {
10065 recorded.sequence()
10066 } else {
10067 let last = state
10068 .recorded_commands
10069 .last()
10070 .map(RecordedCommand::sequence)
10071 .unwrap_or(0);
10072 last.checked_add(u64::try_from(state.commands.len()).unwrap_or(u64::MAX))
10073 .and_then(|sequence| sequence.checked_add(1))
10074 .ok_or_else(|| {
10075 Error::InvalidParallelGroup(ParallelGroupError {
10076 reason: "sequence_overflow",
10077 member_path: Vec::new(),
10078 message: "parallel group sequence identity overflowed u64".to_string(),
10079 })
10080 })?
10081 }
10082 };
10083
10084 self.leaves = parallel_descriptors(operations, base_sequence)?
10085 .into_iter()
10086 .map(|descriptor| {
10087 let call = parallel_leaf_call(
10088 &self.ctx,
10089 descriptor.operation,
10090 descriptor.group_path.clone(),
10091 );
10092 ParallelLeaf {
10093 call,
10094 member_path: descriptor.member_path,
10095 group_path: descriptor.group_path,
10096 result: None,
10097 }
10098 })
10099 .collect();
10100 Ok(())
10101 }
10102
10103 fn poll_avro_value(
10104 mut self: Pin<&mut Self>,
10105 cx: &mut TaskContext<'_>,
10106 ) -> Poll<Result<Vec<ParallelAvroResult>>> {
10107 if self.operations.is_some() {
10108 if let Err(error) = self.initialize() {
10109 return Poll::Ready(Err(error));
10110 }
10111 }
10112 if self.leaves.is_empty() {
10113 return Poll::Ready(Ok(Vec::new()));
10114 }
10115
10116 let mut failures = Vec::new();
10117 let mut pending = false;
10118 for (index, leaf) in self.leaves.iter_mut().enumerate() {
10119 if leaf.result.is_some() {
10120 continue;
10121 }
10122 match leaf.call.poll_avro_value(cx) {
10123 Poll::Ready(Ok(result)) => leaf.result = Some(result),
10124 Poll::Ready(Err(error)) => failures.push((index, error)),
10125 Poll::Pending => pending = true,
10126 }
10127 }
10128
10129 if !failures.is_empty() {
10130 if let Some(position) = failures
10131 .iter()
10132 .position(|(_, error)| workflow_task_integrity_error(error))
10133 {
10134 return Poll::Ready(Err(failures.remove(position).1));
10135 }
10136 failures.sort_by_key(|(index, _)| *index);
10137 let (failed_index, cause) = failures.remove(0);
10138 let failed = &self.leaves[failed_index];
10139 let completed = self
10140 .leaves
10141 .iter()
10142 .filter_map(|leaf| {
10143 leaf.result
10144 .clone()
10145 .and_then(|result| result.into_json_result().ok())
10146 .map(|result| ParallelCompletion {
10147 member_path: leaf.member_path.clone(),
10148 result,
10149 })
10150 })
10151 .collect();
10152 let group_id = failed
10153 .group_path
10154 .first()
10155 .map(|entry| entry.parallel_group_id.clone())
10156 .unwrap_or_default();
10157 return Poll::Ready(Err(Error::ParallelFailed(ParallelFailure {
10158 group_id,
10159 member_path: failed.member_path.clone(),
10160 group_path: failed.group_path.clone(),
10161 completed,
10162 cause: Box::new(cause),
10163 })));
10164 }
10165 if pending {
10166 return Poll::Pending;
10167 }
10168
10169 let mut flat_results = self
10170 .leaves
10171 .iter_mut()
10172 .map(|leaf| leaf.result.take().expect("completed parallel leaf"))
10173 .collect::<Vec<_>>()
10174 .into_iter();
10175 let results = parallel_results_for_shape(
10176 self.shape.as_ref().expect("initialized parallel shape"),
10177 &mut flat_results,
10178 );
10179 Poll::Ready(Ok(match results {
10180 ParallelAvroResult::Group(results) => results,
10181 ParallelAvroResult::Activity(_)
10182 | ParallelAvroResult::ChildWorkflow(_)
10183 | ParallelAvroResult::Timer
10184 | ParallelAvroResult::Signal(_)
10185 | ParallelAvroResult::Condition(_) => {
10186 unreachable!("root parallel shape is a group")
10187 }
10188 }))
10189 }
10190}
10191
10192fn parallel_results_for_shape(
10193 shape: &ParallelShape,
10194 flat_results: &mut impl Iterator<Item = ParallelAvroResult>,
10195) -> ParallelAvroResult {
10196 match shape {
10197 ParallelShape::Leaf => flat_results.next().expect("one result per parallel leaf"),
10198 ParallelShape::Group(children) => ParallelAvroResult::Group(
10199 children
10200 .iter()
10201 .map(|child| parallel_results_for_shape(child, flat_results))
10202 .collect(),
10203 ),
10204 }
10205}
10206
10207impl Future for ParallelCall {
10208 type Output = Result<Vec<ParallelResult>>;
10209
10210 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
10211 self.poll_avro_value(cx)
10212 .map_ok(|results| {
10213 results
10214 .into_iter()
10215 .map(ParallelAvroResult::into_json_result)
10216 .collect::<Result<Vec<_>>>()
10217 })
10218 .map_ok(|result| result)
10219 .flatten_result()
10220 }
10221}
10222
10223#[derive(Clone, Debug)]
10224struct SelectionMemberPlan {
10225 key: SelectionKey,
10226 index: usize,
10227 base_sequence: u64,
10228 size: usize,
10229 kind: String,
10230 shape: ParallelShape,
10231 leaf_start: usize,
10232}
10233
10234fn selection_operation_kind(operation: &ParallelOperation) -> &'static str {
10235 match operation {
10236 ParallelOperation::Activity { .. } => "activity",
10237 ParallelOperation::ChildWorkflow { .. } => "child",
10238 ParallelOperation::Timer(_) => "timer",
10239 ParallelOperation::Signal(_) => "signal",
10240 ParallelOperation::Condition { .. } => "condition",
10241 ParallelOperation::Group(_) => "group",
10242 }
10243}
10244
10245fn selection_operation_shape(operation: &ParallelOperation) -> ParallelShape {
10246 match operation {
10247 ParallelOperation::Group(children) => parallel_shape(children),
10248 _ => ParallelShape::Leaf,
10249 }
10250}
10251
10252fn selection_descriptors(
10253 operations: Vec<(SelectionKey, ParallelOperation)>,
10254 base_sequence: u64,
10255) -> Result<(Vec<ParallelDescriptor>, Vec<SelectionMemberPlan>)> {
10256 if operations.is_empty() {
10257 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10258 reason: "selection_empty",
10259 member_path: Vec::new(),
10260 message: "durable selection requires at least one operation".to_string(),
10261 }));
10262 }
10263 let operation_refs = operations
10264 .iter()
10265 .map(|(_, operation)| operation)
10266 .collect::<Vec<_>>();
10267 let total_size = operation_refs
10268 .iter()
10269 .map(|operation| match operation {
10270 ParallelOperation::Group(children) => parallel_leaf_count(children),
10271 _ => 1,
10272 })
10273 .sum::<usize>();
10274 if total_size > MAX_PARALLEL_OPERATIONS {
10275 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10276 reason: "fan_out_limit_exceeded",
10277 member_path: Vec::new(),
10278 message: format!(
10279 "selection contains {total_size} durable leaves; the limit is {MAX_PARALLEL_OPERATIONS}"
10280 ),
10281 }));
10282 }
10283 let group_kind = {
10284 let mut kind = None;
10285 for operation in &operation_refs {
10286 let operation_kind = parallel_operation_kind(operation).unwrap_or("mixed");
10287 match kind {
10288 None => kind = Some(operation_kind),
10289 Some(current) if current == operation_kind => {}
10290 Some(_) => {
10291 kind = Some("mixed");
10292 break;
10293 }
10294 }
10295 }
10296 kind.unwrap_or("mixed")
10297 };
10298
10299 let mut descriptors = Vec::with_capacity(total_size);
10300 let mut members = Vec::with_capacity(operations.len());
10301 let mut cursor = 0usize;
10302 let mut seen_keys: Vec<SelectionKey> = Vec::new();
10303 for (member_index, (key, operation)) in operations.into_iter().enumerate() {
10304 if matches!(&key, SelectionKey::Name(value) if value.is_empty()) {
10305 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10306 reason: "selection_key_invalid",
10307 member_path: vec![member_index],
10308 message: "selection member keys must be non-empty strings or non-negative integers"
10309 .to_string(),
10310 }));
10311 }
10312 if seen_keys.contains(&key) {
10313 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10314 reason: "selection_key_duplicate",
10315 member_path: vec![member_index],
10316 message: format!("selection member key {key:?} is duplicated"),
10317 }));
10318 }
10319 seen_keys.push(key.clone());
10320 let member_size = match &operation {
10321 ParallelOperation::Group(children) => parallel_leaf_count(children),
10322 _ => 1,
10323 };
10324 if member_size == 0 {
10325 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10326 reason: "selection_member_empty",
10327 member_path: vec![member_index],
10328 message: "a selection member must contain at least one durable leaf".to_string(),
10329 }));
10330 }
10331 let member_base = base_sequence
10332 .checked_add(u64::try_from(cursor).unwrap_or(u64::MAX))
10333 .ok_or(Error::TimerDurationOverflow)?;
10334 let member_kind = selection_operation_kind(&operation).to_string();
10335 let member_shape = selection_operation_shape(&operation);
10336 let leaf_start = descriptors.len();
10337 match operation {
10338 ParallelOperation::Group(children) => {
10339 validate_parallel_operations(&children, &mut vec![member_index], false)?;
10340 for mut descriptor in parallel_descriptors(children, member_base)? {
10341 let flat_index = cursor + descriptor.offset;
10342 descriptor.group_path.insert(
10343 0,
10344 selection_group_entry(
10345 base_sequence,
10346 total_size,
10347 flat_index,
10348 group_kind,
10349 &SelectionMemberMetadata {
10350 key: key.clone(),
10351 index: member_index,
10352 base_sequence: member_base,
10353 size: member_size,
10354 kind: member_kind.clone(),
10355 },
10356 ),
10357 );
10358 descriptor.member_path.insert(0, member_index);
10359 descriptor.offset = flat_index;
10360 descriptors.push(descriptor);
10361 }
10362 }
10363 operation => {
10364 validate_parallel_operations(
10365 std::slice::from_ref(&operation),
10366 &mut Vec::new(),
10367 true,
10368 )?;
10369 descriptors.push(ParallelDescriptor {
10370 operation,
10371 offset: cursor,
10372 member_path: vec![member_index],
10373 group_path: vec![selection_group_entry(
10374 base_sequence,
10375 total_size,
10376 cursor,
10377 group_kind,
10378 &SelectionMemberMetadata {
10379 key: key.clone(),
10380 index: member_index,
10381 base_sequence: member_base,
10382 size: member_size,
10383 kind: member_kind.clone(),
10384 },
10385 )],
10386 });
10387 }
10388 }
10389 members.push(SelectionMemberPlan {
10390 key,
10391 index: member_index,
10392 base_sequence: member_base,
10393 size: member_size,
10394 kind: member_kind,
10395 shape: member_shape,
10396 leaf_start,
10397 });
10398 cursor += member_size;
10399 }
10400 Ok((descriptors, members))
10401}
10402
10403struct SelectionLeaf {
10404 call: ParallelLeafCall,
10405 outcome: Option<Result<ParallelAvroResult>>,
10406}
10407
10408#[derive(Clone)]
10410pub struct DurableOperationHandle {
10411 ctx: WorkflowContext,
10412 pub key: SelectionKey,
10413 pub index: usize,
10414 pub kind: String,
10415 pub identity: String,
10416 pub base_sequence: u64,
10417 pub size: usize,
10418 pub selection_group_id: String,
10419 shape: ParallelShape,
10420}
10421
10422impl std::fmt::Debug for DurableOperationHandle {
10423 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10424 formatter
10425 .debug_struct("DurableOperationHandle")
10426 .field("key", &self.key)
10427 .field("index", &self.index)
10428 .field("kind", &self.kind)
10429 .field("identity", &self.identity)
10430 .field("base_sequence", &self.base_sequence)
10431 .field("size", &self.size)
10432 .field("selection_group_id", &self.selection_group_id)
10433 .finish()
10434 }
10435}
10436
10437impl DurableOperationHandle {
10438 pub fn await_result(&self) -> DurableOperationAwaitCall {
10440 DurableOperationAwaitCall {
10441 handle: self.clone(),
10442 }
10443 }
10444
10445 pub fn cancel(&self) -> CancelDurableOperationCall {
10449 CancelDurableOperationCall {
10450 handle: self.clone(),
10451 emitted: false,
10452 }
10453 }
10454}
10455
10456#[derive(Debug)]
10458pub struct SelectionResult {
10459 pub key: SelectionKey,
10460 pub index: usize,
10461 pub kind: String,
10462 pub identity: String,
10463 pub value: Option<ParallelResult>,
10464 pub failure: Option<Error>,
10465 pub winner: DurableOperationHandle,
10466 pub handles: Vec<DurableOperationHandle>,
10467}
10468
10469impl SelectionResult {
10470 pub fn succeeded(&self) -> bool {
10471 self.failure.is_none()
10472 }
10473
10474 pub fn handle(&self, key: &SelectionKey) -> Option<&DurableOperationHandle> {
10475 self.handles.iter().find(|handle| &handle.key == key)
10476 }
10477
10478 pub fn remaining(&self) -> Vec<&DurableOperationHandle> {
10479 self.handles
10480 .iter()
10481 .filter(|handle| handle.index != self.index)
10482 .collect()
10483 }
10484
10485 pub fn into_result(self) -> Result<ParallelResult> {
10486 match (self.value, self.failure) {
10487 (Some(value), None) => Ok(value),
10488 (_, Some(error)) => Err(error),
10489 _ => Err(Error::WorkerLoop(
10490 "selection result contained neither a value nor a failure".to_string(),
10491 )),
10492 }
10493 }
10494}
10495
10496pub struct SelectCall {
10498 ctx: WorkflowContext,
10499 operations: Option<Vec<(SelectionKey, ParallelOperation)>>,
10500 members: Vec<SelectionMemberPlan>,
10501 leaves: Vec<SelectionLeaf>,
10502 group_id: Option<String>,
10503}
10504
10505impl SelectCall {
10506 fn new(ctx: WorkflowContext, operations: Vec<(SelectionKey, ParallelOperation)>) -> Self {
10507 Self {
10508 ctx,
10509 operations: Some(operations),
10510 members: Vec::new(),
10511 leaves: Vec::new(),
10512 group_id: None,
10513 }
10514 }
10515
10516 fn initialize(&mut self) -> Result<()> {
10517 let operations = self.operations.take().unwrap_or_default();
10518 let base_sequence = {
10519 let state = self
10520 .ctx
10521 .state
10522 .lock()
10523 .map_err(|_| Error::WorkflowStatePoisoned)?;
10524 if let Some(marker) = state.selection_markers.get(state.selection_marker_cursor) {
10525 marker.selection_group_base_sequence
10526 } else if let Some(recorded) = state.recorded_commands.get(state.command_cursor) {
10527 recorded.sequence()
10528 } else {
10529 let last = state
10530 .recorded_commands
10531 .last()
10532 .map(RecordedCommand::sequence)
10533 .unwrap_or(0);
10534 last.checked_add(u64::try_from(state.commands.len()).unwrap_or(u64::MAX))
10535 .and_then(|sequence| sequence.checked_add(1))
10536 .ok_or(Error::TimerDurationOverflow)?
10537 }
10538 };
10539 let (descriptors, members) = selection_descriptors(operations, base_sequence)?;
10540 let group_id = format!("select-calls:{base_sequence}:{}", descriptors.len());
10541 self.leaves = descriptors
10542 .into_iter()
10543 .map(|descriptor| SelectionLeaf {
10544 call: parallel_leaf_call(&self.ctx, descriptor.operation, descriptor.group_path),
10545 outcome: None,
10546 })
10547 .collect();
10548 self.members = members;
10549 self.group_id = Some(group_id);
10550 Ok(())
10551 }
10552}
10553
10554impl Future for SelectCall {
10555 type Output = Result<SelectionResult>;
10556
10557 fn poll(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
10558 if self.operations.is_some() {
10559 if let Err(error) = self.initialize() {
10560 return Poll::Ready(Err(error));
10561 }
10562 }
10563
10564 for leaf in &mut self.leaves {
10565 if leaf.outcome.is_some() {
10566 continue;
10567 }
10568 if let Poll::Ready(outcome) = leaf.call.poll_avro_value(cx) {
10569 if outcome
10570 .as_ref()
10571 .err()
10572 .is_some_and(workflow_task_integrity_error)
10573 {
10574 return Poll::Ready(outcome.map(|_| unreachable!()));
10575 }
10576 leaf.outcome = Some(outcome);
10577 }
10578 }
10579
10580 let all_members_terminal = self.leaves.iter().all(|leaf| leaf.outcome.is_some());
10581 let selection_member_range = self
10582 .members
10583 .first()
10584 .map(|member| member.base_sequence)
10585 .zip(self.leaves.len().try_into().ok())
10586 .map(|(base_sequence, size): (u64, u64)| {
10587 base_sequence..base_sequence.saturating_add(size)
10588 });
10589 let marker = {
10590 let mut state = match self.ctx.state.lock() {
10591 Ok(state) => state,
10592 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
10593 };
10594 let marker = state
10595 .selection_markers
10596 .get(state.selection_marker_cursor)
10597 .cloned();
10598 if marker.is_none()
10599 && all_members_terminal
10600 && selection_member_range.as_ref().is_some_and(|member_range| {
10601 state
10602 .recorded_commands
10603 .iter()
10604 .any(|command| member_range.contains(&command.sequence()))
10605 })
10606 {
10607 state.matched_recorded_pending = true;
10613 }
10614 marker
10615 };
10616 let Some(marker) = marker else {
10617 return Poll::Pending;
10618 };
10619 if self.group_id.as_deref() != Some(marker.selection_group_id.as_str())
10620 || marker.selection_group_size != self.leaves.len()
10621 || self.members.first().map(|member| member.base_sequence)
10622 != Some(marker.selection_group_base_sequence)
10623 {
10624 return Poll::Ready(Err(invalid_recorded_history(
10625 "selection_group_shape_mismatch",
10626 marker.selection_group_base_sequence,
10627 self.group_id
10628 .as_deref()
10629 .unwrap_or("initialized selection group"),
10630 &marker.selection_group_id,
10631 "recorded selection group differs from current workflow code",
10632 )));
10633 }
10634 let Some(member_position) = self.members.iter().position(|member| {
10635 member.key == marker.member_key
10636 && member.index == marker.member_index
10637 && member.base_sequence == marker.member_base_sequence
10638 && member.size == marker.member_size
10639 && member.kind == marker.operation_kind
10640 }) else {
10641 return Poll::Ready(Err(invalid_recorded_history(
10642 "selection_member_shape_mismatch",
10643 marker.member_base_sequence,
10644 "winner member matching current workflow code",
10645 &format!("{:?}", marker.member_key),
10646 "recorded selection winner differs from the authored member identity",
10647 )));
10648 };
10649 let member = self.members[member_position].clone();
10650 let (handles, resolution_sequence) = {
10651 let mut state = match self.ctx.state.lock() {
10652 Ok(state) => state,
10653 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
10654 };
10655 let identities = self
10656 .members
10657 .iter()
10658 .map(|candidate| {
10659 selection_operation_identity(
10660 &state,
10661 &candidate.kind,
10662 candidate.base_sequence,
10663 candidate.size,
10664 )
10665 })
10666 .collect::<Vec<_>>();
10667 if let Some((position, missing)) = identities
10668 .iter()
10669 .enumerate()
10670 .find(|(_, identity)| identity.is_empty())
10671 .map(|(position, identity)| (position, identity.clone()))
10672 {
10673 let candidate = &self.members[position];
10674 return Poll::Ready(Err(invalid_recorded_history(
10675 "selection_operation_identity_missing",
10676 candidate.base_sequence,
10677 &format!(
10678 "durable {} resource identity from scheduled/open history",
10679 candidate.kind
10680 ),
10681 &missing,
10682 "selection member history is missing its canonical durable identity",
10683 )));
10684 }
10685 let expected_winner_identity = &identities[member_position];
10686 let resolution_sequence = match validated_selection_resolution_sequence(
10687 &state,
10688 &marker,
10689 &member,
10690 expected_winner_identity,
10691 ) {
10692 Ok(sequence) => sequence,
10693 Err(error) => return Poll::Ready(Err(error)),
10694 };
10695 let handles = self
10696 .members
10697 .iter()
10698 .zip(identities)
10699 .map(|(member, identity)| DurableOperationHandle {
10700 ctx: self.ctx.clone(),
10701 key: member.key.clone(),
10702 index: member.index,
10703 kind: member.kind.clone(),
10704 identity,
10705 base_sequence: member.base_sequence,
10706 size: member.size,
10707 selection_group_id: marker.selection_group_id.clone(),
10708 shape: member.shape.clone(),
10709 })
10710 .collect::<Vec<_>>();
10711 if let Err(error) = validate_selection_cancellations_for_handles(&state, &handles) {
10712 return Poll::Ready(Err(error));
10713 }
10714 state.selection_marker_cursor += 1;
10715 (handles, resolution_sequence)
10716 };
10717
10718 let mut winner_failure = None;
10719 let mut flat_results = Vec::with_capacity(member.size);
10720 if marker.outcome == "failed" {
10721 let resolution_offset = match resolution_sequence
10722 .checked_sub(member.base_sequence)
10723 .and_then(|offset| usize::try_from(offset).ok())
10724 {
10725 Some(offset) if offset < member.size => offset,
10726 _ => {
10727 return Poll::Ready(Err(invalid_recorded_history(
10728 "selection_resolution_event_mismatch",
10729 member.base_sequence,
10730 "failure event within selected member bounds",
10731 &resolution_sequence.to_string(),
10732 "selection failure event is outside the authored member",
10733 )))
10734 }
10735 };
10736 let leaf = &mut self.leaves[member.leaf_start + resolution_offset];
10737 match leaf.outcome.take() {
10738 Some(Err(error)) => winner_failure = Some(error),
10739 _ => {
10740 return Poll::Ready(Err(invalid_recorded_history(
10741 "selection_winner_outcome_mismatch",
10742 member.base_sequence,
10743 "exact failed terminal history referenced by SelectionResolved",
10744 "missing or successful resolution event",
10745 "selection winner marker disagrees with terminal operation history",
10746 )))
10747 }
10748 }
10749 } else {
10750 for leaf in &mut self.leaves[member.leaf_start..member.leaf_start + member.size] {
10751 match leaf.outcome.take() {
10752 Some(Ok(result)) => flat_results.push(result),
10753 Some(Err(_)) => {
10754 return Poll::Ready(Err(invalid_recorded_history(
10755 "selection_winner_outcome_mismatch",
10756 member.base_sequence,
10757 "fully completed nested selection member",
10758 "failed durable leaf",
10759 "completed selection winner contains a failed leaf",
10760 )))
10761 }
10762 None => {
10763 return Poll::Ready(Err(invalid_recorded_history(
10764 "selection_winner_unresolved",
10765 member.base_sequence,
10766 "terminal history for every completed winner leaf",
10767 "pending member history",
10768 "completed SelectionResolved member has an unfinished durable barrier",
10769 )))
10770 }
10771 }
10772 }
10773 }
10774 let value = if winner_failure.is_none() {
10775 let mut flat_results = flat_results.into_iter();
10776 let value = parallel_results_for_shape(&member.shape, &mut flat_results);
10777 match value.into_json_result() {
10778 Ok(value) => Some(value),
10779 Err(error) => return Poll::Ready(Err(error)),
10780 }
10781 } else {
10782 None
10783 };
10784 let winner = handles[member_position].clone();
10785 Poll::Ready(Ok(SelectionResult {
10786 key: winner.key.clone(),
10787 index: winner.index,
10788 kind: winner.kind.clone(),
10789 identity: winner.identity.clone(),
10790 value,
10791 failure: winner_failure,
10792 winner,
10793 handles,
10794 }))
10795 }
10796}
10797
10798fn selection_operation_identity(
10799 state: &WorkflowState,
10800 kind: &str,
10801 base_sequence: u64,
10802 size: usize,
10803) -> String {
10804 if kind == "group" {
10805 return format!("group:{base_sequence}:{size}");
10806 }
10807 let fields: &[&str] = match kind {
10808 "activity" => &["activity_execution_id"],
10809 "child" => &["child_workflow_run_id"],
10810 "timer" => &["timer_id"],
10811 "signal" => &["signal_wait_id"],
10812 "condition" => &["condition_wait_id"],
10813 _ => &[],
10814 };
10815 for sequence in base_sequence..base_sequence.saturating_add(size as u64) {
10816 for event in state
10817 .history_events
10818 .iter()
10819 .filter(|event| durable_event_sequence(event) == Some(sequence))
10820 {
10821 for field in fields {
10822 if let Some(identity) = event.payload.get(*field).and_then(Value::as_str) {
10823 if !identity.is_empty() {
10824 return identity.to_string();
10825 }
10826 }
10827 }
10828 }
10829 }
10830 String::new()
10831}
10832
10833fn validated_selection_resolution_sequence(
10834 state: &WorkflowState,
10835 marker: &SelectionMarker,
10836 member: &SelectionMemberPlan,
10837 expected_identity: &str,
10838) -> Result<u64> {
10839 if expected_identity.is_empty() {
10840 return Err(invalid_recorded_history(
10841 "selection_operation_identity_missing",
10842 member.base_sequence,
10843 &format!(
10844 "durable {} resource identity from scheduled/open history",
10845 member.kind
10846 ),
10847 "missing operation identity",
10848 "selection member history is missing its canonical durable identity",
10849 ));
10850 }
10851 if marker.operation_identity != expected_identity {
10852 return Err(invalid_recorded_history(
10853 "selection_operation_identity_mismatch",
10854 member.base_sequence,
10855 expected_identity,
10856 &marker.operation_identity,
10857 "selection winner identity does not match durable scheduled/open history",
10858 ));
10859 }
10860
10861 let failure_types = [
10862 "ActivityFailed",
10863 "ActivityCancelled",
10864 "ActivityTimedOut",
10865 "ChildRunFailed",
10866 "ChildRunCancelled",
10867 "ChildRunTerminated",
10868 ];
10869 let success_types = [
10870 "ActivityCompleted",
10871 "ChildRunCompleted",
10872 "TimerFired",
10873 "SignalApplied",
10874 "ConditionWaitSatisfied",
10875 "ConditionWaitTimedOut",
10876 ];
10877 let terminal_types: &[&str] = if marker.outcome == "failed" {
10878 &failure_types
10879 } else {
10880 &success_types
10881 };
10882 let mut candidates = Vec::new();
10883 for event in state.history_events.iter() {
10884 let Some(sequence) = durable_event_sequence(event) else {
10885 continue;
10886 };
10887 if sequence < member.base_sequence
10888 || sequence >= member.base_sequence.saturating_add(member.size as u64)
10889 || !terminal_types.contains(&event.event_type.as_str())
10890 {
10891 continue;
10892 }
10893 let event_id = event
10894 .raw
10895 .get("id")
10896 .or_else(|| event.raw.get("event_id"))
10897 .and_then(Value::as_str)
10898 .filter(|value| !value.is_empty())
10899 .ok_or_else(|| {
10900 invalid_recorded_history(
10901 "selection_resolution_event_id_missing",
10902 member.base_sequence,
10903 "terminal selection history with a durable event id",
10904 &event.payload.to_string(),
10905 "selection terminal history cannot be bound to its winner marker",
10906 )
10907 })?;
10908 candidates.push((event_id.to_string(), event.event_type.clone(), sequence));
10909 }
10910 let resolution = if marker.outcome == "failed" {
10911 candidates.first()
10912 } else {
10913 candidates.last()
10914 };
10915 let Some((event_id, event_type, sequence)) = resolution else {
10916 return Err(invalid_recorded_history(
10917 "selection_resolution_event_missing",
10918 member.base_sequence,
10919 "terminal history for the selected member",
10920 &format!("{:?}", marker.member_key),
10921 "selection winner marker has no matching durable terminal event",
10922 ));
10923 };
10924 if event_id != &marker.resolution_event_id || event_type != &marker.resolution_event_type {
10925 return Err(invalid_recorded_history(
10926 "selection_resolution_event_mismatch",
10927 member.base_sequence,
10928 &format!("{event_type}:{event_id}"),
10929 &format!(
10930 "{}:{}",
10931 marker.resolution_event_type, marker.resolution_event_id
10932 ),
10933 "selection winner marker does not reference the event that made its member terminal",
10934 ));
10935 }
10936 Ok(*sequence)
10937}
10938
10939fn recorded_selection_member_outcome(
10940 state: &WorkflowState,
10941 handle: &DurableOperationHandle,
10942) -> Result<Option<ParallelResult>> {
10943 for event in state.history_events.iter() {
10944 let Some(sequence) = durable_event_sequence(event) else {
10945 continue;
10946 };
10947 if sequence < handle.base_sequence
10948 || sequence >= handle.base_sequence.saturating_add(handle.size as u64)
10949 || !matches!(
10950 event.event_type.as_str(),
10951 "ActivityFailed"
10952 | "ActivityCancelled"
10953 | "ActivityTimedOut"
10954 | "ChildRunFailed"
10955 | "ChildRunCancelled"
10956 | "ChildRunTerminated"
10957 )
10958 {
10959 continue;
10960 }
10961 let Some(command) = state
10962 .recorded_commands
10963 .iter()
10964 .find(|command| command.sequence() == sequence)
10965 else {
10966 continue;
10967 };
10968 match command {
10969 RecordedCommand::Activity {
10970 outcome: Some(Err(failure)),
10971 ..
10972 } => return Err(Error::ActivityFailed(failure.clone())),
10973 RecordedCommand::ChildWorkflow {
10974 outcome: Some(Err(failure)),
10975 ..
10976 } => return Err(Error::ChildWorkflowFailed(failure.clone())),
10977 _ => {}
10978 }
10979 }
10980
10981 let mut results = Vec::with_capacity(handle.size);
10982 for sequence in handle.base_sequence..handle.base_sequence.saturating_add(handle.size as u64) {
10983 let Some(command) = state
10984 .recorded_commands
10985 .iter()
10986 .find(|command| command.sequence() == sequence)
10987 else {
10988 return Ok(None);
10989 };
10990 let result = match command {
10991 RecordedCommand::Activity { outcome, .. } => match outcome {
10992 Some(Ok(value)) => ParallelAvroResult::Activity(value.clone()),
10993 Some(Err(failure)) => return Err(Error::ActivityFailed(failure.clone())),
10994 None => return Ok(None),
10995 },
10996 RecordedCommand::Timer { fired, .. } => {
10997 if !fired {
10998 return Ok(None);
10999 }
11000 ParallelAvroResult::Timer
11001 }
11002 RecordedCommand::ChildWorkflow { outcome, .. } => match outcome {
11003 Some(Ok(value)) => ParallelAvroResult::ChildWorkflow(value.clone()),
11004 Some(Err(failure)) => return Err(Error::ChildWorkflowFailed(failure.clone())),
11005 None => return Ok(None),
11006 },
11007 RecordedCommand::SignalWait { value, .. } => match value {
11008 Some(value) => ParallelAvroResult::Signal(value.clone()),
11009 None => return Ok(None),
11010 },
11011 RecordedCommand::ConditionWait { result, .. } => match result {
11012 Some(result) => ParallelAvroResult::Condition(*result),
11013 None => return Ok(None),
11014 },
11015 other => {
11016 return Err(command_mismatch(
11017 other,
11018 format!("selected {} member", handle.kind),
11019 ))
11020 }
11021 };
11022 results.push(result);
11023 }
11024 let mut results = results.into_iter();
11025 parallel_results_for_shape(&handle.shape, &mut results)
11026 .into_json_result()
11027 .map(Some)
11028}
11029
11030fn recorded_selection_member_is_terminal(
11031 state: &WorkflowState,
11032 handle: &DurableOperationHandle,
11033) -> bool {
11034 let mut completed = 0usize;
11035 let mut all_completed = true;
11036 for sequence in handle.base_sequence..handle.base_sequence.saturating_add(handle.size as u64) {
11037 let Some(command) = state
11038 .recorded_commands
11039 .iter()
11040 .find(|command| command.sequence() == sequence)
11041 else {
11042 all_completed = false;
11043 continue;
11044 };
11045 let terminal = match command {
11046 RecordedCommand::Activity {
11047 outcome: Some(Err(_)),
11048 ..
11049 }
11050 | RecordedCommand::ChildWorkflow {
11051 outcome: Some(Err(_)),
11052 ..
11053 } => return true,
11054 RecordedCommand::Activity { outcome, .. } => outcome.is_some(),
11055 RecordedCommand::ChildWorkflow { outcome, .. } => outcome.is_some(),
11056 RecordedCommand::Timer { fired, .. } => *fired,
11057 RecordedCommand::SignalWait { value, .. } => value.is_some(),
11058 RecordedCommand::ConditionWait { result, .. } => result.is_some(),
11059 RecordedCommand::SearchAttributes { .. }
11060 | RecordedCommand::SideEffect { .. }
11061 | RecordedCommand::VersionMarker { .. }
11062 | RecordedCommand::Memo { .. } => false,
11063 };
11064 if !terminal {
11065 all_completed = false;
11066 continue;
11067 }
11068 completed += 1;
11069 }
11070 all_completed && completed == handle.size
11071}
11072
11073fn selection_cancellation_for_handle(
11074 state: &WorkflowState,
11075 handle: &DurableOperationHandle,
11076) -> Result<bool> {
11077 let Some(marker) = state.cancelled_selection_members.iter().find(|recorded| {
11078 recorded.selection_group_id == handle.selection_group_id
11079 && recorded.member_base_sequence == handle.base_sequence
11080 }) else {
11081 return Ok(false);
11082 };
11083 validate_selection_cancellation_marker(marker, handle)?;
11084 Ok(true)
11085}
11086
11087fn validate_selection_cancellations_for_handles(
11088 state: &WorkflowState,
11089 handles: &[DurableOperationHandle],
11090) -> Result<()> {
11091 let Some(group_id) = handles
11092 .first()
11093 .map(|handle| handle.selection_group_id.as_str())
11094 else {
11095 return Ok(());
11096 };
11097 for marker in state
11098 .cancelled_selection_members
11099 .iter()
11100 .filter(|marker| marker.selection_group_id == group_id)
11101 {
11102 let Some(handle) = handles
11103 .iter()
11104 .find(|handle| handle.base_sequence == marker.member_base_sequence)
11105 else {
11106 return Err(invalid_recorded_history(
11107 "selection_cancellation_member_mismatch",
11108 marker.member_base_sequence,
11109 "SelectionOperationCancelled matching an authored selection handle",
11110 &format!("{marker:?}"),
11111 "selection cancellation member base does not name an authored member",
11112 ));
11113 };
11114 validate_selection_cancellation_marker(marker, handle)?;
11115 }
11116 Ok(())
11117}
11118
11119fn validate_selection_cancellation_marker(
11120 marker: &SelectionCancellation,
11121 handle: &DurableOperationHandle,
11122) -> Result<()> {
11123 if marker.selection_group_id != handle.selection_group_id
11124 || marker.member_key != handle.key
11125 || marker.member_index != handle.index
11126 || marker.member_base_sequence != handle.base_sequence
11127 || marker.member_size != handle.size
11128 || marker.operation_kind != handle.kind
11129 || marker.operation_identity != handle.identity
11130 {
11131 return Err(invalid_recorded_history(
11132 "selection_cancellation_member_mismatch",
11133 handle.base_sequence,
11134 "SelectionOperationCancelled matching the authored selection handle",
11135 &format!("{marker:?}"),
11136 "selection cancellation history targets different authored member metadata",
11137 ));
11138 }
11139 Ok(())
11140}
11141
11142pub struct DurableOperationAwaitCall {
11144 handle: DurableOperationHandle,
11145}
11146
11147impl Future for DurableOperationAwaitCall {
11148 type Output = Result<ParallelResult>;
11149
11150 fn poll(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11151 let state = match self.handle.ctx.state.lock() {
11152 Ok(state) => state,
11153 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11154 };
11155 match selection_cancellation_for_handle(&state, &self.handle) {
11156 Err(error) => return Poll::Ready(Err(error)),
11157 Ok(false) => {}
11158 Ok(true) => {
11159 return Poll::Ready(Err(Error::DurableOperationCancelled(
11160 DurableOperationCancelled {
11161 selection_group_id: self.handle.selection_group_id.clone(),
11162 member_key: self.handle.key.clone(),
11163 member_index: self.handle.index,
11164 operation_kind: self.handle.kind.clone(),
11165 operation_identity: self.handle.identity.clone(),
11166 },
11167 )));
11168 }
11169 }
11170 match recorded_selection_member_outcome(&state, &self.handle) {
11171 Ok(Some(result)) => Poll::Ready(Ok(result)),
11172 Ok(None) => Poll::Pending,
11173 Err(error) => Poll::Ready(Err(error)),
11174 }
11175 }
11176}
11177
11178pub struct CancelDurableOperationCall {
11180 handle: DurableOperationHandle,
11181 emitted: bool,
11182}
11183
11184impl Future for CancelDurableOperationCall {
11185 type Output = Result<()>;
11186
11187 fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11188 let ctx = self.handle.ctx.clone();
11189 let mut state = match ctx.state.lock() {
11190 Ok(state) => state,
11191 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11192 };
11193 match selection_cancellation_for_handle(&state, &self.handle) {
11194 Err(error) => return Poll::Ready(Err(error)),
11195 Ok(true) => return Poll::Ready(Ok(())),
11196 Ok(false) => {}
11197 }
11198 if recorded_selection_member_is_terminal(&state, &self.handle) {
11199 return Poll::Ready(Ok(()));
11200 }
11201 if !self.emitted {
11202 state.commands.push(json!({
11203 "type": "cancel_selection_operation",
11204 "selection_group_id": self.handle.selection_group_id,
11205 "member_key": self.handle.key,
11206 "member_index": self.handle.index,
11207 "member_base_sequence": self.handle.base_sequence,
11208 "member_size": self.handle.size,
11209 "operation_kind": self.handle.kind,
11210 "operation_identity": self.handle.identity,
11211 }));
11212 self.emitted = true;
11213 }
11214 Poll::Pending
11218 }
11219}
11220
11221trait PollNestedResultExt<T> {
11222 fn flatten_result(self) -> Poll<Result<T>>;
11223}
11224
11225impl<T> PollNestedResultExt<T> for Poll<Result<Result<T>>> {
11226 fn flatten_result(self) -> Poll<Result<T>> {
11227 match self {
11228 Poll::Ready(Ok(result)) => Poll::Ready(result),
11229 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
11230 Poll::Pending => Poll::Pending,
11231 }
11232 }
11233}
11234
11235struct SagaCompensation {
11236 activity_type: String,
11237 options: ActivityOptions,
11238 arguments: AvroValue,
11239 registration_order: usize,
11240}
11241
11242pub struct Saga {
11250 ctx: WorkflowContext,
11251 compensations: Vec<SagaCompensation>,
11252}
11253
11254impl Saga {
11255 fn new(ctx: WorkflowContext) -> Self {
11256 Self {
11257 ctx,
11258 compensations: Vec::new(),
11259 }
11260 }
11261
11262 pub fn add_compensation<T: Serialize>(
11263 &mut self,
11264 activity_type: impl Into<String>,
11265 args: T,
11266 ) -> Result<&mut Self> {
11267 self.add_compensation_with_options(activity_type, ActivityOptions::new(), args)
11268 }
11269
11270 pub fn add_compensation_with_options<T: Serialize>(
11271 &mut self,
11272 activity_type: impl Into<String>,
11273 options: ActivityOptions,
11274 args: T,
11275 ) -> Result<&mut Self> {
11276 let activity_type = activity_type.into();
11277 if activity_type.trim().is_empty() || activity_type.trim() != activity_type {
11278 return Err(Error::Codec(
11279 "saga compensation activity type must be non-empty without surrounding whitespace"
11280 .to_string(),
11281 ));
11282 }
11283 options.validate().map_err(Error::InvalidActivityOptions)?;
11284 let arguments = AvroValue::from_serialize(&args)?;
11285 let registration_order = self.compensations.len() + 1;
11286 self.compensations.push(SagaCompensation {
11287 activity_type,
11288 options,
11289 arguments,
11290 registration_order,
11291 });
11292 Ok(self)
11293 }
11294
11295 pub async fn compensate(mut self, initiating_failure: Error) -> Error {
11297 while let Some(compensation) = self.compensations.pop() {
11298 if let Err(compensation_failure) = self
11299 .ctx
11300 .activity_with_options(
11301 compensation.activity_type.clone(),
11302 compensation.options,
11303 compensation.arguments,
11304 )
11305 .await
11306 {
11307 if workflow_task_integrity_error(&compensation_failure) {
11308 return compensation_failure;
11309 }
11310 return Error::SagaCompensationFailed(SagaCompensationFailure {
11311 initiating_failure: Box::new(initiating_failure),
11312 compensation_failure: Box::new(compensation_failure),
11313 compensation_activity_type: compensation.activity_type,
11314 compensation_registration_order: compensation.registration_order,
11315 });
11316 }
11317 }
11318 initiating_failure
11319 }
11320
11321 pub async fn finish<T>(self, outcome: Result<T>) -> Result<T> {
11323 match outcome {
11324 Ok(value) => Ok(value),
11325 Err(error) => Err(self.compensate(error).await),
11326 }
11327 }
11328}
11329
11330pub struct ActivityCall {
11331 ctx: WorkflowContext,
11332 activity_type: String,
11333 options: ActivityOptions,
11334 args: Option<Result<AvroValue>>,
11335 scheduled: bool,
11336 parallel_group_path: Vec<ParallelGroupMetadata>,
11337}
11338
11339impl ActivityCall {
11340 fn poll_avro_value(
11341 mut self: Pin<&mut Self>,
11342 _cx: &mut TaskContext<'_>,
11343 ) -> Poll<Result<AvroValue>> {
11344 let ctx = self.ctx.clone();
11345 let mut state = match ctx.state.lock() {
11346 Ok(state) => state,
11347 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11348 };
11349
11350 if self.scheduled {
11351 return Poll::Pending;
11352 }
11353
11354 let options = match self.options.validate() {
11355 Ok(options) => options,
11356 Err(error) => {
11357 return Poll::Ready(Err(Error::InvalidActivityOptions(error)));
11358 }
11359 };
11360 let task_queue = options
11361 .task_queue
11362 .clone()
11363 .unwrap_or_else(|| state.task_queue.clone());
11364 let current_recorded_options = RecordedActivityOptions {
11365 task_queue: RecordedSnapshotValue::Known(Some(task_queue.clone())),
11366 execution_mode: RecordedSnapshotValue::Known(None),
11369 retry_policy: current_activity_retry_snapshot(&options),
11370 };
11371
11372 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
11373 let sequence = recorded.sequence();
11374 match recorded {
11375 RecordedCommand::Activity {
11376 activity_type,
11377 options: recorded_options,
11378 outcome,
11379 parallel_group_path,
11380 ..
11381 } => {
11382 if let Err(error) = ensure_parallel_path_matches(
11383 sequence,
11384 parallel_group_path.as_deref(),
11385 &self.parallel_group_path,
11386 ) {
11387 return Poll::Ready(Err(error));
11388 }
11389 if let Some(recorded_type) = activity_type {
11390 if recorded_type != self.activity_type {
11391 return Poll::Ready(Err(Error::NonDeterministicReplay(
11392 ReplayFailure::new(
11393 "recorded_command_detail_mismatch",
11394 Some(sequence),
11395 Some(format!("activity:{recorded_type}")),
11396 Some(format!("activity:{}", self.activity_type)),
11397 "recorded activity type differs from the current workflow command",
11398 ),
11399 )));
11400 }
11401 }
11402 if let Some(recorded_options) = recorded_options {
11403 if !recorded_options
11404 .task_queue
11405 .matches_current(¤t_recorded_options.task_queue)
11406 {
11407 return Poll::Ready(Err(Error::NonDeterministicReplay(
11408 ReplayFailure::new(
11409 "activity_task_queue_mismatch",
11410 Some(sequence),
11411 Some(activity_options_description(&recorded_options)),
11412 Some(activity_options_description(¤t_recorded_options)),
11413 "recorded activity task queue differs from the current workflow command",
11414 ),
11415 )));
11416 }
11417 if !recorded_options
11418 .execution_mode
11419 .matches_current(¤t_recorded_options.execution_mode)
11420 {
11421 return Poll::Ready(Err(Error::NonDeterministicReplay(
11422 ReplayFailure::new(
11423 "activity_execution_mode_mismatch",
11424 Some(sequence),
11425 Some(activity_options_description(&recorded_options)),
11426 Some(activity_options_description(¤t_recorded_options)),
11427 "recorded activity execution mode differs from the current workflow command",
11428 ),
11429 )));
11430 }
11431 if !recorded_options
11432 .retry_policy
11433 .matches_current(¤t_recorded_options.retry_policy)
11434 {
11435 return Poll::Ready(Err(Error::NonDeterministicReplay(
11436 ReplayFailure::new(
11437 "activity_retry_policy_mismatch",
11438 Some(sequence),
11439 Some(activity_options_description(&recorded_options)),
11440 Some(activity_options_description(¤t_recorded_options)),
11441 "recorded activity retry policy differs from the current workflow command",
11442 ),
11443 )));
11444 }
11445 }
11446 state.command_cursor += 1;
11447 if let Some(outcome) = outcome {
11448 return Poll::Ready(outcome.map_err(Error::ActivityFailed));
11449 }
11450 state.matched_recorded_pending = true;
11451 self.scheduled = true;
11452 return Poll::Pending;
11453 }
11454 other => {
11455 return Poll::Ready(Err(command_mismatch(
11456 &other,
11457 format!("activity:{}", self.activity_type),
11458 )));
11459 }
11460 }
11461 }
11462
11463 if !self.scheduled {
11464 let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
11465 Ok(args) => args,
11466 Err(error) => return Poll::Ready(Err(error)),
11467 };
11468 let arguments = normalize_avro_arguments(args);
11469 let envelope = match encode_typed_envelope(&arguments, &state.payload_codec) {
11470 Ok(envelope) => envelope,
11471 Err(error) => return Poll::Ready(Err(error)),
11472 };
11473
11474 let mut command = serde_json::Map::from_iter([
11475 ("type".to_string(), json!("schedule_activity")),
11476 (
11477 "activity_type".to_string(),
11478 json!(self.activity_type.clone()),
11479 ),
11480 ("queue".to_string(), json!(task_queue)),
11481 ("arguments".to_string(), envelope),
11482 ]);
11483 for (field, value) in [
11484 ("start_to_close_timeout", options.start_to_close_timeout),
11485 (
11486 "schedule_to_start_timeout",
11487 options.schedule_to_start_timeout,
11488 ),
11489 (
11490 "schedule_to_close_timeout",
11491 options.schedule_to_close_timeout,
11492 ),
11493 ("heartbeat_timeout", options.heartbeat_timeout),
11494 ] {
11495 if let Some(value) = value {
11496 command.insert(field.to_string(), json!(value));
11497 }
11498 }
11499 if let Some(retry_policy) = options.retry_policy {
11500 command.insert("retry_policy".to_string(), retry_policy);
11501 }
11502 apply_parallel_group_path(&mut command, &self.parallel_group_path);
11503 state.commands.push(Value::Object(command));
11504 self.scheduled = true;
11505 }
11506
11507 Poll::Pending
11508 }
11509}
11510
11511impl Future for ActivityCall {
11512 type Output = Result<Value>;
11513
11514 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11515 match self.poll_avro_value(cx) {
11516 Poll::Ready(Ok(value)) => Poll::Ready(value.into_json()),
11517 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
11518 Poll::Pending => Poll::Pending,
11519 }
11520 }
11521}
11522
11523pub struct TimerCall {
11525 ctx: WorkflowContext,
11526 delay_seconds: Option<u64>,
11527 scheduled: bool,
11528 matched_pending: bool,
11529 parallel_group_path: Vec<ParallelGroupMetadata>,
11530}
11531
11532impl Future for TimerCall {
11533 type Output = Result<()>;
11534
11535 fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11536 if self.matched_pending {
11537 return Poll::Pending;
11538 }
11539
11540 let ctx = self.ctx.clone();
11541 let Some(requested_delay) = self.delay_seconds else {
11542 return Poll::Ready(Err(Error::TimerDurationOverflow));
11543 };
11544 let mut state = match ctx.state.lock() {
11545 Ok(state) => state,
11546 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11547 };
11548
11549 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
11550 match recorded {
11551 RecordedCommand::Timer {
11552 sequence,
11553 delay_seconds,
11554 fired,
11555 parallel_group_path,
11556 ..
11557 } => {
11558 if let Err(error) = ensure_parallel_path_matches(
11559 sequence,
11560 parallel_group_path.as_deref(),
11561 &self.parallel_group_path,
11562 ) {
11563 return Poll::Ready(Err(error));
11564 }
11565 if delay_seconds != requested_delay {
11566 return Poll::Ready(Err(Error::NonDeterministicReplay(
11567 ReplayFailure::new(
11568 "timer_delay_mismatch",
11569 Some(sequence),
11570 Some(format!("timer:{delay_seconds}s")),
11571 Some(format!("timer:{requested_delay}s")),
11572 "recorded timer delay differs from the current workflow command",
11573 ),
11574 )));
11575 }
11576 state.command_cursor += 1;
11577 if fired {
11578 return Poll::Ready(Ok(()));
11579 }
11580 state.matched_recorded_pending = true;
11581 self.scheduled = true;
11582 self.matched_pending = true;
11583 return Poll::Pending;
11584 }
11585 other => return Poll::Ready(Err(command_mismatch(&other, "timer"))),
11586 }
11587 }
11588
11589 if !self.scheduled {
11590 let mut command = serde_json::Map::from_iter([
11591 ("type".to_string(), json!("start_timer")),
11592 ("delay_seconds".to_string(), json!(requested_delay)),
11593 ]);
11594 apply_parallel_group_path(&mut command, &self.parallel_group_path);
11595 state.commands.push(Value::Object(command));
11596 self.scheduled = true;
11597 }
11598
11599 Poll::Pending
11600 }
11601}
11602
11603pub struct ConditionWaitCall {
11605 ctx: WorkflowContext,
11606 options: ConditionWaitOptions,
11607 predicate: Box<dyn Fn() -> Result<bool> + Send + 'static>,
11608 occurrence_id: Option<String>,
11609 opened_wait: bool,
11610 parallel_group_path: Vec<ParallelGroupMetadata>,
11611}
11612
11613impl Future for ConditionWaitCall {
11614 type Output = Result<ConditionWaitResult>;
11615
11616 fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11617 if self.opened_wait {
11618 return Poll::Pending;
11619 }
11620
11621 let options = match self.options.validate() {
11622 Ok(options) => options,
11623 Err(error) => return Poll::Ready(Err(Error::InvalidConditionWaitOptions(error))),
11624 };
11625 let ctx = self.ctx.clone();
11626 let occurrence_id = match self.occurrence_id.as_ref() {
11627 Some(occurrence_id) => occurrence_id.clone(),
11628 None => {
11629 let mut state = match ctx.state.lock() {
11630 Ok(state) => state,
11631 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11632 };
11633 let ordinal = state.condition_wait_occurrence_counter;
11634 state.condition_wait_occurrence_counter = match ordinal.checked_add(1) {
11635 Some(next) => next,
11636 None => {
11637 return Poll::Ready(Err(Error::WorkerLoop(
11638 "condition wait occurrence counter overflowed".to_string(),
11639 )))
11640 }
11641 };
11642 let occurrence_id = format!("{CONDITION_WAIT_OCCURRENCE_PREFIX}{ordinal}");
11643 drop(state);
11644 self.occurrence_id = Some(occurrence_id.clone());
11645 occurrence_id
11646 }
11647 };
11648
11649 let recorded_result = {
11650 let mut state = match ctx.state.lock() {
11651 Ok(state) => state,
11652 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11653 };
11654 let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() else {
11655 drop(state);
11656 return self.poll_new_condition(options);
11657 };
11658 if !matches!(recorded, RecordedCommand::ConditionWait { .. }) {
11659 return Poll::Ready(Err(command_mismatch(&recorded, "condition wait")));
11660 }
11661
11662 let mut cursor = state.command_cursor;
11663 let mut result = None;
11664 loop {
11665 let Some(RecordedCommand::ConditionWait {
11666 sequence,
11667 occurrence_id: recorded_occurrence_id,
11668 condition_key,
11669 predicate_identity,
11670 timeout_seconds,
11671 result: recorded_result,
11672 parallel_group_path,
11673 ..
11674 }) = state.recorded_commands.get(cursor)
11675 else {
11676 break;
11677 };
11678
11679 if cursor > state.command_cursor && recorded_occurrence_id != &occurrence_id {
11680 break;
11681 }
11682 if let Err(error) = ensure_parallel_path_matches(
11683 *sequence,
11684 parallel_group_path.as_deref(),
11685 &self.parallel_group_path,
11686 ) {
11687 return Poll::Ready(Err(error));
11688 }
11689 if let Err(error) = validate_recorded_condition_wait(
11690 *sequence,
11691 recorded_occurrence_id,
11692 condition_key.as_deref(),
11693 predicate_identity,
11694 *timeout_seconds,
11695 &occurrence_id,
11696 &options,
11697 ) {
11698 return Poll::Ready(Err(error));
11699 }
11700 if result == Some(ConditionWaitResult::TimedOut) {
11701 return Poll::Ready(Err(Error::NonDeterministicReplay(ReplayFailure::new(
11702 "condition_wait_reopened_after_timeout",
11703 Some(*sequence),
11704 Some("timed-out condition is terminal".to_string()),
11705 Some("another physical wait-open".to_string()),
11706 "condition history reopened one logical wait after its durable timeout",
11707 ))));
11708 }
11709 result = *recorded_result;
11710 cursor += 1;
11711 }
11712 state.command_cursor = cursor;
11713 result
11714 };
11715
11716 if let Some(result) = recorded_result {
11717 return Poll::Ready(Ok(result));
11718 }
11719
11720 self.poll_open_condition(options)
11721 }
11722}
11723
11724impl ConditionWaitCall {
11725 fn poll_new_condition(
11726 self: Pin<&mut Self>,
11727 options: ValidatedConditionWaitOptions,
11728 ) -> Poll<Result<ConditionWaitResult>> {
11729 self.poll_open_condition(options)
11730 }
11731
11732 fn poll_open_condition(
11733 mut self: Pin<&mut Self>,
11734 options: ValidatedConditionWaitOptions,
11735 ) -> Poll<Result<ConditionWaitResult>> {
11736 let selection_member = self
11737 .parallel_group_path
11738 .first()
11739 .is_some_and(|entry| entry.parallel_group_mode.as_deref() == Some("select"));
11740 match (self.predicate)() {
11741 Ok(true) if !selection_member => {
11742 return Poll::Ready(Ok(ConditionWaitResult::Satisfied))
11743 }
11744 Ok(_) => {}
11745 Err(error) => return Poll::Ready(Err(error)),
11746 }
11747 if options.timeout_seconds == Some(0) && !selection_member {
11748 return Poll::Ready(Ok(ConditionWaitResult::TimedOut));
11749 }
11750
11751 let ctx = self.ctx.clone();
11752 let mut state = match ctx.state.lock() {
11753 Ok(state) => state,
11754 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11755 };
11756 let mut command = serde_json::Map::from_iter([
11757 ("type".to_string(), json!("open_condition_wait")),
11758 (
11759 "condition_wait_occurrence_id".to_string(),
11760 json!(self.occurrence_id.as_deref().unwrap_or_default()),
11761 ),
11762 ("condition_key".to_string(), json!(options.condition_key)),
11763 (
11764 "condition_definition_fingerprint".to_string(),
11765 json!(options.predicate_identity),
11766 ),
11767 ]);
11768 if let Some(timeout_seconds) = options.timeout_seconds {
11769 command.insert("timeout_seconds".to_string(), json!(timeout_seconds));
11770 }
11771 apply_parallel_group_path(&mut command, &self.parallel_group_path);
11772 state.commands.push(Value::Object(command));
11773 drop(state);
11774 self.opened_wait = true;
11775 Poll::Pending
11776 }
11777}
11778
11779fn validate_recorded_condition_wait(
11780 sequence: u64,
11781 recorded_occurrence_id: &str,
11782 recorded_key: Option<&str>,
11783 recorded_predicate_identity: &str,
11784 recorded_timeout_seconds: Option<u64>,
11785 current_occurrence_id: &str,
11786 current: &ValidatedConditionWaitOptions,
11787) -> Result<()> {
11788 if recorded_occurrence_id != current_occurrence_id {
11789 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
11790 "condition_wait_occurrence_mismatch",
11791 Some(sequence),
11792 Some(recorded_occurrence_id.to_string()),
11793 Some(current_occurrence_id.to_string()),
11794 "recorded condition occurrence differs from the current authored wait position",
11795 )));
11796 }
11797 if recorded_key != Some(current.condition_key.as_str()) {
11798 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
11799 "condition_wait_key_mismatch",
11800 Some(sequence),
11801 recorded_key.map(str::to_string),
11802 Some(current.condition_key.clone()),
11803 "recorded condition identity differs from the current workflow wait",
11804 )));
11805 }
11806 if recorded_predicate_identity != current.predicate_identity {
11807 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
11808 "condition_wait_predicate_mismatch",
11809 Some(sequence),
11810 Some(recorded_predicate_identity.to_string()),
11811 Some(current.predicate_identity.clone()),
11812 "recorded condition predicate behavior differs from current workflow code",
11813 )));
11814 }
11815 if recorded_timeout_seconds != current.timeout_seconds {
11816 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
11817 "condition_wait_timeout_mismatch",
11818 Some(sequence),
11819 recorded_timeout_seconds.map(|seconds| format!("{seconds}s")),
11820 current.timeout_seconds.map(|seconds| format!("{seconds}s")),
11821 "recorded condition timeout differs from the current workflow wait",
11822 )));
11823 }
11824 Ok(())
11825}
11826
11827pub struct ChildWorkflowCall {
11829 ctx: WorkflowContext,
11830 workflow_type: String,
11831 options: ChildWorkflowOptions,
11832 args: Option<Result<AvroValue>>,
11833 scheduled: bool,
11834 matched_pending: bool,
11835 parallel_group_path: Vec<ParallelGroupMetadata>,
11836}
11837
11838impl ChildWorkflowCall {
11839 fn poll_avro_value(
11840 mut self: Pin<&mut Self>,
11841 _cx: &mut TaskContext<'_>,
11842 ) -> Poll<Result<ChildWorkflowAvroResult>> {
11843 if self.matched_pending {
11844 return Poll::Pending;
11845 }
11846
11847 let ctx = self.ctx.clone();
11848 let mut state = match ctx.state.lock() {
11849 Ok(state) => state,
11850 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11851 };
11852
11853 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
11854 let sequence = recorded.sequence();
11855 match recorded {
11856 RecordedCommand::ChildWorkflow {
11857 workflow_type,
11858 outcome,
11859 parallel_group_path,
11860 ..
11861 } => {
11862 if let Err(error) = ensure_parallel_path_matches(
11863 sequence,
11864 parallel_group_path.as_deref(),
11865 &self.parallel_group_path,
11866 ) {
11867 return Poll::Ready(Err(error));
11868 }
11869 if let Some(recorded_type) = workflow_type {
11870 if recorded_type != self.workflow_type {
11871 return Poll::Ready(Err(Error::NonDeterministicReplay(
11872 ReplayFailure::new(
11873 "recorded_command_detail_mismatch",
11874 Some(sequence),
11875 Some(format!("child workflow:{recorded_type}")),
11876 Some(format!("child workflow:{}", self.workflow_type)),
11877 "recorded child workflow type differs from the current workflow command",
11878 ),
11879 )));
11880 }
11881 }
11882 state.command_cursor += 1;
11883 if let Some(outcome) = outcome {
11884 return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed));
11885 }
11886 state.matched_recorded_pending = true;
11887 self.scheduled = true;
11888 self.matched_pending = true;
11889 return Poll::Pending;
11890 }
11891 other => {
11892 return Poll::Ready(Err(command_mismatch(
11893 &other,
11894 format!("child workflow:{}", self.workflow_type),
11895 )));
11896 }
11897 }
11898 }
11899
11900 if !self.scheduled {
11901 if self.options.task_queue.trim().is_empty() {
11902 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
11903 "task_queue must not be empty".to_string(),
11904 )));
11905 }
11906 for (name, value) in [
11907 (
11908 "execution_timeout_seconds",
11909 self.options.execution_timeout_seconds,
11910 ),
11911 ("run_timeout_seconds", self.options.run_timeout_seconds),
11912 ] {
11913 if value == Some(0) {
11914 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(format!(
11915 "{name} must be at least 1"
11916 ))));
11917 }
11918 }
11919
11920 let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
11921 Ok(args) => args,
11922 Err(error) => return Poll::Ready(Err(error)),
11923 };
11924 let arguments = match encode_typed_envelope(
11925 &normalize_avro_arguments(args),
11926 &state.payload_codec,
11927 ) {
11928 Ok(arguments) => arguments,
11929 Err(error) => return Poll::Ready(Err(error)),
11930 };
11931 let mut command = json!({
11932 "type": "start_child_workflow",
11933 "workflow_type": self.workflow_type,
11934 "queue": self.options.task_queue,
11935 "parent_close_policy": self.options.parent_close_policy.as_str(),
11936 "arguments": arguments,
11937 });
11938 let object = command
11939 .as_object_mut()
11940 .expect("child workflow command is always an object");
11941 if let Some(policy) = &self.options.retry_policy {
11942 let mut retry_policy = serde_json::Map::new();
11943 if let Some(max_attempts) = policy.max_attempts {
11944 if max_attempts == 0 {
11945 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
11946 "retry_policy.max_attempts must be at least 1".to_string(),
11947 )));
11948 }
11949 retry_policy.insert("max_attempts".to_string(), json!(max_attempts));
11950 }
11951 if !policy.backoff_seconds.is_empty() {
11952 retry_policy
11953 .insert("backoff_seconds".to_string(), json!(policy.backoff_seconds));
11954 }
11955 if !policy.non_retryable_error_types.is_empty() {
11956 retry_policy.insert(
11957 "non_retryable_error_types".to_string(),
11958 json!(policy.non_retryable_error_types),
11959 );
11960 }
11961 if retry_policy.is_empty() {
11962 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
11963 "retry_policy must configure at least one field".to_string(),
11964 )));
11965 }
11966 object.insert("retry_policy".to_string(), Value::Object(retry_policy));
11967 }
11968 if let Some(seconds) = self.options.execution_timeout_seconds {
11969 object.insert("execution_timeout_seconds".to_string(), json!(seconds));
11970 }
11971 if let Some(seconds) = self.options.run_timeout_seconds {
11972 object.insert("run_timeout_seconds".to_string(), json!(seconds));
11973 }
11974 apply_parallel_group_path(object, &self.parallel_group_path);
11975 state.commands.push(command);
11976 self.scheduled = true;
11977 }
11978
11979 Poll::Pending
11980 }
11981}
11982
11983impl Future for ChildWorkflowCall {
11984 type Output = Result<ChildWorkflowResult>;
11985
11986 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11987 match self.poll_avro_value(cx) {
11988 Poll::Ready(Ok(result)) => match result.result.into_json() {
11989 Ok(projected) => Poll::Ready(Ok(ChildWorkflowResult {
11990 parent: result.parent,
11991 child: result.child,
11992 child_workflow_type: result.child_workflow_type,
11993 result: projected,
11994 })),
11995 Err(error) => Poll::Ready(Err(error)),
11996 },
11997 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
11998 Poll::Pending => Poll::Pending,
11999 }
12000 }
12001}
12002
12003fn command_mismatch(recorded: &RecordedCommand, actual: impl Into<String>) -> Error {
12004 Error::NonDeterministicReplay(ReplayFailure::new(
12005 "recorded_command_mismatch",
12006 Some(recorded.sequence()),
12007 Some(recorded.shape().to_string()),
12008 Some(actual.into()),
12009 "current workflow command does not match the recorded durable command sequence",
12010 ))
12011}
12012
12013pub struct SignalCall {
12014 ctx: WorkflowContext,
12015 signal_name: String,
12016 runtime_reserved_allowed: bool,
12017 opened_wait: bool,
12018 matched_pending: bool,
12019 parallel_group_path: Vec<ParallelGroupMetadata>,
12020}
12021
12022impl SignalCall {
12023 fn poll_avro_value(
12024 mut self: Pin<&mut Self>,
12025 _cx: &mut TaskContext<'_>,
12026 ) -> Poll<Result<Vec<AvroValue>>> {
12027 if self.matched_pending {
12028 return Poll::Pending;
12029 }
12030 if !self.runtime_reserved_allowed {
12031 if let Err(error) = validate_user_signal_name(&self.signal_name) {
12032 return Poll::Ready(Err(error));
12033 }
12034 }
12035
12036 let ctx = self.ctx.clone();
12037 let mut state = match ctx.state.lock() {
12038 Ok(state) => state,
12039 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12040 };
12041
12042 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
12043 match recorded {
12044 RecordedCommand::SignalWait {
12045 sequence,
12046 signal_name,
12047 value,
12048 parallel_group_path,
12049 } => {
12050 if let Err(error) = ensure_parallel_path_matches(
12051 sequence,
12052 parallel_group_path.as_deref(),
12053 &self.parallel_group_path,
12054 ) {
12055 return Poll::Ready(Err(error));
12056 }
12057 if signal_name != self.signal_name {
12058 return Poll::Ready(Err(Error::NonDeterministicReplay(
12059 ReplayFailure::new(
12060 "recorded_command_detail_mismatch",
12061 Some(sequence),
12062 Some(format!("signal wait:{signal_name}")),
12063 Some(format!("signal wait:{}", self.signal_name)),
12064 "recorded signal name differs from the current workflow command",
12065 ),
12066 )));
12067 }
12068
12069 state.command_cursor += 1;
12070 if let Some(value) = value {
12071 return Poll::Ready(Ok(value));
12072 }
12073 if state
12074 .resume_signal
12075 .as_ref()
12076 .is_some_and(|signal| signal.signal_name == self.signal_name)
12077 {
12078 let signal = state
12079 .resume_signal
12080 .take()
12081 .expect("matching resume signal is present");
12082 return Poll::Ready(Ok(signal.arguments));
12083 }
12084
12085 state.matched_recorded_pending = true;
12086 self.opened_wait = true;
12087 self.matched_pending = true;
12088 return Poll::Pending;
12089 }
12090 other => {
12091 return Poll::Ready(Err(command_mismatch(
12092 &other,
12093 format!("signal wait:{}", self.signal_name),
12094 )));
12095 }
12096 }
12097 }
12098
12099 if state
12100 .resume_signal
12101 .as_ref()
12102 .is_some_and(|signal| signal.signal_name == self.signal_name)
12103 {
12104 let signal = state
12105 .resume_signal
12106 .take()
12107 .expect("matching resume signal is present");
12108 return Poll::Ready(Ok(signal.arguments));
12109 }
12110
12111 if !self.opened_wait {
12112 let mut command = serde_json::Map::from_iter([
12113 ("type".to_string(), json!("open_signal_wait")),
12114 ("signal_name".to_string(), json!(self.signal_name)),
12115 ]);
12116 apply_parallel_group_path(&mut command, &self.parallel_group_path);
12117 state.commands.push(Value::Object(command));
12118 self.opened_wait = true;
12119 }
12120
12121 Poll::Pending
12122 }
12123}
12124
12125impl Future for SignalCall {
12126 type Output = Result<Vec<Value>>;
12127
12128 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12129 match self.poll_avro_value(cx) {
12130 Poll::Ready(Ok(values)) => Poll::Ready(
12131 values
12132 .into_iter()
12133 .map(AvroValue::into_json)
12134 .collect::<Result<Vec<_>>>(),
12135 ),
12136 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
12137 Poll::Pending => Poll::Pending,
12138 }
12139 }
12140}
12141
12142#[derive(Clone, Debug)]
12143pub struct ActivityContext {
12144 client: Client,
12145 pub task_id: String,
12146 pub activity_attempt_id: String,
12147 pub lease_owner: String,
12148 pub activity_type: String,
12149 pub attempt_number: u64,
12150 pub task_queue: String,
12151 pub worker_id: String,
12152}
12153
12154impl ActivityContext {
12155 pub async fn heartbeat<T: Serialize>(&self, details: T) -> Result<ActivityHeartbeatResponse> {
12156 self.client
12157 .heartbeat_activity_task(
12158 &self.task_id,
12159 &self.activity_attempt_id,
12160 &self.lease_owner,
12161 details,
12162 )
12163 .await
12164 }
12165}
12166
12167fn decode_task_avro_arguments(value: Option<&Value>, codec: &str) -> Result<AvroValue> {
12168 validate_payload_codec(codec)?;
12169 match value {
12170 Some(value) => Ok(normalize_avro_arguments(decode_wire_avro_value(
12171 value, codec,
12172 )?)),
12173 None => Ok(AvroValue::Array(Vec::new())),
12174 }
12175}
12176
12177fn decode_resume_signal(task: &WorkflowTask) -> Result<Option<ResumeSignal>> {
12178 let Some(signal_name) = task
12179 .signal_name
12180 .as_deref()
12181 .filter(|value| !value.is_empty())
12182 else {
12183 return Ok(None);
12184 };
12185 let decoded = decode_task_avro_arguments(task.signal_arguments.as_ref(), &task.payload_codec)?;
12186 let AvroValue::Array(arguments) = decoded else {
12187 unreachable!("normalize_avro_arguments always returns an array");
12188 };
12189
12190 Ok(Some(ResumeSignal {
12191 signal_name: signal_name.to_string(),
12192 arguments,
12193 }))
12194}
12195
12196fn validate_workflow_task_payloads(task: &WorkflowTask) -> Result<()> {
12197 validate_payload_codec(&task.payload_codec)?;
12198 validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)?;
12199 validate_optional_inbound_payload(task.signal_arguments.as_ref(), &task.payload_codec)?;
12200 for event in &task.history_events {
12201 validate_history_event_payloads(event, &task.payload_codec)?;
12202 }
12203 Ok(())
12204}
12205
12206fn validate_activity_task_payloads(task: &ActivityTask) -> Result<()> {
12207 validate_payload_codec(&task.payload_codec)?;
12208 validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)
12209}
12210
12211fn validate_query_task_payloads(task: &QueryTask) -> Result<()> {
12212 validate_payload_codec(&task.payload_codec)?;
12213 validate_optional_inbound_payload(task.workflow_arguments.as_ref(), &task.payload_codec)?;
12214 validate_optional_inbound_payload(task.query_arguments.as_ref(), &task.payload_codec)?;
12215 for event in &task.history_events {
12216 validate_history_event_payloads(event, &task.payload_codec)?;
12217 }
12218
12219 let Some(export) = task.history_export.as_ref() else {
12220 return Ok(());
12221 };
12222 let export_codec = match export.get("payloads") {
12223 Some(payloads) => declared_payload_codec(payloads, "codec")?,
12224 None => None,
12225 }
12226 .unwrap_or(&task.payload_codec);
12227 validate_payload_codec(export_codec)?;
12228
12229 if let Some(events) = export.get("history_events").and_then(Value::as_array) {
12230 for event in events {
12231 let event_type = event
12232 .get("event_type")
12233 .or_else(|| event.get("type"))
12234 .and_then(Value::as_str)
12235 .unwrap_or_default();
12236 if let Some(payload) = event.get("payload") {
12237 validate_history_payloads(event_type, payload, export_codec)?;
12238 }
12239 }
12240 }
12241 for signal in export
12242 .get("signals")
12243 .and_then(Value::as_array)
12244 .into_iter()
12245 .flatten()
12246 {
12247 let codec = declared_payload_codec(signal, "payload_codec")?.unwrap_or(export_codec);
12248 validate_payload_codec(codec)?;
12249 validate_optional_inbound_payload(signal.get("arguments"), codec)?;
12250 }
12251 for activity in export
12252 .get("activities")
12253 .and_then(Value::as_array)
12254 .into_iter()
12255 .flatten()
12256 {
12257 let codec = declared_payload_codec(activity, "payload_codec")?.unwrap_or(export_codec);
12258 validate_payload_codec(codec)?;
12259 validate_optional_inbound_payload(activity.get("arguments"), codec)?;
12260 validate_optional_inbound_payload(activity.get("result"), codec)?;
12261 }
12262 Ok(())
12263}
12264
12265fn validate_history_event_payloads(event: &HistoryEvent, fallback_codec: &str) -> Result<()> {
12266 validate_history_payloads(&event.event_type, &event.payload, fallback_codec)
12267}
12268
12269fn validate_history_payloads(
12270 event_type: &str,
12271 payload: &Value,
12272 fallback_codec: &str,
12273) -> Result<()> {
12274 let codec = declared_payload_codec(payload, "payload_codec")?.unwrap_or(fallback_codec);
12275 validate_payload_codec(codec)?;
12276 for field in history_payload_fields(event_type) {
12277 validate_optional_inbound_payload(payload.get(*field), codec)?;
12278 }
12279 Ok(())
12280}
12281
12282const SIGNAL_HISTORY_PAYLOAD_FIELDS: &[&str] = &["value", "input", "arguments"];
12283
12284fn history_payload_fields(event_type: &str) -> &'static [&'static str] {
12285 match event_type {
12286 "ActivityCompleted" => &["result"],
12287 "SignalReceived" | "SignalApplied" => SIGNAL_HISTORY_PAYLOAD_FIELDS,
12288 "UpdateAccepted" | "UpdateRejected" | "UpdateApplied" => &["arguments"],
12289 "UpdateCompleted" | "SideEffectRecorded" => &["result"],
12290 "ChildRunCompleted" => &["result", "output"],
12291 "WorkflowCompleted" => &["output"],
12292 "ServiceCallStarted"
12293 | "ServiceCallCompleted"
12294 | "ServiceCallFailed"
12295 | "ServiceCallCancelled" => &["request_payload", "response_payload"],
12296 _ => &[],
12297 }
12298}
12299
12300fn signal_history_payload(payload: &Value) -> Option<&Value> {
12301 SIGNAL_HISTORY_PAYLOAD_FIELDS
12302 .iter()
12303 .find_map(|field| payload.get(*field))
12304}
12305
12306fn declared_payload_codec<'a>(value: &'a Value, field: &str) -> Result<Option<&'a str>> {
12307 match value.get(field) {
12308 None => Ok(None),
12309 Some(Value::String(codec)) => Ok(Some(codec)),
12310 Some(_) => Err(invalid_payload_envelope()),
12311 }
12312}
12313
12314fn validate_optional_inbound_payload(value: Option<&Value>, codec: &str) -> Result<()> {
12315 validate_payload_codec(codec)?;
12316 if let Some(value) = value.filter(|value| !value.is_null()) {
12317 decode_wire_avro_value(value, codec)?;
12318 }
12319 Ok(())
12320}
12321
12322fn recorded_parallel_group_entry(payload: &Value, sequence: u64) -> Result<ParallelGroupMetadata> {
12323 let group_id = payload_string(payload, "parallel_group_id").ok_or_else(|| {
12324 invalid_recorded_history(
12325 "parallel_group_metadata_invalid",
12326 sequence,
12327 "non-empty parallel_group_id",
12328 &payload.to_string(),
12329 "parallel-group history is missing its stable identity",
12330 )
12331 })?;
12332 let kind = payload_string(payload, "parallel_group_kind").ok_or_else(|| {
12333 invalid_recorded_history(
12334 "parallel_group_metadata_invalid",
12335 sequence,
12336 "activity, child, timer, signal, condition, or mixed group kind",
12337 &payload.to_string(),
12338 "parallel-group history is missing its group kind",
12339 )
12340 })?;
12341 if !matches!(
12342 kind.as_str(),
12343 "activity" | "child" | "timer" | "signal" | "condition" | "mixed"
12344 ) {
12345 return Err(invalid_recorded_history(
12346 "parallel_group_metadata_invalid",
12347 sequence,
12348 "activity, child, timer, signal, condition, or mixed group kind",
12349 &kind,
12350 "parallel-group history contains an unsupported group kind",
12351 ));
12352 }
12353 let base_sequence = payload
12354 .get("parallel_group_base_sequence")
12355 .and_then(value_as_u64)
12356 .filter(|value| *value > 0)
12357 .ok_or_else(|| {
12358 invalid_recorded_history(
12359 "parallel_group_metadata_invalid",
12360 sequence,
12361 "positive parallel_group_base_sequence",
12362 &payload.to_string(),
12363 "parallel-group history contains an invalid base sequence",
12364 )
12365 })?;
12366 let size = payload
12367 .get("parallel_group_size")
12368 .and_then(value_as_u64)
12369 .and_then(|value| usize::try_from(value).ok())
12370 .filter(|value| (1..=MAX_PARALLEL_OPERATIONS).contains(value))
12371 .ok_or_else(|| {
12372 invalid_recorded_history(
12373 "parallel_group_metadata_invalid",
12374 sequence,
12375 "bounded positive parallel_group_size",
12376 &payload.to_string(),
12377 "parallel-group history contains an invalid group size",
12378 )
12379 })?;
12380 let index = payload
12381 .get("parallel_group_index")
12382 .and_then(value_as_u64)
12383 .and_then(|value| usize::try_from(value).ok())
12384 .filter(|value| *value < size)
12385 .ok_or_else(|| {
12386 invalid_recorded_history(
12387 "parallel_group_metadata_invalid",
12388 sequence,
12389 "parallel_group_index within group bounds",
12390 &payload.to_string(),
12391 "parallel-group history contains an invalid member index",
12392 )
12393 })?;
12394 if base_sequence.checked_add(u64::try_from(index).unwrap_or(u64::MAX)) != Some(sequence) {
12395 return Err(invalid_recorded_history(
12396 "parallel_group_metadata_invalid",
12397 sequence,
12398 "base sequence plus member index equals workflow sequence",
12399 &payload.to_string(),
12400 "parallel-group path does not preserve durable workflow position",
12401 ));
12402 }
12403 let mode = payload
12404 .get("parallel_group_mode")
12405 .and_then(Value::as_str)
12406 .unwrap_or("all");
12407 if !matches!(mode, "all" | "select") {
12408 return Err(invalid_recorded_history(
12409 "parallel_group_metadata_invalid",
12410 sequence,
12411 "parallel group mode all or select",
12412 mode,
12413 "parallel-group history contains an unsupported group mode",
12414 ));
12415 }
12416 let expected_id = if mode == "select" {
12417 format!("select-calls:{base_sequence}:{size}")
12418 } else {
12419 format!("{}:{base_sequence}:{size}", parallel_group_prefix(&kind))
12420 };
12421 if group_id != expected_id {
12422 return Err(invalid_recorded_history(
12423 "parallel_group_metadata_invalid",
12424 sequence,
12425 &expected_id,
12426 &group_id,
12427 "parallel-group history contains an incompatible stable group ID",
12428 ));
12429 }
12430 let selection_member_key = if mode == "select" {
12431 Some(selection_key_from_value(
12432 payload.get("selection_member_key"),
12433 sequence,
12434 )?)
12435 } else {
12436 None
12437 };
12438 let selection_member_index = if mode == "select" {
12439 Some(required_parallel_usize(
12440 payload,
12441 "selection_member_index",
12442 sequence,
12443 )?)
12444 } else {
12445 None
12446 };
12447 let selection_member_base_sequence = if mode == "select" {
12448 Some(
12449 payload
12450 .get("selection_member_base_sequence")
12451 .and_then(value_as_u64)
12452 .filter(|value| *value >= base_sequence)
12453 .ok_or_else(|| {
12454 invalid_recorded_history(
12455 "parallel_group_metadata_invalid",
12456 sequence,
12457 "selection member base within its group",
12458 &payload.to_string(),
12459 "selection history contains an invalid member base sequence",
12460 )
12461 })?,
12462 )
12463 } else {
12464 None
12465 };
12466 let selection_member_size = if mode == "select" {
12467 let member_size = required_parallel_usize(payload, "selection_member_size", sequence)?;
12468 if member_size == 0 {
12469 return Err(invalid_recorded_history(
12470 "parallel_group_metadata_invalid",
12471 sequence,
12472 "positive selection member size",
12473 &payload.to_string(),
12474 "selection history contains an invalid member size",
12475 ));
12476 }
12477 Some(member_size)
12478 } else {
12479 None
12480 };
12481 let selection_member_kind = if mode == "select" {
12482 let kind = payload_string(payload, "selection_member_kind").ok_or_else(|| {
12483 invalid_recorded_history(
12484 "parallel_group_metadata_invalid",
12485 sequence,
12486 "selection member operation kind",
12487 &payload.to_string(),
12488 "selection history is missing its authored member kind",
12489 )
12490 })?;
12491 if !matches!(
12492 kind.as_str(),
12493 "activity" | "child" | "timer" | "signal" | "condition" | "group"
12494 ) {
12495 return Err(invalid_recorded_history(
12496 "parallel_group_metadata_invalid",
12497 sequence,
12498 "activity, child, timer, signal, condition, or group selection member kind",
12499 &kind,
12500 "selection history contains an unsupported member kind",
12501 ));
12502 }
12503 Some(kind)
12504 } else {
12505 None
12506 };
12507 if let (Some(member_base), Some(member_size)) =
12508 (selection_member_base_sequence, selection_member_size)
12509 {
12510 let member_end = member_base
12511 .checked_add(u64::try_from(member_size).unwrap_or(u64::MAX))
12512 .ok_or_else(|| {
12513 invalid_recorded_history(
12514 "parallel_group_metadata_invalid",
12515 sequence,
12516 "bounded selection member range",
12517 &payload.to_string(),
12518 "selection member range overflowed",
12519 )
12520 })?;
12521 let group_end = base_sequence
12522 .checked_add(u64::try_from(size).unwrap_or(u64::MAX))
12523 .unwrap_or(u64::MAX);
12524 if sequence < member_base || sequence >= member_end || member_end > group_end {
12525 return Err(invalid_recorded_history(
12526 "parallel_group_metadata_invalid",
12527 sequence,
12528 "workflow sequence within one bounded selection member",
12529 &payload.to_string(),
12530 "selection member range does not contain its durable leaf",
12531 ));
12532 }
12533 }
12534 Ok(ParallelGroupMetadata {
12535 parallel_group_id: group_id,
12536 parallel_group_kind: kind,
12537 parallel_group_base_sequence: base_sequence,
12538 parallel_group_size: size,
12539 parallel_group_index: index,
12540 parallel_group_mode: (mode == "select").then(|| "select".to_string()),
12541 selection_member_key,
12542 selection_member_index,
12543 selection_member_base_sequence,
12544 selection_member_size,
12545 selection_member_kind,
12546 })
12547}
12548
12549fn required_parallel_usize(payload: &Value, field: &str, sequence: u64) -> Result<usize> {
12550 payload
12551 .get(field)
12552 .and_then(value_as_u64)
12553 .and_then(|value| usize::try_from(value).ok())
12554 .ok_or_else(|| {
12555 invalid_recorded_history(
12556 "parallel_group_metadata_invalid",
12557 sequence,
12558 &format!("non-negative integer {field}"),
12559 &payload.to_string(),
12560 "selection history contains invalid member metadata",
12561 )
12562 })
12563}
12564
12565fn selection_key_from_value(value: Option<&Value>, sequence: u64) -> Result<SelectionKey> {
12566 match value {
12567 Some(Value::String(value)) if !value.is_empty() => Ok(SelectionKey::Name(value.clone())),
12568 Some(value) => value_as_u64(value)
12569 .and_then(|value| usize::try_from(value).ok())
12570 .map(SelectionKey::Index)
12571 .ok_or_else(|| {
12572 invalid_recorded_history(
12573 "selection_member_key_invalid",
12574 sequence,
12575 "non-empty string or non-negative integer member key",
12576 &value.to_string(),
12577 "selection history contains an invalid member key",
12578 )
12579 }),
12580 None => Err(invalid_recorded_history(
12581 "selection_member_key_missing",
12582 sequence,
12583 "selection_member_key",
12584 "<missing>",
12585 "selection history is missing its stable member key",
12586 )),
12587 }
12588}
12589
12590fn recorded_parallel_group_path(
12591 events: &[&HistoryEvent],
12592 sequence: u64,
12593) -> Result<Option<Vec<ParallelGroupMetadata>>> {
12594 let mut recorded: Option<Vec<ParallelGroupMetadata>> = None;
12595 for event in events {
12596 let payload = &event.payload;
12597 let has_metadata = payload.get("parallel_group_path").is_some()
12598 || payload.get("parallel_group_id").is_some()
12599 || payload.get("parallel_group_kind").is_some()
12600 || payload.get("parallel_group_base_sequence").is_some()
12601 || payload.get("parallel_group_size").is_some()
12602 || payload.get("parallel_group_index").is_some()
12603 || payload.get("parallel_group_mode").is_some()
12604 || payload.get("selection_member_key").is_some();
12605 if !has_metadata {
12606 continue;
12607 }
12608
12609 let top_level = recorded_parallel_group_entry(payload, sequence)?;
12610 let path = match payload.get("parallel_group_path") {
12611 None => vec![top_level.clone()],
12612 Some(Value::Array(entries)) if !entries.is_empty() => entries
12613 .iter()
12614 .map(|entry| recorded_parallel_group_entry(entry, sequence))
12615 .collect::<Result<Vec<_>>>()?,
12616 Some(value) => {
12617 return Err(invalid_recorded_history(
12618 "parallel_group_metadata_invalid",
12619 sequence,
12620 "non-empty parallel_group_path list",
12621 &value.to_string(),
12622 "parallel-group history contains an invalid group path",
12623 ));
12624 }
12625 };
12626 if path.last() != Some(&top_level) {
12627 return Err(invalid_recorded_history(
12628 "parallel_group_metadata_invalid",
12629 sequence,
12630 &serde_json::to_string(&path.last()).unwrap_or_default(),
12631 &serde_json::to_string(&top_level).unwrap_or_default(),
12632 "parallel-group top-level fields do not match the innermost path entry",
12633 ));
12634 }
12635 if recorded.as_ref().is_some_and(|existing| existing != &path) {
12636 return Err(invalid_recorded_history(
12637 "parallel_group_history_conflict",
12638 sequence,
12639 &serde_json::to_string(&recorded.as_ref()).unwrap_or_default(),
12640 &serde_json::to_string(&path).unwrap_or_default(),
12641 "parallel-group metadata changed between scheduling and resolution history",
12642 ));
12643 }
12644 recorded = Some(path);
12645 }
12646 Ok(recorded)
12647}
12648
12649fn recorded_commands(
12650 events: &[HistoryEvent],
12651 fallback_codec: &str,
12652 parent: WorkflowIdentity,
12653) -> Result<Vec<RecordedCommand>> {
12654 let mut events_by_sequence: BTreeMap<u64, Vec<&HistoryEvent>> = BTreeMap::new();
12655 let mut last_new_sequence = None;
12656
12657 for event in events {
12658 let is_activity = matches!(
12659 event.event_type.as_str(),
12660 "ActivityScheduled"
12661 | "ActivityStarted"
12662 | "ActivityHeartbeatRecorded"
12663 | "ActivityRetryScheduled"
12664 | "ActivityCompleted"
12665 | "ActivityFailed"
12666 | "ActivityCancelled"
12667 | "ActivityTimedOut"
12668 );
12669 let is_workflow_timer = matches!(
12670 event.event_type.as_str(),
12671 "TimerScheduled" | "TimerCancelled" | "TimerFired"
12672 ) && !is_internal_timer_event(event);
12673 let is_child_workflow = matches!(
12674 event.event_type.as_str(),
12675 "ChildWorkflowScheduled"
12676 | "ChildRunCompleted"
12677 | "ChildRunFailed"
12678 | "ChildRunCancelled"
12679 | "ChildRunTerminated"
12680 );
12681 let is_signal_wait = is_recorded_signal_wait_event(event);
12682 let is_condition_wait = is_recorded_condition_wait_event(event);
12683 let is_search_attributes = event.event_type == "SearchAttributesUpserted";
12684 let is_side_effect = event.event_type == "SideEffectRecorded";
12685 let is_version_marker = event.event_type == "VersionMarkerRecorded";
12686 let is_memo = event.event_type == "MemoUpserted";
12687 if !is_activity
12688 && !is_workflow_timer
12689 && !is_child_workflow
12690 && !is_signal_wait
12691 && !is_condition_wait
12692 && !is_search_attributes
12693 && !is_side_effect
12694 && !is_version_marker
12695 && !is_memo
12696 {
12697 continue;
12698 }
12699
12700 let sequence = durable_event_sequence(event).ok_or_else(|| {
12701 Error::NonDeterministicReplay(ReplayFailure::new(
12702 "durable_command_sequence_missing",
12703 None,
12704 Some("positive workflow sequence".to_string()),
12705 Some(event.event_type.clone()),
12706 "durable command history event has no workflow sequence",
12707 ))
12708 })?;
12709 if sequence == 0 {
12710 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12711 "durable_command_sequence_invalid",
12712 Some(sequence),
12713 Some("positive workflow sequence".to_string()),
12714 Some(sequence.to_string()),
12715 "durable command history uses an invalid workflow sequence",
12716 )));
12717 }
12718 if !events_by_sequence.contains_key(&sequence) {
12719 if let Some(previous) = last_new_sequence {
12720 if sequence < previous {
12721 return Err(invalid_recorded_history(
12722 "durable_command_sequence_mismatch",
12723 sequence,
12724 &format!("workflow sequence greater than {previous}"),
12725 &sequence.to_string(),
12726 "durable commands are not strictly ordered by their recorded workflow sequence",
12727 ));
12728 }
12729 }
12730 last_new_sequence = Some(sequence);
12731 }
12732 events_by_sequence.entry(sequence).or_default().push(event);
12733 }
12734
12735 let commands: Vec<RecordedCommand> = events_by_sequence
12736 .into_iter()
12737 .map(|(sequence, sequence_events)| {
12738 let activity_events: Vec<_> = sequence_events
12739 .iter()
12740 .copied()
12741 .filter(|event| event.event_type.starts_with("Activity"))
12742 .collect();
12743 let timer_events: Vec<_> = sequence_events
12744 .iter()
12745 .copied()
12746 .filter(|event| event.event_type.starts_with("Timer"))
12747 .collect();
12748 let child_events: Vec<_> = sequence_events
12749 .iter()
12750 .copied()
12751 .filter(|event| {
12752 event.event_type == "ChildWorkflowScheduled"
12753 || event.event_type.starts_with("ChildRun")
12754 })
12755 .collect();
12756 let signal_wait_events: Vec<_> = sequence_events
12757 .iter()
12758 .copied()
12759 .filter(|event| is_recorded_signal_wait_event(event))
12760 .collect();
12761 let condition_wait_events: Vec<_> = sequence_events
12762 .iter()
12763 .copied()
12764 .filter(|event| is_recorded_condition_wait_event(event))
12765 .collect();
12766 let search_attribute_events: Vec<_> = sequence_events
12767 .iter()
12768 .copied()
12769 .filter(|event| event.event_type == "SearchAttributesUpserted")
12770 .collect();
12771 let side_effect_events: Vec<_> = sequence_events
12772 .iter()
12773 .copied()
12774 .filter(|event| event.event_type == "SideEffectRecorded")
12775 .collect();
12776 let version_marker_events: Vec<_> = sequence_events
12777 .iter()
12778 .copied()
12779 .filter(|event| event.event_type == "VersionMarkerRecorded")
12780 .collect();
12781 let memo_events: Vec<_> = sequence_events
12782 .iter()
12783 .copied()
12784 .filter(|event| event.event_type == "MemoUpserted")
12785 .collect();
12786
12787 let command_kind_count = usize::from(!activity_events.is_empty())
12788 + usize::from(!timer_events.is_empty())
12789 + usize::from(!child_events.is_empty())
12790 + usize::from(!signal_wait_events.is_empty())
12791 + usize::from(!condition_wait_events.is_empty())
12792 + usize::from(!search_attribute_events.is_empty())
12793 + usize::from(!side_effect_events.is_empty())
12794 + usize::from(!version_marker_events.is_empty())
12795 + usize::from(!memo_events.is_empty());
12796 if command_kind_count > 1 {
12797 let actual = [
12798 (!activity_events.is_empty()).then_some("activity"),
12799 (!timer_events.is_empty()).then_some("timer"),
12800 (!child_events.is_empty()).then_some("child workflow"),
12801 (!signal_wait_events.is_empty()).then_some("signal wait"),
12802 (!condition_wait_events.is_empty()).then_some("condition wait"),
12803 (!search_attribute_events.is_empty()).then_some("search-attribute update"),
12804 (!side_effect_events.is_empty()).then_some("side effect"),
12805 (!version_marker_events.is_empty()).then_some("version marker"),
12806 (!memo_events.is_empty()).then_some("memo upsert"),
12807 ]
12808 .into_iter()
12809 .flatten()
12810 .collect::<Vec<_>>()
12811 .join(" and ");
12812 return Err(invalid_recorded_history(
12813 "durable_command_sequence_collision",
12814 sequence,
12815 "one durable command kind",
12816 &actual,
12817 "one workflow sequence records more than one durable command kind",
12818 ));
12819 }
12820
12821 if !activity_events.is_empty() {
12822 let parallel_group_path =
12823 recorded_parallel_group_path(&activity_events, sequence)?;
12824 let scheduled_count = activity_events
12825 .iter()
12826 .filter(|event| event.event_type == "ActivityScheduled")
12827 .count();
12828 if scheduled_count > 1 {
12829 return Err(invalid_recorded_history(
12830 "duplicate_activity_schedule",
12831 sequence,
12832 "at most one ActivityScheduled event",
12833 "multiple ActivityScheduled events",
12834 "activity history schedules more than one command at one workflow sequence",
12835 ));
12836 }
12837 let activity_type = activity_events.iter().find_map(|event| {
12838 event
12839 .payload
12840 .get("activity_type")
12841 .or_else(|| event.payload.get("activity_name"))
12842 .and_then(Value::as_str)
12843 .map(str::to_string)
12844 });
12845 if activity_events.iter().filter_map(|event| {
12846 event
12847 .payload
12848 .get("activity_type")
12849 .or_else(|| event.payload.get("activity_name"))
12850 .and_then(Value::as_str)
12851 }).any(|candidate| Some(candidate) != activity_type.as_deref()) {
12852 return Err(invalid_recorded_history(
12853 "activity_identity_mismatch",
12854 sequence,
12855 activity_type.as_deref().unwrap_or("one activity identity"),
12856 "conflicting activity identities",
12857 "activity lifecycle events at one workflow sequence disagree on identity",
12858 ));
12859 }
12860 let terminal: Vec<_> = activity_events
12861 .iter()
12862 .copied()
12863 .filter(|event| {
12864 matches!(
12865 event.event_type.as_str(),
12866 "ActivityCompleted"
12867 | "ActivityFailed"
12868 | "ActivityCancelled"
12869 | "ActivityTimedOut"
12870 )
12871 })
12872 .collect();
12873 let duplicate_delivery = terminal.first().is_some_and(|first| {
12874 terminal.iter().all(|event| {
12875 event.event_type == first.event_type && event.payload == first.payload
12876 })
12877 });
12878 if terminal.len() > 1 && !duplicate_delivery {
12879 return Err(invalid_recorded_history(
12880 "duplicate_activity_terminal_event",
12881 sequence,
12882 "at most one terminal activity event",
12883 "multiple terminal activity events",
12884 "activity history settles one command more than once",
12885 ));
12886 }
12887 let outcome = terminal
12888 .first()
12889 .map(|event| activity_outcome(event, fallback_codec, activity_type.clone()))
12890 .transpose()?;
12891 let options = activity_events
12892 .iter()
12893 .find(|event| event.event_type == "ActivityScheduled")
12894 .and_then(|event| event.payload.get("activity"))
12895 .and_then(Value::as_object)
12896 .map(|activity| RecordedActivityOptions {
12897 task_queue: recorded_optional_string(activity, "queue"),
12898 execution_mode: recorded_optional_string(activity, "execution_mode"),
12899 retry_policy: recorded_activity_retry_snapshot(
12900 activity.get("retry_policy"),
12901 ),
12902 });
12903 return Ok(RecordedCommand::Activity {
12904 sequence,
12905 activity_type,
12906 options,
12907 outcome,
12908 parallel_group_path,
12909 });
12910 }
12911
12912 if !child_events.is_empty() {
12913 let parallel_group_path = recorded_parallel_group_path(&child_events, sequence)?;
12914 let scheduled: Vec<_> = child_events
12915 .iter()
12916 .copied()
12917 .filter(|event| event.event_type == "ChildWorkflowScheduled")
12918 .collect();
12919 if scheduled.len() != 1 {
12920 return Err(invalid_recorded_history(
12921 "child_workflow_schedule_missing_or_duplicate",
12922 sequence,
12923 "one ChildWorkflowScheduled event",
12924 &format!("{} ChildWorkflowScheduled events", scheduled.len()),
12925 "child workflow replay requires exactly one recorded schedule event",
12926 ));
12927 }
12928 let workflow_type = child_events.iter().find_map(|event| {
12929 event
12930 .payload
12931 .get("child_workflow_type")
12932 .or_else(|| event.payload.get("workflow_type"))
12933 .and_then(Value::as_str)
12934 .filter(|value| !value.is_empty())
12935 .map(str::to_string)
12936 });
12937 if child_events
12938 .iter()
12939 .filter_map(|event| {
12940 event
12941 .payload
12942 .get("child_workflow_type")
12943 .or_else(|| event.payload.get("workflow_type"))
12944 .and_then(Value::as_str)
12945 })
12946 .any(|candidate| Some(candidate) != workflow_type.as_deref())
12947 {
12948 return Err(invalid_recorded_history(
12949 "child_workflow_identity_mismatch",
12950 sequence,
12951 workflow_type
12952 .as_deref()
12953 .unwrap_or("one child workflow type"),
12954 "conflicting child workflow types",
12955 "child workflow lifecycle events at one sequence disagree on type",
12956 ));
12957 }
12958 let mut outcomes = child_workflow_outcomes(
12959 &child_events.iter().map(|event| (*event).clone()).collect::<Vec<_>>(),
12960 fallback_codec,
12961 parent.clone(),
12962 )?;
12963 let terminal_events = child_events
12964 .iter()
12965 .copied()
12966 .filter(|event| event.event_type.starts_with("ChildRun"))
12967 .collect::<Vec<_>>();
12968 let duplicate_delivery = terminal_events.first().is_some_and(|first| {
12969 terminal_events.iter().all(|event| {
12970 event.event_type == first.event_type && event.payload == first.payload
12971 })
12972 });
12973 if outcomes.len() > 1 && !duplicate_delivery {
12974 return Err(invalid_recorded_history(
12975 "duplicate_child_workflow_terminal_event",
12976 sequence,
12977 "at most one terminal child event",
12978 "multiple terminal child events",
12979 "child workflow history settles one command more than once",
12980 ));
12981 }
12982 return Ok(RecordedCommand::ChildWorkflow {
12983 sequence,
12984 workflow_type,
12985 outcome: outcomes.pop(),
12986 parallel_group_path,
12987 });
12988 }
12989
12990 if !signal_wait_events.is_empty() {
12991 let opened: Vec<_> = signal_wait_events
12992 .iter()
12993 .copied()
12994 .filter(|event| event.event_type == "SignalWaitOpened")
12995 .collect();
12996 if opened.len() != 1 {
12997 return Err(invalid_recorded_history(
12998 "signal_wait_open_missing_or_duplicate",
12999 sequence,
13000 "one SignalWaitOpened event",
13001 &format!("{} SignalWaitOpened events", opened.len()),
13002 "signal replay requires exactly one canonical wait-open event",
13003 ));
13004 }
13005
13006 let applied: Vec<_> = signal_wait_events
13007 .iter()
13008 .copied()
13009 .filter(|event| event.event_type == "SignalApplied")
13010 .collect();
13011 if applied.len() > 1 {
13012 return Err(invalid_recorded_history(
13013 "duplicate_signal_wait_apply",
13014 sequence,
13015 "at most one SignalApplied event",
13016 "multiple SignalApplied events",
13017 "signal history applies one durable wait more than once",
13018 ));
13019 }
13020
13021 let signal_names = signal_wait_events
13022 .iter()
13023 .map(|event| required_signal_wait_name(event, sequence))
13024 .collect::<Result<Vec<_>>>()?;
13025 let signal_name = signal_names
13026 .first()
13027 .expect("signal wait events are not empty")
13028 .clone();
13029 if signal_names.iter().any(|candidate| candidate != &signal_name) {
13030 return Err(invalid_recorded_history(
13031 "signal_wait_identity_mismatch",
13032 sequence,
13033 &signal_name,
13034 "conflicting signal names",
13035 "signal wait lifecycle events at one workflow sequence disagree on identity",
13036 ));
13037 }
13038 let value = applied
13039 .first()
13040 .map(|event| decode_signal_event_arguments(event, fallback_codec))
13041 .transpose()?;
13042 return Ok(RecordedCommand::SignalWait {
13043 sequence,
13044 signal_name,
13045 value,
13046 parallel_group_path: recorded_parallel_group_path(
13047 &signal_wait_events,
13048 sequence,
13049 )?,
13050 });
13051 }
13052
13053 if !condition_wait_events.is_empty() {
13054 return recorded_condition_wait(
13055 sequence,
13056 &condition_wait_events,
13057 events,
13058 );
13059 }
13060
13061 if !search_attribute_events.is_empty() {
13062 if search_attribute_events.len() != 1 {
13063 return Err(invalid_recorded_history(
13064 "duplicate_search_attribute_update",
13065 sequence,
13066 "one SearchAttributesUpserted event",
13067 &format!(
13068 "{} SearchAttributesUpserted events",
13069 search_attribute_events.len()
13070 ),
13071 "search-attribute history records one workflow command more than once",
13072 ));
13073 }
13074 let payload = &search_attribute_events[0].payload;
13075 let attributes = payload
13076 .get("attributes")
13077 .filter(|value| value.as_object().is_some_and(|values| !values.is_empty()))
13078 .cloned()
13079 .ok_or_else(|| {
13080 invalid_recorded_history(
13081 "search_attribute_update_missing",
13082 sequence,
13083 "non-empty attributes object",
13084 "missing or invalid attributes",
13085 "search-attribute history is missing its recorded mutation",
13086 )
13087 })?;
13088 let attribute_types =
13089 recorded_search_attribute_types(payload, &attributes, sequence)?;
13090 return Ok(RecordedCommand::SearchAttributes {
13091 sequence,
13092 attributes,
13093 attribute_types,
13094 });
13095 }
13096
13097 if !side_effect_events.is_empty() {
13098 if side_effect_events.len() != 1 {
13099 return Err(invalid_recorded_history(
13100 "duplicate_side_effect_record",
13101 sequence,
13102 "one SideEffectRecorded event",
13103 &format!("{} SideEffectRecorded events", side_effect_events.len()),
13104 "side-effect history records one workflow command more than once",
13105 ));
13106 }
13107 let event = side_effect_events[0];
13108 let result = event.payload.get("result").ok_or_else(|| {
13109 invalid_recorded_history(
13110 "side_effect_result_missing",
13111 sequence,
13112 "recorded result payload",
13113 "missing result",
13114 "side-effect history is missing its recorded value",
13115 )
13116 })?;
13117 let has_published_envelope = result.as_str().is_some()
13118 || result.as_object().is_some_and(|envelope| {
13119 envelope.get("codec").and_then(Value::as_str).is_some()
13120 && envelope.get("blob").and_then(Value::as_str).is_some()
13121 });
13122 if !has_published_envelope {
13123 return Err(invalid_recorded_history(
13124 "side_effect_payload_malformed",
13125 sequence,
13126 "payload blob or {codec, blob} envelope",
13127 &result.to_string(),
13128 "side-effect history result does not use a published payload envelope",
13129 ));
13130 }
13131 let codec = event
13132 .payload
13133 .get("payload_codec")
13134 .and_then(Value::as_str)
13135 .unwrap_or(fallback_codec);
13136 let value = decode_wire_avro_value(result, codec).map_err(|error| {
13137 if error.to_string().contains("unsupported_payload_codec") {
13138 return error;
13139 }
13140
13141 invalid_recorded_history(
13142 "side_effect_payload_incompatible",
13143 sequence,
13144 &format!("valid {codec} payload envelope"),
13145 &error.to_string(),
13146 "side-effect history payload cannot be decoded with its recorded codec",
13147 )
13148 })?;
13149 return Ok(RecordedCommand::SideEffect { sequence, value });
13150 }
13151
13152 if !version_marker_events.is_empty() {
13153 if version_marker_events.len() != 1 {
13154 return Err(invalid_recorded_history(
13155 "duplicate_version_marker_record",
13156 sequence,
13157 "one VersionMarkerRecorded event",
13158 &format!("{} VersionMarkerRecorded events", version_marker_events.len()),
13159 "version-marker history records one workflow command more than once",
13160 ));
13161 }
13162 let payload = &version_marker_events[0].payload;
13163 let change_id = payload
13164 .get("change_id")
13165 .and_then(Value::as_str)
13166 .filter(|value| !value.is_empty())
13167 .map(str::to_string)
13168 .ok_or_else(|| {
13169 invalid_recorded_history(
13170 "version_marker_field_missing",
13171 sequence,
13172 "non-empty change_id",
13173 "missing or invalid change_id",
13174 "version-marker history is missing its stable change ID",
13175 )
13176 })?;
13177 let version = required_version_i32(payload, "version", sequence)?;
13178 let min_supported = required_version_i32(payload, "min_supported", sequence)?;
13179 let max_supported = required_version_i32(payload, "max_supported", sequence)?;
13180 if min_supported > max_supported || version < min_supported || version > max_supported {
13181 return Err(invalid_recorded_history(
13182 "version_marker_history_range_invalid",
13183 sequence,
13184 "min_supported <= version <= max_supported",
13185 &format!("{min_supported} <= {version} <= {max_supported}"),
13186 "recorded version marker contains an internally incompatible range",
13187 ));
13188 }
13189 return Ok(RecordedCommand::VersionMarker {
13190 sequence,
13191 change_id,
13192 version,
13193 });
13194 }
13195
13196 if !memo_events.is_empty() {
13197 if memo_events.len() != 1 {
13198 return Err(invalid_recorded_history(
13199 "duplicate_memo_upsert_record",
13200 sequence,
13201 "one MemoUpserted event",
13202 &format!("{} MemoUpserted events", memo_events.len()),
13203 "memo history records one workflow update more than once",
13204 ));
13205 }
13206 let payload = &memo_events[0].payload;
13207 let entries = payload.get("entries").cloned().ok_or_else(|| {
13208 invalid_recorded_history(
13209 "memo_entries_missing",
13210 sequence,
13211 "memo entries object",
13212 "missing entries",
13213 "MemoUpserted history is missing replay identity entries",
13214 )
13215 })?;
13216 let entries = decode_memo_history_map(&entries, true).map_err(|error| {
13217 invalid_recorded_history(
13218 "memo_entries_invalid",
13219 sequence,
13220 "valid canonical memo entries",
13221 &error.to_string(),
13222 "MemoUpserted history contains invalid replay identity entries",
13223 )
13224 })?;
13225 let merged = payload.get("merged").cloned().ok_or_else(|| {
13226 invalid_recorded_history(
13227 "memo_merged_projection_missing",
13228 sequence,
13229 "merged memo projection",
13230 "missing merged",
13231 "MemoUpserted history is missing its merged projection",
13232 )
13233 })?;
13234 decode_memo_history_map(&merged, false).map_err(|error| {
13235 invalid_recorded_history(
13236 "memo_merged_projection_invalid",
13237 sequence,
13238 "valid merged memo projection",
13239 &error.to_string(),
13240 "MemoUpserted history contains an invalid merged projection",
13241 )
13242 })?;
13243
13244 return Ok(RecordedCommand::Memo { sequence, entries });
13245 }
13246 let scheduled: Vec<_> = timer_events
13247 .iter()
13248 .copied()
13249 .filter(|event| event.event_type == "TimerScheduled")
13250 .collect();
13251 let fired: Vec<_> = timer_events
13252 .iter()
13253 .copied()
13254 .filter(|event| event.event_type == "TimerFired")
13255 .collect();
13256 if scheduled.len() != 1 {
13257 return Err(invalid_recorded_history(
13258 "timer_schedule_missing_or_duplicate",
13259 sequence,
13260 "one TimerScheduled event",
13261 &format!("{} TimerScheduled events", scheduled.len()),
13262 "timer replay requires exactly one recorded schedule event",
13263 ));
13264 }
13265 if fired.len() > 1 {
13266 return Err(invalid_recorded_history(
13267 "duplicate_timer_fire",
13268 sequence,
13269 "at most one TimerFired event",
13270 "multiple TimerFired events",
13271 "timer history contains more than one fire event for a workflow sequence",
13272 ));
13273 }
13274
13275 let scheduled = scheduled[0];
13276 let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
13277 let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
13278 if let Some(fired) = fired.first() {
13279 let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
13280 if fired_timer_id != timer_id {
13281 return Err(invalid_recorded_history(
13282 "timer_identity_mismatch",
13283 sequence,
13284 &timer_id,
13285 &fired_timer_id,
13286 "TimerFired does not correspond to the recorded TimerScheduled event",
13287 ));
13288 }
13289 let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
13290 if fired_delay != delay_seconds {
13291 return Err(invalid_recorded_history(
13292 "timer_history_delay_mismatch",
13293 sequence,
13294 &delay_seconds.to_string(),
13295 &fired_delay.to_string(),
13296 "TimerScheduled and TimerFired record different delays",
13297 ));
13298 }
13299 }
13300
13301 Ok(RecordedCommand::Timer {
13302 sequence,
13303 delay_seconds,
13304 fired: !fired.is_empty(),
13305 parallel_group_path: recorded_parallel_group_path(&timer_events, sequence)?,
13306 })
13307 })
13308 .collect::<Result<_>>()?;
13309
13310 let mut marker_sequences = HashMap::new();
13311 for command in &commands {
13312 if let RecordedCommand::VersionMarker {
13313 sequence,
13314 change_id,
13315 ..
13316 } = command
13317 {
13318 if let Some(first_sequence) = marker_sequences.insert(change_id.clone(), *sequence) {
13319 return Err(invalid_recorded_history(
13320 "duplicate_version_marker",
13321 *sequence,
13322 &format!("one marker for change ID {change_id:?}"),
13323 &format!("markers at sequences {first_sequence} and {sequence}"),
13324 "workflow history contains duplicate markers for one stable change ID",
13325 ));
13326 }
13327 }
13328 }
13329
13330 Ok(commands)
13331}
13332
13333fn required_version_i32(payload: &Value, field: &str, sequence: u64) -> Result<i32> {
13334 payload
13335 .get(field)
13336 .and_then(Value::as_i64)
13337 .and_then(|value| i32::try_from(value).ok())
13338 .ok_or_else(|| {
13339 invalid_recorded_history(
13340 "version_marker_field_missing",
13341 sequence,
13342 &format!("integer {field}"),
13343 "missing or out-of-range integer",
13344 "version-marker history is missing a required integer field",
13345 )
13346 })
13347}
13348
13349fn durable_event_sequence(event: &HistoryEvent) -> Option<u64> {
13350 event
13351 .payload
13352 .get("sequence")
13353 .or_else(|| event.payload.get("workflow_sequence"))
13354 .or_else(|| event.raw.get("sequence"))
13355 .or_else(|| event.raw.get("workflow_sequence"))
13356 .and_then(value_as_u64)
13357}
13358
13359fn is_internal_timer_event(event: &HistoryEvent) -> bool {
13360 matches!(
13361 event
13362 .payload
13363 .get("timer_kind")
13364 .or_else(|| event.raw.get("timer_kind"))
13365 .and_then(Value::as_str),
13366 Some("condition_timeout" | "signal_timeout")
13367 )
13368}
13369
13370fn is_recorded_condition_wait_event(event: &HistoryEvent) -> bool {
13371 matches!(
13372 event.event_type.as_str(),
13373 "ConditionWaitOpened" | "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
13374 )
13375}
13376
13377fn recorded_condition_wait(
13378 sequence: u64,
13379 condition_events: &[&HistoryEvent],
13380 all_events: &[HistoryEvent],
13381) -> Result<RecordedCommand> {
13382 let opened = condition_events
13383 .iter()
13384 .copied()
13385 .filter(|event| event.event_type == "ConditionWaitOpened")
13386 .collect::<Vec<_>>();
13387 if opened.len() != 1 {
13388 return Err(invalid_recorded_history(
13389 "condition_wait_open_missing_or_duplicate",
13390 sequence,
13391 "one ConditionWaitOpened event",
13392 &format!("{} ConditionWaitOpened events", opened.len()),
13393 "condition replay requires exactly one canonical wait-open event",
13394 ));
13395 }
13396 let terminal = condition_events
13397 .iter()
13398 .copied()
13399 .filter(|event| {
13400 matches!(
13401 event.event_type.as_str(),
13402 "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
13403 )
13404 })
13405 .collect::<Vec<_>>();
13406 if terminal.len() > 1 {
13407 return Err(invalid_recorded_history(
13408 "duplicate_condition_wait_terminal_event",
13409 sequence,
13410 "at most one condition terminal event",
13411 "multiple condition terminal events",
13412 "condition history settles one durable wait more than once",
13413 ));
13414 }
13415
13416 let opened = opened[0];
13417 let condition_wait_id = required_condition_wait_id(opened, sequence)?;
13418 let occurrence_id = required_condition_wait_occurrence_id(opened, sequence)?;
13419 for event in condition_events
13420 .iter()
13421 .copied()
13422 .filter(|event| !std::ptr::eq(*event, opened))
13423 {
13424 let event_wait_id = required_condition_wait_id(event, sequence)?;
13425 if event_wait_id != condition_wait_id {
13426 return Err(invalid_recorded_history(
13427 "condition_wait_id_mismatch",
13428 sequence,
13429 &condition_wait_id,
13430 &event_wait_id,
13431 "condition lifecycle events at one sequence disagree on wait identity",
13432 ));
13433 }
13434 let event_occurrence_id = required_condition_wait_occurrence_id(event, sequence)?;
13435 if event_occurrence_id != occurrence_id {
13436 return Err(invalid_recorded_history(
13437 "condition_wait_occurrence_history_mismatch",
13438 sequence,
13439 &occurrence_id,
13440 &event_occurrence_id,
13441 "condition lifecycle events at one sequence disagree on authored occurrence identity",
13442 ));
13443 }
13444 }
13445
13446 let condition_key = optional_non_empty_history_string(opened, "condition_key");
13447 let predicate_identity = opened
13448 .payload
13449 .get("condition_definition_fingerprint")
13450 .and_then(Value::as_str)
13451 .filter(|value| !value.is_empty())
13452 .map(str::to_string)
13453 .ok_or_else(|| {
13454 invalid_recorded_history(
13455 "condition_wait_predicate_fingerprint_missing",
13456 sequence,
13457 "non-empty condition_definition_fingerprint",
13458 &opened.event_type,
13459 "canonical condition history is missing its predicate identity",
13460 )
13461 })?;
13462 let timeout_seconds = optional_history_u64(opened, "timeout_seconds", sequence)?;
13463 for event in condition_events
13464 .iter()
13465 .copied()
13466 .filter(|event| !std::ptr::eq(*event, opened))
13467 {
13468 for (field, opened_value) in [
13469 ("condition_key", condition_key.as_deref()),
13470 (
13471 "condition_definition_fingerprint",
13472 Some(predicate_identity.as_str()),
13473 ),
13474 ] {
13475 if let Some(value) = optional_non_empty_history_string(event, field) {
13476 if opened_value.is_some_and(|opened_value| opened_value != value) {
13477 return Err(invalid_recorded_history(
13478 "condition_wait_definition_history_mismatch",
13479 sequence,
13480 opened_value.unwrap_or_default(),
13481 &value,
13482 "condition lifecycle events disagree on the recorded definition",
13483 ));
13484 }
13485 }
13486 }
13487 if let Some(event_timeout) = optional_history_u64(event, "timeout_seconds", sequence)? {
13488 if timeout_seconds.is_some_and(|opened_timeout| opened_timeout != event_timeout) {
13489 return Err(invalid_recorded_history(
13490 "condition_wait_definition_history_mismatch",
13491 sequence,
13492 &format!("{}s", timeout_seconds.unwrap_or_default()),
13493 &format!("{event_timeout}s"),
13494 "condition lifecycle events disagree on the recorded timeout",
13495 ));
13496 }
13497 }
13498 }
13499
13500 let timeout_timer_events = all_events
13501 .iter()
13502 .filter(|event| {
13503 matches!(
13504 event.event_type.as_str(),
13505 "TimerScheduled" | "TimerCancelled" | "TimerFired"
13506 ) && event.payload.get("timer_kind").and_then(Value::as_str)
13507 == Some("condition_timeout")
13508 && event
13509 .payload
13510 .get("condition_wait_id")
13511 .and_then(Value::as_str)
13512 == Some(condition_wait_id.as_str())
13513 })
13514 .collect::<Vec<_>>();
13515 let scheduled = timeout_timer_events
13516 .iter()
13517 .copied()
13518 .filter(|event| event.event_type == "TimerScheduled")
13519 .collect::<Vec<_>>();
13520 let fired = timeout_timer_events
13521 .iter()
13522 .copied()
13523 .filter(|event| event.event_type == "TimerFired")
13524 .collect::<Vec<_>>();
13525 if scheduled.len() > 1 || fired.len() > 1 || (!fired.is_empty() && scheduled.len() != 1) {
13526 return Err(invalid_recorded_history(
13527 "condition_wait_timeout_history_invalid",
13528 sequence,
13529 "one timeout schedule and at most one fire",
13530 &format!("{} schedules and {} fires", scheduled.len(), fired.len()),
13531 "condition timeout history has a missing or duplicate lifecycle event",
13532 ));
13533 }
13534 if let Some(scheduled) = scheduled.first() {
13535 let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
13536 let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
13537 if timeout_seconds.is_some_and(|timeout| timeout != delay_seconds) {
13538 return Err(invalid_recorded_history(
13539 "condition_wait_timeout_delay_mismatch",
13540 sequence,
13541 &format!("{}s", timeout_seconds.unwrap_or_default()),
13542 &format!("{delay_seconds}s"),
13543 "condition timeout timer differs from the wait definition",
13544 ));
13545 }
13546 if let Some(fired) = fired.first() {
13547 let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
13548 let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
13549 if fired_timer_id != timer_id || fired_delay != delay_seconds {
13550 return Err(invalid_recorded_history(
13551 "condition_wait_timeout_identity_mismatch",
13552 sequence,
13553 &format!("{timer_id}:{delay_seconds}s"),
13554 &format!("{fired_timer_id}:{fired_delay}s"),
13555 "condition timeout fire does not match its durable schedule",
13556 ));
13557 }
13558 }
13559 }
13560
13561 let result = terminal.first().map(|event| {
13562 if event.event_type == "ConditionWaitTimedOut" {
13563 ConditionWaitResult::TimedOut
13564 } else {
13565 ConditionWaitResult::Satisfied
13566 }
13567 });
13568 let result = if !fired.is_empty() {
13569 if result == Some(ConditionWaitResult::Satisfied) {
13570 return Err(invalid_recorded_history(
13571 "condition_wait_terminal_conflict",
13572 sequence,
13573 "one satisfied or timed-out outcome",
13574 "satisfied event and fired timeout",
13575 "condition history records conflicting terminal outcomes",
13576 ));
13577 }
13578 Some(ConditionWaitResult::TimedOut)
13579 } else {
13580 result
13581 };
13582
13583 Ok(RecordedCommand::ConditionWait {
13584 sequence,
13585 occurrence_id,
13586 condition_key,
13587 predicate_identity,
13588 timeout_seconds,
13589 result,
13590 parallel_group_path: recorded_parallel_group_path(condition_events, sequence)?,
13591 })
13592}
13593
13594fn required_condition_wait_occurrence_id(event: &HistoryEvent, sequence: u64) -> Result<String> {
13595 event
13596 .payload
13597 .get("condition_wait_occurrence_id")
13598 .and_then(Value::as_str)
13599 .filter(|value| !value.is_empty())
13600 .map(str::to_string)
13601 .ok_or_else(|| {
13602 invalid_recorded_history(
13603 "condition_wait_occurrence_id_missing",
13604 sequence,
13605 "non-empty condition_wait_occurrence_id",
13606 &event.event_type,
13607 "condition history is missing authored occurrence identity",
13608 )
13609 })
13610}
13611
13612fn required_condition_wait_id(event: &HistoryEvent, sequence: u64) -> Result<String> {
13613 event
13614 .payload
13615 .get("condition_wait_id")
13616 .and_then(Value::as_str)
13617 .filter(|value| !value.is_empty())
13618 .map(str::to_string)
13619 .ok_or_else(|| {
13620 invalid_recorded_history(
13621 "condition_wait_id_missing",
13622 sequence,
13623 "non-empty condition_wait_id",
13624 &event.event_type,
13625 "canonical condition history is missing its durable wait identity",
13626 )
13627 })
13628}
13629
13630fn optional_non_empty_history_string(event: &HistoryEvent, field: &str) -> Option<String> {
13631 event
13632 .payload
13633 .get(field)
13634 .and_then(Value::as_str)
13635 .filter(|value| !value.is_empty())
13636 .map(str::to_string)
13637}
13638
13639fn optional_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<Option<u64>> {
13640 match event.payload.get(field) {
13641 None | Some(Value::Null) => Ok(None),
13642 Some(value) => value_as_u64(value).map(Some).ok_or_else(|| {
13643 invalid_recorded_history(
13644 "condition_wait_definition_invalid",
13645 sequence,
13646 &format!("non-negative integer {field}"),
13647 &value.to_string(),
13648 "condition history contains an invalid numeric definition field",
13649 )
13650 }),
13651 }
13652}
13653
13654fn required_signal_wait_name(event: &HistoryEvent, sequence: u64) -> Result<String> {
13655 event
13656 .payload
13657 .get("signal_name")
13658 .or_else(|| event.raw.get("signal_name"))
13659 .and_then(Value::as_str)
13660 .filter(|value| !value.is_empty())
13661 .map(str::to_string)
13662 .ok_or_else(|| {
13663 invalid_recorded_history(
13664 "signal_wait_name_missing",
13665 sequence,
13666 "non-empty signal_name",
13667 &event.event_type,
13668 "canonical signal-wait history is missing its signal identity",
13669 )
13670 })
13671}
13672
13673fn is_recorded_signal_wait_event(event: &HistoryEvent) -> bool {
13674 matches!(
13675 event.event_type.as_str(),
13676 "SignalWaitOpened" | "SignalApplied"
13677 )
13678}
13679
13680fn required_history_string(event: &HistoryEvent, field: &str, sequence: u64) -> Result<String> {
13681 event
13682 .payload
13683 .get(field)
13684 .and_then(Value::as_str)
13685 .filter(|value| !value.is_empty())
13686 .map(str::to_string)
13687 .ok_or_else(|| {
13688 invalid_recorded_history(
13689 "timer_history_field_missing",
13690 sequence,
13691 field,
13692 &event.event_type,
13693 "timer history is missing a required identity field",
13694 )
13695 })
13696}
13697
13698fn required_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<u64> {
13699 event
13700 .payload
13701 .get(field)
13702 .and_then(value_as_u64)
13703 .ok_or_else(|| {
13704 invalid_recorded_history(
13705 "timer_history_field_missing",
13706 sequence,
13707 field,
13708 &event.event_type,
13709 "timer history is missing a required numeric field",
13710 )
13711 })
13712}
13713
13714fn recorded_search_attribute_types(
13715 payload: &Value,
13716 attributes: &Value,
13717 sequence: u64,
13718) -> Result<RecordedSnapshotValue<BTreeMap<String, String>>> {
13719 let Some(raw_types) = payload.get("attribute_types") else {
13720 return Ok(RecordedSnapshotValue::Unknown);
13724 };
13725 let Some(raw_types) = raw_types.as_object() else {
13726 return Err(invalid_recorded_history(
13727 "search_attribute_types_malformed",
13728 sequence,
13729 "canonical attribute type map",
13730 &raw_types.to_string(),
13731 "search-attribute history contains malformed type identity",
13732 ));
13733 };
13734 let attribute_keys = attributes
13735 .as_object()
13736 .expect("recorded search attributes were validated as an object");
13737 let mut types = BTreeMap::new();
13738 for (key, value) in raw_types {
13739 let Some(attribute_type) = value.as_str() else {
13740 return Err(invalid_recorded_history(
13741 "search_attribute_types_malformed",
13742 sequence,
13743 "canonical string type name",
13744 &value.to_string(),
13745 "search-attribute history contains a non-string type identity",
13746 ));
13747 };
13748 if !attribute_keys.contains_key(key)
13749 || !matches!(
13750 attribute_type,
13751 "string" | "keyword" | "keyword_list" | "int" | "float" | "bool" | "datetime"
13752 )
13753 {
13754 return Err(invalid_recorded_history(
13755 "search_attribute_types_malformed",
13756 sequence,
13757 "canonical types for keys present in attributes",
13758 &format!("{key}:{attribute_type}"),
13759 "search-attribute history contains unsupported or orphaned type identity",
13760 ));
13761 }
13762 types.insert(key.clone(), attribute_type.to_string());
13763 }
13764 Ok(RecordedSnapshotValue::Known(types))
13765}
13766
13767fn invalid_recorded_history(
13768 reason: &str,
13769 sequence: u64,
13770 expected: &str,
13771 actual: &str,
13772 message: &str,
13773) -> Error {
13774 Error::NonDeterministicReplay(ReplayFailure::new(
13775 reason,
13776 Some(sequence),
13777 Some(expected.to_string()),
13778 Some(actual.to_string()),
13779 message,
13780 ))
13781}
13782
13783type ActivityOutcome = std::result::Result<AvroValue, ActivityFailure>;
13784
13785fn activity_outcome(
13786 event: &HistoryEvent,
13787 fallback_codec: &str,
13788 recorded_activity_type: Option<String>,
13789) -> Result<ActivityOutcome> {
13790 if event.event_type == "ActivityCompleted" {
13791 let codec = event
13792 .payload
13793 .get("payload_codec")
13794 .and_then(Value::as_str)
13795 .unwrap_or(fallback_codec);
13796 return Ok(Ok(decode_wire_avro_value(
13797 event.payload.get("result").unwrap_or(&Value::Null),
13798 codec,
13799 )?));
13800 }
13801
13802 let payload = &event.payload;
13803 let (kind, fallback_reason, fallback_message) = match event.event_type.as_str() {
13804 "ActivityFailed" => (ActivityFailureKind::Failed, "activity", "activity failed"),
13805 "ActivityCancelled" => (
13806 ActivityFailureKind::Cancelled,
13807 "cancelled",
13808 "activity was cancelled",
13809 ),
13810 "ActivityTimedOut" => (
13811 ActivityFailureKind::TimedOut,
13812 "timeout",
13813 "activity timed out",
13814 ),
13815 _ => unreachable!("activity_outcome is called only for terminal activity events"),
13816 };
13817 let exception = payload
13818 .get("exception")
13819 .filter(|value| !value.is_null())
13820 .cloned();
13821 let failure_category = payload_string(payload, "failure_category");
13822 let timeout_kind = payload_string(payload, "timeout_kind");
13823 let reason = payload_string(payload, "reason").unwrap_or_else(|| match kind {
13824 ActivityFailureKind::Failed => failure_category
13825 .clone()
13826 .unwrap_or_else(|| fallback_reason.to_string()),
13827 ActivityFailureKind::Cancelled => fallback_reason.to_string(),
13828 ActivityFailureKind::TimedOut => timeout_kind
13829 .clone()
13830 .unwrap_or_else(|| fallback_reason.to_string()),
13831 });
13832 let message = payload_string(payload, "message")
13833 .or_else(|| {
13834 exception
13835 .as_ref()
13836 .and_then(|value| payload_string(value, "message"))
13837 })
13838 .unwrap_or_else(|| fallback_message.to_string());
13839
13840 Ok(Err(ActivityFailure {
13841 kind,
13842 reason,
13843 message,
13844 activity_execution_id: payload_string(payload, "activity_execution_id"),
13845 activity_attempt_id: payload_string(payload, "activity_attempt_id"),
13846 activity_type: payload_string(payload, "activity_type")
13847 .or_else(|| payload_string(payload, "activity_name"))
13848 .or(recorded_activity_type),
13849 activity_class: payload_string(payload, "activity_class"),
13850 attempt_number: payload.get("attempt_number").and_then(value_as_u64),
13851 failure_id: payload_string(payload, "failure_id"),
13852 failure_category,
13853 timeout_kind,
13854 non_retryable: payload
13855 .get("non_retryable")
13856 .and_then(Value::as_bool)
13857 .unwrap_or(false),
13858 exception_type: payload_string(payload, "exception_type").or_else(|| {
13859 exception
13860 .as_ref()
13861 .and_then(|value| payload_string(value, "type"))
13862 }),
13863 exception_class: payload_string(payload, "exception_class").or_else(|| {
13864 exception
13865 .as_ref()
13866 .and_then(|value| payload_string(value, "class"))
13867 }),
13868 code: payload
13869 .get("code")
13870 .filter(|value| !value.is_null())
13871 .cloned(),
13872 exception,
13873 }))
13874}
13875
13876type ChildWorkflowOutcome = std::result::Result<ChildWorkflowAvroResult, ChildWorkflowFailure>;
13877
13878fn child_workflow_outcomes(
13879 events: &[HistoryEvent],
13880 fallback_codec: &str,
13881 parent: WorkflowIdentity,
13882) -> Result<Vec<ChildWorkflowOutcome>> {
13883 let mut outcomes = Vec::new();
13884
13885 for event in events {
13886 let kind = match event.event_type.as_str() {
13887 "ChildRunCompleted" => None,
13888 "ChildRunFailed" => Some((
13889 ChildWorkflowFailureKind::Failed,
13890 "child_workflow",
13891 "child workflow failed",
13892 )),
13893 "ChildRunCancelled" => Some((
13894 ChildWorkflowFailureKind::Cancelled,
13895 "cancelled",
13896 "child workflow was cancelled",
13897 )),
13898 "ChildRunTerminated" => Some((
13899 ChildWorkflowFailureKind::Terminated,
13900 "terminated",
13901 "child workflow was terminated",
13902 )),
13903 _ => continue,
13904 };
13905 let payload = &event.payload;
13906 let child_workflow_id = payload_string(payload, "child_workflow_instance_id");
13907 let child_workflow_run_id = payload_string(payload, "child_workflow_run_id");
13908 let child_workflow_type = payload_string(payload, "child_workflow_type");
13909
13910 if let Some((kind, reason, fallback_message)) = kind {
13911 let exception = payload
13912 .get("exception")
13913 .filter(|value| !value.is_null())
13914 .cloned();
13915 let message = payload_string(payload, "message")
13916 .or_else(|| {
13917 exception
13918 .as_ref()
13919 .and_then(|value| payload_string(value, "message"))
13920 })
13921 .unwrap_or_else(|| fallback_message.to_string());
13922 let exception_type = payload_string(payload, "exception_type").or_else(|| {
13923 exception
13924 .as_ref()
13925 .and_then(|value| payload_string(value, "type"))
13926 });
13927 let exception_class = payload_string(payload, "exception_class").or_else(|| {
13928 exception
13929 .as_ref()
13930 .and_then(|value| payload_string(value, "class"))
13931 });
13932 outcomes.push(Err(ChildWorkflowFailure {
13933 kind,
13934 reason: reason.to_string(),
13935 message,
13936 parent_workflow_id: parent.workflow_id.clone(),
13937 parent_workflow_run_id: parent.run_id.clone(),
13938 child_workflow_id,
13939 child_workflow_run_id,
13940 child_workflow_type,
13941 failure_id: payload_string(payload, "failure_id"),
13942 failure_category: payload_string(payload, "failure_category"),
13943 exception_type,
13944 exception_class,
13945 non_retryable: payload
13946 .get("non_retryable")
13947 .and_then(Value::as_bool)
13948 .unwrap_or(false),
13949 code: payload
13950 .get("code")
13951 .filter(|value| !value.is_null())
13952 .cloned(),
13953 exception,
13954 }));
13955 continue;
13956 }
13957
13958 let codec = payload
13959 .get("payload_codec")
13960 .and_then(Value::as_str)
13961 .unwrap_or(fallback_codec);
13962 let result = payload
13963 .get("result")
13964 .or_else(|| payload.get("output"))
13965 .unwrap_or(&Value::Null);
13966 outcomes.push(Ok(ChildWorkflowAvroResult {
13967 parent: parent.clone(),
13968 child: WorkflowIdentity {
13969 workflow_id: child_workflow_id,
13970 run_id: child_workflow_run_id,
13971 },
13972 child_workflow_type,
13973 result: decode_wire_avro_value(result, codec)?,
13974 }));
13975 }
13976
13977 Ok(outcomes)
13978}
13979
13980fn payload_string(payload: &Value, key: &str) -> Option<String> {
13981 payload
13982 .get(key)
13983 .and_then(Value::as_str)
13984 .filter(|value| !value.is_empty())
13985 .map(str::to_string)
13986}
13987
13988fn workflow_failure_command(error: &Error) -> Value {
13989 let (exception_type, exception_class, properties) = match error {
13990 Error::ActivityFailed(failure) => (
13991 match failure.kind {
13992 ActivityFailureKind::Failed => "ActivityFailed",
13993 ActivityFailureKind::Cancelled => "ActivityCancelled",
13994 ActivityFailureKind::TimedOut => "ActivityTimedOut",
13995 },
13996 "durable_workflow::ActivityFailure",
13997 json!({
13998 "reason": failure.reason,
13999 "activity_execution_id": failure.activity_execution_id,
14000 "activity_attempt_id": failure.activity_attempt_id,
14001 "activity_type": failure.activity_type,
14002 "activity_class": failure.activity_class,
14003 "attempt_number": failure.attempt_number,
14004 "failure_id": failure.failure_id,
14005 "failure_category": failure.failure_category,
14006 "timeout_kind": failure.timeout_kind,
14007 "activity_non_retryable": failure.non_retryable,
14008 "activity_exception_type": failure.exception_type,
14009 "activity_exception_class": failure.exception_class,
14010 "activity_code": failure.code,
14011 "activity_exception": failure.exception,
14012 }),
14013 ),
14014 Error::ChildWorkflowFailed(failure) => (
14015 match failure.kind {
14016 ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
14017 ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
14018 ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
14019 },
14020 "durable_workflow::ChildWorkflowFailure",
14021 json!({
14022 "reason": failure.reason,
14023 "parent_workflow_id": failure.parent_workflow_id,
14024 "parent_workflow_run_id": failure.parent_workflow_run_id,
14025 "child_workflow_id": failure.child_workflow_id,
14026 "child_workflow_run_id": failure.child_workflow_run_id,
14027 "child_workflow_type": failure.child_workflow_type,
14028 "failure_id": failure.failure_id,
14029 "failure_category": failure.failure_category,
14030 "child_exception_type": failure.exception_type,
14031 "child_exception_class": failure.exception_class,
14032 "child_non_retryable": failure.non_retryable,
14033 "child_code": failure.code,
14034 "child_exception": failure.exception,
14035 }),
14036 ),
14037 Error::ParallelFailed(failure) => (
14038 "ParallelFailed",
14039 "durable_workflow::ParallelFailure",
14040 json!({
14041 "parallel_group_id": failure.group_id,
14042 "parallel_member_path": failure.member_path,
14043 "parallel_group_path": failure.group_path,
14044 "completed_members": failure.completed.iter().map(|completion| &completion.member_path).collect::<Vec<_>>(),
14045 "cause_type": workflow_error_type(&failure.cause),
14046 "cause_message": failure.cause.to_string(),
14047 }),
14048 ),
14049 Error::SagaCompensationFailed(failure) => (
14050 "SagaCompensationFailed",
14051 "durable_workflow::SagaCompensationFailure",
14052 json!({
14053 "initiating_failure_type": workflow_error_type(&failure.initiating_failure),
14054 "initiating_failure_message": failure.initiating_failure.to_string(),
14055 "compensation_activity_type": failure.compensation_activity_type,
14056 "compensation_registration_order": failure.compensation_registration_order,
14057 "compensation_failure_type": workflow_error_type(&failure.compensation_failure),
14058 "compensation_failure_message": failure.compensation_failure.to_string(),
14059 }),
14060 ),
14061 Error::WorkflowCancellationRequested(_) => (
14062 "WorkflowCancellationRequested",
14063 "durable_workflow::WorkflowCancellationRequested",
14064 json!({"reason": "cancelled"}),
14065 ),
14066 Error::NonDeterministicReplay(_) => (
14067 "NonDeterministicReplay",
14068 "durable_workflow::Error",
14069 Value::Null,
14070 ),
14071 _ => ("RustWorkflowError", "durable_workflow::Error", Value::Null),
14072 };
14073 let non_retryable = match error {
14074 Error::ActivityFailed(failure) => failure.non_retryable,
14075 Error::ChildWorkflowFailed(failure) => failure.non_retryable,
14076 Error::ParallelFailed(failure) => workflow_error_non_retryable(&failure.cause),
14077 Error::SagaCompensationFailed(failure) => {
14078 workflow_error_non_retryable(&failure.compensation_failure)
14079 }
14080 Error::WorkflowCancellationRequested(_) => true,
14081 Error::NonDeterministicReplay(_) => true,
14082 _ => false,
14083 };
14084
14085 json!({
14086 "type": "fail_workflow",
14087 "message": error.to_string(),
14088 "exception_type": exception_type,
14089 "exception_class": exception_class,
14090 "non_retryable": non_retryable,
14091 "exception": {
14092 "type": exception_type,
14093 "class": exception_class,
14094 "message": error.to_string(),
14095 "properties": properties,
14096 }
14097 })
14098}
14099
14100fn workflow_error_type(error: &Error) -> &'static str {
14101 match error {
14102 Error::ActivityFailed(failure) => match failure.kind {
14103 ActivityFailureKind::Failed => "ActivityFailed",
14104 ActivityFailureKind::Cancelled => "ActivityCancelled",
14105 ActivityFailureKind::TimedOut => "ActivityTimedOut",
14106 },
14107 Error::ChildWorkflowFailed(failure) => match failure.kind {
14108 ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
14109 ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
14110 ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
14111 },
14112 Error::ParallelFailed(_) => "ParallelFailed",
14113 Error::SagaCompensationFailed(_) => "SagaCompensationFailed",
14114 Error::WorkflowCancellationRequested(_) => "WorkflowCancellationRequested",
14115 Error::NonDeterministicReplay(_) => "NonDeterministicReplay",
14116 _ => "RustWorkflowError",
14117 }
14118}
14119
14120fn workflow_error_non_retryable(error: &Error) -> bool {
14121 match error {
14122 Error::ActivityFailed(failure) => failure.non_retryable,
14123 Error::ChildWorkflowFailed(failure) => failure.non_retryable,
14124 Error::ParallelFailed(failure) => workflow_error_non_retryable(&failure.cause),
14125 Error::SagaCompensationFailed(failure) => {
14126 workflow_error_non_retryable(&failure.compensation_failure)
14127 }
14128 Error::WorkflowCancellationRequested(_) | Error::NonDeterministicReplay(_) => true,
14129 _ => false,
14130 }
14131}
14132
14133fn workflow_task_integrity_error(error: &Error) -> bool {
14134 matches!(
14135 error,
14136 Error::NonDeterministicReplay(_)
14137 | Error::Protocol(_)
14138 | Error::MissingWorkflowCommandIdentity
14139 | Error::WorkflowStatePoisoned
14140 )
14141}
14142
14143fn decode_signal_event_arguments(
14144 event: &HistoryEvent,
14145 fallback_codec: &str,
14146) -> Result<Vec<AvroValue>> {
14147 let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
14148 validate_payload_codec(codec)?;
14149 let raw = signal_history_payload(&event.payload);
14150 let decoded = match raw.filter(|value| !value.is_null()) {
14151 Some(value) => decode_wire_avro_value(value, codec)?,
14152 None => AvroValue::Array(Vec::new()),
14153 };
14154 let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
14155 unreachable!("normalize_avro_arguments always returns an array");
14156 };
14157 Ok(arguments)
14158}
14159
14160fn decode_update_event_arguments(
14161 event: &HistoryEvent,
14162 fallback_codec: &str,
14163) -> Result<Vec<AvroValue>> {
14164 let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
14165 validate_payload_codec(codec)?;
14166 let decoded = match event
14167 .payload
14168 .get("arguments")
14169 .filter(|value| !value.is_null())
14170 {
14171 Some(value) => decode_wire_avro_value(value, codec)?,
14172 None => AvroValue::Array(Vec::new()),
14173 };
14174 let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
14175 unreachable!("normalize_avro_arguments always returns an array");
14176 };
14177 Ok(arguments)
14178}
14179
14180fn hydrate_query_history_from_export(task: &mut QueryTask) -> Result<()> {
14181 let Some(export_events) = task
14182 .history_export
14183 .as_ref()
14184 .and_then(|export| export.get("history_events"))
14185 .and_then(Value::as_array)
14186 else {
14187 return Ok(());
14188 };
14189
14190 if export_events.len() > task.history_events.len() {
14191 task.history_events = serde_json::from_value(Value::Array(export_events.clone()))?;
14192 }
14193
14194 Ok(())
14195}
14196
14197fn enrich_query_history_from_export(task: &mut QueryTask) -> Result<()> {
14198 let Some(export) = task.history_export.as_ref() else {
14199 return Ok(());
14200 };
14201 let signals = export
14202 .get("signals")
14203 .and_then(Value::as_array)
14204 .cloned()
14205 .unwrap_or_default();
14206 let activities = export
14207 .get("activities")
14208 .and_then(Value::as_array)
14209 .cloned()
14210 .unwrap_or_default();
14211 let export_codec = export
14212 .get("payloads")
14213 .and_then(|payloads| payloads.get("codec"))
14214 .and_then(Value::as_str)
14215 .unwrap_or(&task.payload_codec)
14216 .to_string();
14217 let mut signal_name_offsets: HashMap<String, usize> = HashMap::new();
14218
14219 for event in &mut task.history_events {
14220 if event.event_type == "ActivityCompleted" {
14221 let sequence = event
14222 .payload
14223 .get("sequence")
14224 .or_else(|| event.payload.get("workflow_sequence"))
14225 .and_then(value_as_u64);
14226 let Some(activity) = sequence.and_then(|sequence| {
14227 activities.iter().find(|activity| {
14228 activity.get("sequence").and_then(value_as_u64) == Some(sequence)
14229 })
14230 }) else {
14231 continue;
14232 };
14233 let Some(payload) = event.payload.as_object_mut() else {
14234 continue;
14235 };
14236 if missing_payload(payload.get("result")) {
14237 if let Some(result) = activity
14238 .get("result")
14239 .filter(|value| !missing_payload(Some(value)))
14240 {
14241 payload.insert("result".to_string(), result.clone());
14242 }
14243 }
14244 for field in ["payload_codec", "activity_type"] {
14245 if payload
14246 .get(field)
14247 .and_then(Value::as_str)
14248 .unwrap_or_default()
14249 .is_empty()
14250 {
14251 if let Some(value) = activity.get(field) {
14252 payload.insert(field.to_string(), value.clone());
14253 }
14254 }
14255 }
14256 continue;
14257 }
14258
14259 if event.event_type != "SignalReceived" && event.event_type != "SignalApplied" {
14260 continue;
14261 }
14262 let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
14263 let command_id = event
14264 .payload
14265 .get("workflow_command_id")
14266 .or_else(|| event.raw.get("workflow_command_id"))
14267 .and_then(Value::as_str);
14268 let signal_name = event
14269 .payload
14270 .get("signal_name")
14271 .and_then(Value::as_str)
14272 .unwrap_or_default()
14273 .to_string();
14274 let matched = signals
14275 .iter()
14276 .find(|signal| {
14277 signal_id.is_some() && signal.get("id").and_then(Value::as_str) == signal_id
14278 })
14279 .or_else(|| {
14280 signals.iter().find(|signal| {
14281 command_id.is_some()
14282 && signal.get("command_id").and_then(Value::as_str) == command_id
14283 })
14284 })
14285 .or_else(|| {
14286 let offset = signal_name_offsets.entry(signal_name.clone()).or_default();
14287 let signal = signals
14288 .iter()
14289 .filter(|signal| {
14290 signal.get("name").and_then(Value::as_str) == Some(signal_name.as_str())
14291 })
14292 .nth(*offset);
14293 if signal.is_some() {
14294 *offset += 1;
14295 }
14296 signal
14297 });
14298 let Some(signal) = matched else {
14299 continue;
14300 };
14301 let signal_codec = signal
14302 .get("payload_codec")
14303 .and_then(Value::as_str)
14304 .unwrap_or(&export_codec);
14305 let Some(payload) = event.payload.as_object_mut() else {
14306 continue;
14307 };
14308 if missing_payload(payload.get("arguments")) {
14309 if let Some(arguments) = signal
14310 .get("arguments")
14311 .filter(|value| !missing_payload(Some(value)))
14312 {
14313 let envelope = match arguments {
14314 Value::String(blob) => json!({"codec": signal_codec, "blob": blob}),
14315 other => other.clone(),
14316 };
14317 payload.insert("arguments".to_string(), envelope);
14318 }
14319 }
14320 if payload
14321 .get("payload_codec")
14322 .and_then(Value::as_str)
14323 .unwrap_or_default()
14324 .is_empty()
14325 {
14326 payload.insert("payload_codec".to_string(), json!(signal_codec));
14327 }
14328 }
14329
14330 Ok(())
14331}
14332
14333fn missing_payload(value: Option<&Value>) -> bool {
14334 match value {
14335 None | Some(Value::Null) => true,
14336 Some(Value::String(value)) => value.is_empty(),
14337 Some(_) => false,
14338 }
14339}
14340
14341fn query_signal_events(task: &QueryTask) -> Result<Vec<QuerySignal>> {
14342 let export_signals = task
14343 .history_export
14344 .as_ref()
14345 .and_then(|export| export.get("signals"))
14346 .and_then(Value::as_array)
14347 .cloned()
14348 .unwrap_or_default();
14349 let export_codec = task
14350 .history_export
14351 .as_ref()
14352 .and_then(|export| export.get("payloads"))
14353 .and_then(|payloads| payloads.get("codec"))
14354 .and_then(Value::as_str)
14355 .unwrap_or(&task.payload_codec);
14356 let mut name_offsets: HashMap<String, usize> = HashMap::new();
14357 let mut signals = Vec::new();
14358
14359 for event in &task.history_events {
14360 if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
14361 continue;
14362 }
14363
14364 let name = event
14365 .payload
14366 .get("signal_name")
14367 .and_then(Value::as_str)
14368 .unwrap_or_default();
14369 if name.is_empty() {
14370 continue;
14371 }
14372 let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
14373 let command_id = event
14374 .payload
14375 .get("workflow_command_id")
14376 .or_else(|| event.raw.get("workflow_command_id"))
14377 .and_then(Value::as_str);
14378 let matched_export = export_signals
14379 .iter()
14380 .find(|candidate| {
14381 signal_id.is_some() && candidate.get("id").and_then(Value::as_str) == signal_id
14382 })
14383 .or_else(|| {
14384 export_signals.iter().find(|candidate| {
14385 command_id.is_some()
14386 && candidate.get("command_id").and_then(Value::as_str) == command_id
14387 })
14388 })
14389 .or_else(|| {
14390 let offset = name_offsets.entry(name.to_string()).or_default();
14391 let candidate = export_signals
14392 .iter()
14393 .filter(|candidate| candidate.get("name").and_then(Value::as_str) == Some(name))
14394 .nth(*offset);
14395 if candidate.is_some() {
14396 *offset += 1;
14397 }
14398 candidate
14399 });
14400 let codec = event
14401 .payload
14402 .get("payload_codec")
14403 .and_then(Value::as_str)
14404 .or_else(|| {
14405 matched_export
14406 .and_then(|signal| signal.get("payload_codec"))
14407 .and_then(Value::as_str)
14408 })
14409 .unwrap_or(export_codec);
14410 let raw_arguments = signal_history_payload(&event.payload)
14411 .filter(|value| !value.is_null())
14412 .or_else(|| matched_export.and_then(|signal| signal.get("arguments")));
14413 let (arguments, avro_arguments) = decode_query_signal_arguments(raw_arguments, codec)?;
14414 let workflow_sequence = event
14415 .payload
14416 .get("workflow_sequence")
14417 .and_then(value_as_u64)
14418 .or_else(|| {
14419 matched_export
14420 .and_then(|signal| signal.get("workflow_sequence"))
14421 .and_then(value_as_u64)
14422 });
14423
14424 signals.push(QuerySignal {
14425 id: signal_id.map(str::to_string).or_else(|| {
14426 matched_export
14427 .and_then(|signal| signal.get("id"))
14428 .and_then(Value::as_str)
14429 .map(str::to_string)
14430 }),
14431 name: name.to_string(),
14432 arguments,
14433 avro_arguments,
14434 workflow_sequence,
14435 });
14436 }
14437
14438 if signals.is_empty() {
14439 for signal in export_signals {
14440 if signal.get("status").and_then(Value::as_str) == Some("rejected") {
14441 continue;
14442 }
14443 let Some(name) = signal.get("name").and_then(Value::as_str) else {
14444 continue;
14445 };
14446 let codec = signal
14447 .get("payload_codec")
14448 .and_then(Value::as_str)
14449 .unwrap_or(export_codec);
14450 let (arguments, avro_arguments) =
14451 decode_query_signal_arguments(signal.get("arguments"), codec)?;
14452 signals.push(QuerySignal {
14453 id: signal.get("id").and_then(Value::as_str).map(str::to_string),
14454 name: name.to_string(),
14455 arguments,
14456 avro_arguments,
14457 workflow_sequence: signal.get("workflow_sequence").and_then(value_as_u64),
14458 });
14459 }
14460 signals.sort_by_key(|signal| signal.workflow_sequence.unwrap_or(u64::MAX));
14461 }
14462
14463 Ok(signals)
14464}
14465
14466fn decode_query_signal_arguments(
14467 raw: Option<&Value>,
14468 codec: &str,
14469) -> Result<(Vec<Value>, Vec<AvroValue>)> {
14470 validate_payload_codec(codec)?;
14471 let decoded = match raw.filter(|value| !value.is_null()) {
14472 Some(value) => decode_wire_avro_value(value, codec)?,
14473 None => AvroValue::Array(Vec::new()),
14474 };
14475 let AvroValue::Array(avro_arguments) = normalize_avro_arguments(decoded) else {
14476 unreachable!("normalize_avro_arguments always returns an array");
14477 };
14478 let arguments = avro_arguments
14479 .iter()
14480 .cloned()
14481 .map(AvroValue::into_json)
14482 .collect::<Result<Vec<_>>>()?;
14483 Ok((arguments, avro_arguments))
14484}
14485
14486fn value_as_u64(value: &Value) -> Option<u64> {
14487 value
14488 .as_u64()
14489 .or_else(|| value.as_str().and_then(|value| value.parse().ok()))
14490}
14491
14492#[cfg(test)]
14493mod tests {
14494 use super::*;
14495 use std::{
14496 fs,
14497 io::{Read, Write},
14498 net::{SocketAddr, TcpListener, TcpStream},
14499 process::Command as ProcessCommand,
14500 sync::atomic::AtomicUsize,
14501 thread,
14502 };
14503
14504 #[derive(Clone, Copy, Debug)]
14505 enum InvalidTaskPayloadCodec {
14506 Missing,
14507 Null,
14508 NonString,
14509 }
14510
14511 impl InvalidTaskPayloadCodec {
14512 fn label(self) -> &'static str {
14513 match self {
14514 Self::Missing => "missing",
14515 Self::Null => "null",
14516 Self::NonString => "non-string",
14517 }
14518 }
14519
14520 fn apply(self, task: &mut Value) {
14521 let task = task.as_object_mut().expect("task fixture object");
14522 match self {
14523 Self::Missing => {
14524 task.remove("payload_codec");
14525 }
14526 Self::Null => {
14527 task.insert("payload_codec".to_string(), Value::Null);
14528 }
14529 Self::NonString => {
14530 task.insert("payload_codec".to_string(), json!(42));
14531 }
14532 }
14533 }
14534 }
14535
14536 fn fixture_envelope(value: Value) -> Value {
14537 encode_value_envelope(&value, DEFAULT_CODEC).expect("encode Avro test fixture")
14538 }
14539
14540 fn fixture_blob(value: Value) -> String {
14541 encode_payload(&value, DEFAULT_CODEC)
14542 .expect("encode Avro test fixture")
14543 .blob
14544 }
14545
14546 #[test]
14547 fn client_builder_rejects_the_sdk_owned_api_suffix() {
14548 for base_url in [
14549 "http://127.0.0.1:8080/api",
14550 "http://localhost:8080/api/",
14551 "https://runtime.example.test/namespaces/orders/api",
14552 ] {
14553 let error = Client::builder(base_url)
14554 .build()
14555 .expect_err("SDK-owned /api suffix must be rejected during build");
14556
14557 assert!(matches!(error, Error::InvalidBaseUrl), "{base_url}");
14558 assert!(
14559 error.to_string().contains("SDK appends /api automatically"),
14560 "the validation error must explain how to fix the endpoint"
14561 );
14562 }
14563 }
14564
14565 #[test]
14566 fn client_builder_preserves_self_hosted_and_managed_runtime_prefixes() {
14567 for (base_url, expected) in [
14568 ("http://127.0.0.1:8080", "http://127.0.0.1:8080"),
14569 (
14570 "http://localhost:8080/durable-workflow/",
14571 "http://localhost:8080/durable-workflow",
14572 ),
14573 (
14574 "https://runtime.example.test/namespaces/orders",
14575 "https://runtime.example.test/namespaces/orders",
14576 ),
14577 (
14578 "https://runtime.example.test/gateway/api/namespaces/orders",
14579 "https://runtime.example.test/gateway/api/namespaces/orders",
14580 ),
14581 (
14582 "https://api.example.test/runtime/orders/",
14583 "https://api.example.test/runtime/orders",
14584 ),
14585 ] {
14586 let client = Client::builder(base_url)
14587 .build()
14588 .expect("Server and Cloud runtime base URL must remain valid");
14589
14590 assert_eq!(client.base_url, expected);
14591 }
14592 }
14593
14594 #[test]
14595 fn workflow_completion_uses_the_additive_command_protocol_floor() {
14596 assert_eq!(
14597 workflow_completion_protocol_version(&[json!({"type": "complete_workflow"})]),
14598 WORKER_PROTOCOL_VERSION
14599 );
14600 assert_eq!(
14601 workflow_completion_protocol_version(&[json!({
14602 "type": "upsert_search_attributes",
14603 "attributes": {"OrderStatus": "waiting"},
14604 })]),
14605 SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION
14606 );
14607 assert_eq!(
14608 workflow_completion_protocol_version(&[json!({
14609 "type": "upsert_search_attributes",
14610 "attributes": {"OrderStatus": "waiting"},
14611 "attribute_types": {"OrderStatus": "keyword"},
14612 })]),
14613 TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION
14614 );
14615 assert_eq!(
14616 workflow_completion_protocol_version(&[
14617 json!({"type": "upsert_memo", "entries": {"status": "waiting"}}),
14618 json!({"type": "open_condition_wait", "condition_key": "ready"}),
14619 ]),
14620 MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION
14621 );
14622 assert_eq!(
14623 workflow_completion_protocol_version(&[
14624 json!({"type": "upsert_search_attributes", "attributes": {"State": "waiting"}}),
14625 json!({"type": "open_condition_wait", "condition_key": "ready"}),
14626 ]),
14627 CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION
14628 );
14629 assert_eq!(
14630 workflow_completion_protocol_version(&[json!({
14631 "type": "open_condition_wait",
14632 "condition_wait_occurrence_id": "rust:condition-wait:0",
14633 "condition_key": "ready",
14634 })]),
14635 CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
14636 );
14637 assert_eq!(
14638 workflow_completion_protocol_version_with_message_streams(
14639 &[json!({"type": "upsert_memo", "entries": {"status": "waiting"}})],
14640 true,
14641 ),
14642 MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION
14643 );
14644 assert_eq!(
14645 workflow_completion_protocol_version_with_message_streams(
14646 &[json!({
14647 "type": "open_condition_wait",
14648 "condition_wait_occurrence_id": "rust:condition-wait:0",
14649 "condition_key": "ready",
14650 })],
14651 true,
14652 ),
14653 CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
14654 );
14655 }
14656
14657 #[test]
14658 fn portable_worker_affinity_manifest_explicitly_refuses_unimplemented_features() {
14659 let manifest = portable_worker_affinity_capability_manifest();
14660
14661 for capability in ["local_activities", "worker_sessions", "sticky_execution"] {
14662 assert_eq!(manifest[capability]["supported"], json!(false));
14663 assert_eq!(
14664 manifest[capability]["minimum_protocol_version"],
14665 json!(PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION)
14666 );
14667 assert!(manifest[capability]["reason"]
14668 .as_str()
14669 .is_some_and(|reason| !reason.is_empty()));
14670 }
14671 }
14672
14673 fn typed_fidelity_probe() -> AvroValue {
14674 AvroValue::Map(BTreeMap::from([
14675 ("bytes".to_string(), AvroValue::Bytes(vec![0, 0xff])),
14676 ("empty".to_string(), AvroValue::Map(BTreeMap::new())),
14677 (
14678 "numeric".to_string(),
14679 AvroValue::Map(BTreeMap::from([
14680 ("0".to_string(), AvroValue::String("zero".to_string())),
14681 ("1".to_string(), AvroValue::String("one".to_string())),
14682 ])),
14683 ),
14684 (
14685 "nested".to_string(),
14686 AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([(
14687 "enabled".to_string(),
14688 AvroValue::Boolean(true),
14689 )]))]),
14690 ),
14691 (
14692 "projection_collisions".to_string(),
14693 AvroValue::Array(projection_collision_probe()),
14694 ),
14695 ]))
14696 }
14697
14698 fn projection_collision_probe() -> Vec<AvroValue> {
14699 vec![
14700 AvroValue::Map(BTreeMap::from([
14701 ("$type".to_string(), AvroValue::String("bytes".to_string())),
14702 (
14703 "base64".to_string(),
14704 AvroValue::String("ordinary user text".to_string()),
14705 ),
14706 ])),
14707 AvroValue::Map(BTreeMap::from([
14708 ("$type".to_string(), AvroValue::String("map".to_string())),
14709 (
14710 "entries".to_string(),
14711 AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([
14712 ("key".to_string(), AvroValue::String("ordinary".to_string())),
14713 (
14714 "value".to_string(),
14715 AvroValue::String("user map".to_string()),
14716 ),
14717 ]))]),
14718 ),
14719 ])),
14720 ]
14721 }
14722
14723 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
14724 struct TypedContract {
14725 nested: TypedNested,
14726 mode: TypedMode,
14727 optional: Option<String>,
14728 absent: Option<String>,
14729 items: Vec<i64>,
14730 labels: BTreeMap<String, String>,
14731 bytes: serde_bytes::ByteBuf,
14732 signed: i64,
14733 finite: f64,
14734 }
14735
14736 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
14737 struct TypedNested {
14738 enabled: bool,
14739 }
14740
14741 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
14742 enum TypedMode {
14743 Detailed { label: String },
14744 }
14745
14746 fn typed_contract() -> TypedContract {
14747 TypedContract {
14748 nested: TypedNested { enabled: true },
14749 mode: TypedMode::Detailed {
14750 label: "compiler-checked".to_string(),
14751 },
14752 optional: Some("present".to_string()),
14753 absent: None,
14754 items: vec![i64::MIN, 0, i64::MAX],
14755 labels: BTreeMap::from([
14756 ("language".to_string(), "rust".to_string()),
14757 ("wire".to_string(), "avro".to_string()),
14758 ]),
14759 bytes: serde_bytes::ByteBuf::from(vec![0, 0xff, 7]),
14760 signed: -9_223_372_036_854_775_000,
14761 finite: 12.5,
14762 }
14763 }
14764
14765 #[derive(Clone, Debug, Default, PartialEq)]
14766 struct ReplayCounterState {
14767 loaded: Option<String>,
14768 count: i64,
14769 finished: bool,
14770 }
14771
14772 fn replay_counter_worker() -> Worker {
14773 let client = Client::new("http://127.0.0.1:8080").expect("client");
14774 let mut worker = Worker::new(client, "rust-workers");
14775 worker.register_replayed_workflow(
14776 "replay-counter",
14777 ReplayCounterState::default,
14778 |ctx, _input, state| async move {
14779 let loaded = ctx.activity("load-counter", json!([])).await?;
14780 state.update(|current| {
14781 current.loaded = loaded.as_str().map(str::to_string);
14782 })?;
14783 for _ in 0..2 {
14784 let signal = ctx.wait_signal("increment").await?;
14785 let amount = signal.first().and_then(Value::as_i64).unwrap_or_default();
14786 state.update(|current| current.count += amount)?;
14787 }
14788 state.update(|current| current.finished = true)?;
14789 state.read(|current| Ok(json!(current.count)))?
14790 },
14791 );
14792 worker.register_replayed_query::<ReplayCounterState, _, _>(
14793 "replay-counter",
14794 "current",
14795 |_ctx, state, _args| async move {
14796 Ok(json!({
14797 "loaded": state.loaded,
14798 "count": state.count,
14799 "finished": state.finished,
14800 }))
14801 },
14802 );
14803 worker.register_replayed_query::<ReplayCounterState, _, _>(
14804 "replay-counter",
14805 "detached-mutation",
14806 |_ctx, state, _args| async move {
14807 let mut detached = (*state).clone();
14808 detached.count = 999;
14809 Ok(json!(detached.count))
14810 },
14811 );
14812 worker.register_replayed_query::<ReplayCounterState, _, _>(
14813 "replay-counter",
14814 "failed-mutation",
14815 |_ctx, state, _args| async move {
14816 let mut detached = (*state).clone();
14817 detached.count = 999;
14818 Err(Error::WorkerLoop("query refused".to_string()))
14819 },
14820 );
14821 worker
14822 }
14823
14824 fn replay_counter_query(
14825 query_name: &str,
14826 history_events: Value,
14827 run_status: &str,
14828 ) -> QueryTask {
14829 let arguments = fixture_envelope(json!([]));
14830 serde_json::from_value(json!({
14831 "query_task_id": format!("query-{query_name}"),
14832 "workflow_type": "replay-counter",
14833 "query_name": query_name,
14834 "payload_codec": DEFAULT_CODEC,
14835 "workflow_arguments": arguments.clone(),
14836 "query_arguments": arguments,
14837 "history_events": history_events,
14838 "run_status": run_status,
14839 }))
14840 .expect("query task")
14841 }
14842
14843 fn workflow_context(history: Vec<HistoryEvent>) -> WorkflowContext {
14844 workflow_context_with_codec(history, DEFAULT_CODEC)
14845 }
14846
14847 fn workflow_context_with_codec(
14848 history: Vec<HistoryEvent>,
14849 payload_codec: &str,
14850 ) -> WorkflowContext {
14851 WorkflowContext {
14852 state: Arc::new(Mutex::new(
14853 WorkflowState::new_with_identity(
14854 history,
14855 None,
14856 None,
14857 "rust-workers".to_string(),
14858 payload_codec.to_string(),
14859 None,
14860 )
14861 .expect("valid workflow history"),
14862 )),
14863 }
14864 }
14865
14866 fn history_event(event_type: &str, payload: Value) -> HistoryEvent {
14867 HistoryEvent {
14868 event_type: event_type.to_string(),
14869 payload,
14870 raw: HashMap::new(),
14871 }
14872 }
14873
14874 fn parallel_path_entry(
14875 kind: &str,
14876 base: u64,
14877 size: usize,
14878 index: usize,
14879 ) -> ParallelGroupMetadata {
14880 parallel_group_entry(base, size, index, kind)
14881 }
14882
14883 fn parallel_history_event(
14884 event_type: &str,
14885 sequence: u64,
14886 identity_field: &str,
14887 identity: &str,
14888 path: Vec<ParallelGroupMetadata>,
14889 result: Option<Value>,
14890 ) -> HistoryEvent {
14891 let mut payload = serde_json::Map::from_iter([
14892 ("sequence".to_string(), json!(sequence)),
14893 (identity_field.to_string(), json!(identity)),
14894 ]);
14895 let inner = path.last().expect("parallel history path");
14896 apply_parallel_group_path(&mut payload, std::slice::from_ref(inner));
14897 payload.insert("parallel_group_path".to_string(), json!(path));
14898 if let Some(result) = result {
14899 let field = if event_type == "ChildRunCompleted" {
14900 "result"
14901 } else {
14902 "result"
14903 };
14904 payload.insert(field.to_string(), fixture_envelope(result));
14905 payload.insert("payload_codec".to_string(), json!(DEFAULT_CODEC));
14906 }
14907 history_event(event_type, Value::Object(payload))
14908 }
14909
14910 fn nested_parallel_operations() -> Vec<ParallelOperation> {
14911 vec![
14912 ParallelOperation::activity("first", json!([])),
14913 ParallelOperation::group(vec![
14914 ParallelOperation::child_workflow(
14915 "second",
14916 ChildWorkflowOptions::new("child-workers"),
14917 json!([]),
14918 ),
14919 ParallelOperation::activity("third", json!([])),
14920 ]),
14921 ]
14922 }
14923
14924 fn nested_parallel_paths() -> [Vec<ParallelGroupMetadata>; 3] {
14925 let outer = [
14926 parallel_path_entry("mixed", 1, 3, 0),
14927 parallel_path_entry("mixed", 1, 3, 1),
14928 parallel_path_entry("mixed", 1, 3, 2),
14929 ];
14930 [
14931 vec![outer[0].clone()],
14932 vec![outer[1].clone(), parallel_path_entry("mixed", 2, 2, 0)],
14933 vec![outer[2].clone(), parallel_path_entry("mixed", 2, 2, 1)],
14934 ]
14935 }
14936
14937 #[test]
14938 fn parallel_schedules_every_nested_mixed_leaf_with_stable_metadata() {
14939 let ctx = workflow_context(Vec::new());
14940 let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
14941 let mut task_context = TaskContext::from_waker(noop_waker_ref());
14942
14943 assert!(matches!(
14944 call.as_mut().poll(&mut task_context),
14945 Poll::Pending
14946 ));
14947 let commands = ctx.take_commands().expect("parallel commands");
14948 assert_eq!(
14949 commands
14950 .iter()
14951 .map(|command| command["type"].as_str().unwrap_or_default())
14952 .collect::<Vec<_>>(),
14953 [
14954 "schedule_activity",
14955 "start_child_workflow",
14956 "schedule_activity"
14957 ]
14958 );
14959 let paths = nested_parallel_paths();
14960 for (command, path) in commands.iter().zip(paths) {
14961 assert_eq!(command["parallel_group_path"], json!(path));
14962 assert_eq!(
14963 command["parallel_group_id"],
14964 json!(path.last().expect("inner group").parallel_group_id)
14965 );
14966 }
14967 }
14968
14969 fn completed_nested_parallel_history() -> Vec<HistoryEvent> {
14970 let paths = nested_parallel_paths();
14971 let third = parallel_history_event(
14972 "ActivityCompleted",
14973 3,
14974 "activity_type",
14975 "third",
14976 paths[2].clone(),
14977 Some(json!("three")),
14978 );
14979 vec![
14980 parallel_history_event(
14981 "ActivityCompleted",
14982 1,
14983 "activity_type",
14984 "first",
14985 paths[0].clone(),
14986 Some(json!("one")),
14987 ),
14988 parallel_history_event(
14989 "ChildWorkflowScheduled",
14990 2,
14991 "child_workflow_type",
14992 "second",
14993 paths[1].clone(),
14994 None,
14995 ),
14996 parallel_history_event(
14997 "ChildRunCompleted",
14998 2,
14999 "child_workflow_type",
15000 "second",
15001 paths[1].clone(),
15002 Some(json!("two")),
15003 ),
15004 third.clone(),
15005 third,
15006 ]
15007 }
15008
15009 #[test]
15010 fn parallel_replay_rebuilds_input_order_and_tolerates_duplicate_delivery() {
15011 for _restart_or_completed_replay in 0..2 {
15012 let ctx = workflow_context(completed_nested_parallel_history());
15013 let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15014 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15015 let Poll::Ready(Ok(results)) = call.as_mut().poll(&mut task_context) else {
15016 panic!("completed nested parallel history must replay");
15017 };
15018 assert_eq!(
15019 results,
15020 vec![
15021 ParallelResult::Activity(json!("one")),
15022 ParallelResult::Group(vec![
15023 ParallelResult::ChildWorkflow(ChildWorkflowResult {
15024 parent: WorkflowIdentity {
15025 workflow_id: None,
15026 run_id: None,
15027 },
15028 child: WorkflowIdentity {
15029 workflow_id: None,
15030 run_id: None,
15031 },
15032 child_workflow_type: Some("second".to_string()),
15033 result: json!("two"),
15034 }),
15035 ParallelResult::Activity(json!("three")),
15036 ]),
15037 ]
15038 );
15039 assert!(ctx.take_commands().expect("commands").is_empty());
15040 ctx.ensure_history_consumed().expect("history consumed");
15041 }
15042 }
15043
15044 #[test]
15045 fn parallel_failure_keeps_typed_cause_path_and_late_completions() {
15046 let paths = nested_parallel_paths();
15047 let history = vec![
15048 parallel_history_event(
15049 "ActivityCompleted",
15050 1,
15051 "activity_type",
15052 "first",
15053 paths[0].clone(),
15054 Some(json!("one")),
15055 ),
15056 parallel_history_event(
15057 "ChildWorkflowScheduled",
15058 2,
15059 "child_workflow_type",
15060 "second",
15061 paths[1].clone(),
15062 None,
15063 ),
15064 parallel_history_event(
15065 "ChildRunFailed",
15066 2,
15067 "child_workflow_type",
15068 "second",
15069 paths[1].clone(),
15070 None,
15071 ),
15072 parallel_history_event(
15073 "ActivityCompleted",
15074 3,
15075 "activity_type",
15076 "third",
15077 paths[2].clone(),
15078 Some(json!("late")),
15079 ),
15080 ];
15081 let ctx = workflow_context(history);
15082 let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15083 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15084 let outcome = call.as_mut().poll(&mut task_context);
15085 let Poll::Ready(Err(Error::ParallelFailed(failure))) = outcome else {
15086 panic!("one failed child must return a typed partial failure: {outcome:?}");
15087 };
15088 assert_eq!(failure.member_path, [1, 0]);
15089 assert_eq!(failure.group_id, "parallel-calls:1:3");
15090 assert!(matches!(*failure.cause, Error::ChildWorkflowFailed(_)));
15091 assert_eq!(
15092 failure
15093 .completed
15094 .iter()
15095 .map(|completion| completion.member_path.clone())
15096 .collect::<Vec<_>>(),
15097 [vec![0], vec![1, 1]]
15098 );
15099 }
15100
15101 #[test]
15102 fn pending_parallel_history_restarts_without_rescheduling_any_leaf() {
15103 let paths = nested_parallel_paths();
15104 let history = vec![
15105 parallel_history_event(
15106 "ActivityScheduled",
15107 1,
15108 "activity_type",
15109 "first",
15110 paths[0].clone(),
15111 None,
15112 ),
15113 parallel_history_event(
15114 "ChildWorkflowScheduled",
15115 2,
15116 "child_workflow_type",
15117 "second",
15118 paths[1].clone(),
15119 None,
15120 ),
15121 parallel_history_event(
15122 "ActivityScheduled",
15123 3,
15124 "activity_type",
15125 "third",
15126 paths[2].clone(),
15127 None,
15128 ),
15129 ];
15130 for _restart in 0..2 {
15131 let ctx = workflow_context(history.clone());
15132 let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15133 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15134 let outcome = call.as_mut().poll(&mut task_context);
15135 assert!(matches!(outcome, Poll::Pending), "{outcome:?}");
15136 assert!(ctx.take_commands().expect("commands").is_empty());
15137 }
15138 }
15139
15140 fn selection_path(index: usize, key: &str) -> Vec<ParallelGroupMetadata> {
15141 vec![selection_group_entry(
15142 1,
15143 2,
15144 index,
15145 "activity",
15146 &SelectionMemberMetadata {
15147 key: SelectionKey::Name(key.to_string()),
15148 index,
15149 base_sequence: index as u64 + 1,
15150 size: 1,
15151 kind: "activity".to_string(),
15152 },
15153 )]
15154 }
15155
15156 fn selection_activity_event(
15157 event_type: &str,
15158 index: usize,
15159 key: &str,
15160 result: Option<Value>,
15161 ) -> HistoryEvent {
15162 let sequence = index as u64 + 1;
15163 let mut event = parallel_history_event(
15164 event_type,
15165 sequence,
15166 "activity_type",
15167 &format!("{key}-activity"),
15168 selection_path(index, key),
15169 result,
15170 );
15171 event.payload["activity_execution_id"] = json!(format!("activity-{key}"));
15172 event.raw.insert(
15173 "id".to_string(),
15174 json!(if event_type == "ActivityCompleted" {
15175 format!("event-{key}")
15176 } else {
15177 format!("{event_type}-{key}")
15178 }),
15179 );
15180 event
15181 }
15182
15183 fn selection_winner_marker() -> HistoryEvent {
15184 history_event(
15185 "SelectionResolved",
15186 json!({
15187 "selection_group_id": "select-calls:1:2",
15188 "selection_group_base_sequence": 1,
15189 "selection_group_size": 2,
15190 "member_key": "fast",
15191 "member_index": 1,
15192 "member_base_sequence": 2,
15193 "member_size": 1,
15194 "operation_kind": "activity",
15195 "operation_identity": "activity-fast",
15196 "outcome": "completed",
15197 "resolution_event_id": "event-fast",
15198 "resolution_event_type": "ActivityCompleted",
15199 }),
15200 )
15201 }
15202
15203 fn keyed_activity_selection(ctx: &WorkflowContext) -> SelectCall {
15204 ctx.select_keyed(vec![
15205 (
15206 "slow",
15207 ParallelOperation::activity_with_options(
15208 "slow-activity",
15209 ActivityOptions::new().task_queue("default"),
15210 json!([]),
15211 ),
15212 ),
15213 (
15214 "fast",
15215 ParallelOperation::activity_with_options(
15216 "fast-activity",
15217 ActivityOptions::new().task_queue("default"),
15218 json!([]),
15219 ),
15220 ),
15221 ])
15222 }
15223
15224 fn assert_persisted_selection_replay(history: Vec<HistoryEvent>) {
15225 let ctx = workflow_context(history);
15226 let mut call = Box::pin(keyed_activity_selection(&ctx));
15227 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15228 let selected = match call.as_mut().poll(&mut task_context) {
15229 Poll::Ready(Ok(selected)) => selected,
15230 Poll::Ready(Err(error)) => panic!("persisted selection winner must replay: {error:?}"),
15231 Poll::Pending => panic!("persisted selection winner must replay without pending"),
15232 };
15233 assert_eq!(selected.key, SelectionKey::Name("fast".to_string()));
15234 assert_eq!(
15235 selected.value,
15236 Some(ParallelResult::Activity(json!("winner-value")))
15237 );
15238 let slow = selected
15239 .handle(&SelectionKey::Name("slow".to_string()))
15240 .expect("slow handle")
15241 .clone();
15242 let mut await_slow = Box::pin(slow.await_result());
15243 assert!(matches!(
15244 await_slow.as_mut().poll(&mut task_context),
15245 Poll::Ready(Ok(ParallelResult::Activity(value))) if value == json!("loser-value")
15246 ));
15247 assert!(ctx.take_commands().expect("commands").is_empty());
15248 }
15249
15250 const SELECTION_COLD_REPLAY_HISTORY: &str = "DURABLE_WORKFLOW_SELECTION_COLD_REPLAY_HISTORY";
15251
15252 fn canonical_selection_history() -> Vec<HistoryEvent> {
15253 const FIXTURE: &[u8] =
15254 include_bytes!("../tests/fixtures/durable_selection_runtime_history.json");
15255 assert_eq!(
15256 format!("{:x}", Sha256::digest(FIXTURE)),
15257 "51fd8b9c16e978dcef536a5c727b9fdc0ae724d9afc17d9a7837d219f41ee3ba",
15258 );
15259 let fixture: Value = serde_json::from_slice(FIXTURE).expect("canonical selection fixture");
15260
15261 serde_json::from_value(fixture["history"].clone()).expect("canonical selection history")
15262 }
15263
15264 #[test]
15265 fn selection_fresh_process_entrypoint() {
15266 let Ok(path) = std::env::var(SELECTION_COLD_REPLAY_HISTORY) else {
15267 return;
15268 };
15269 let persisted = fs::read(path).expect("persisted selection history");
15270 assert_eq!(
15271 format!("{:x}", Sha256::digest(&persisted)),
15272 "51fd8b9c16e978dcef536a5c727b9fdc0ae724d9afc17d9a7837d219f41ee3ba",
15273 );
15274 let fixture: Value =
15275 serde_json::from_slice(&persisted).expect("valid persisted selection fixture");
15276 let history: Vec<HistoryEvent> = serde_json::from_value(fixture["history"].clone())
15277 .expect("valid persisted selection history");
15278
15279 assert_persisted_selection_replay(history);
15280 }
15281
15282 #[test]
15283 fn selection_starts_every_member_with_stable_keys_and_group_identity() {
15284 let ctx = workflow_context(Vec::new());
15285 let mut call = Box::pin(keyed_activity_selection(&ctx));
15286 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15287
15288 assert!(matches!(
15289 call.as_mut().poll(&mut task_context),
15290 Poll::Pending
15291 ));
15292 let commands = ctx.take_commands().expect("selection commands");
15293 assert_eq!(commands.len(), 2);
15294 assert_eq!(commands[0]["selection_member_key"], json!("slow"));
15295 assert_eq!(commands[1]["selection_member_key"], json!("fast"));
15296 assert!(commands.iter().all(|command| {
15297 command["parallel_group_id"] == json!("select-calls:1:2")
15298 && command["parallel_group_mode"] == json!("select")
15299 }));
15300 }
15301
15302 #[test]
15303 fn selection_key_domain_rejects_empty_authoring_and_malformed_history() {
15304 let ctx = workflow_context(Vec::new());
15305 let mut invalid = Box::pin(ctx.select_keyed(vec![(
15306 "",
15307 ParallelOperation::activity("invalid", json!([])),
15308 )]));
15309 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15310 assert!(matches!(
15311 invalid.as_mut().poll(&mut task_context),
15312 Poll::Ready(Err(Error::InvalidParallelGroup(ParallelGroupError {
15313 reason: "selection_key_invalid",
15314 ..
15315 })))
15316 ));
15317
15318 for invalid_key in [json!(""), json!(-1)] {
15319 let mut event = selection_activity_event("ActivityScheduled", 0, "slow", None);
15320 event.payload["selection_member_key"] = invalid_key.clone();
15321 event.payload["parallel_group_path"][0]["selection_member_key"] = invalid_key;
15322 assert!(matches!(
15323 WorkflowState::new_with_identity(
15324 vec![event],
15325 None,
15326 None,
15327 "rust-workers".to_string(),
15328 DEFAULT_CODEC.to_string(),
15329 None,
15330 ),
15331 Err(Error::NonDeterministicReplay(_))
15332 ));
15333 }
15334 }
15335
15336 #[test]
15337 fn selection_preserves_valid_named_and_numeric_keys() {
15338 let ctx = workflow_context(Vec::new());
15339 let mut selection = Box::pin(ctx.select_keyed(vec![
15340 (
15341 SelectionKey::Index(0),
15342 ParallelOperation::activity("numeric", json!([])),
15343 ),
15344 (
15345 SelectionKey::Name("named".to_string()),
15346 ParallelOperation::timer(Duration::from_secs(1)),
15347 ),
15348 ]));
15349 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15350
15351 assert!(matches!(
15352 selection.as_mut().poll(&mut task_context),
15353 Poll::Pending
15354 ));
15355 let commands = ctx.take_commands().expect("selection commands");
15356 assert_eq!(commands[0]["selection_member_key"], json!(0));
15357 assert_eq!(commands[1]["selection_member_key"], json!("named"));
15358 }
15359
15360 #[test]
15361 fn selection_replays_persisted_winner_and_loser_can_be_awaited_later() {
15362 let history = canonical_selection_history();
15363 assert_persisted_selection_replay(history.clone());
15364
15365 let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
15366 .join("tests/fixtures/durable_selection_runtime_history.json");
15367 let output =
15368 ProcessCommand::new(std::env::current_exe().expect("current Rust test binary"))
15369 .args([
15370 "--exact",
15371 "tests::selection_fresh_process_entrypoint",
15372 "--nocapture",
15373 ])
15374 .env(SELECTION_COLD_REPLAY_HISTORY, &path)
15375 .output()
15376 .expect("run fresh selection replay process");
15377
15378 assert!(
15379 output.status.success(),
15380 "fresh selection replay failed:\nstdout:\n{}\nstderr:\n{}",
15381 String::from_utf8_lossy(&output.stdout),
15382 String::from_utf8_lossy(&output.stderr),
15383 );
15384 }
15385
15386 #[test]
15387 fn selection_waits_durably_when_terminal_members_precede_the_winner_marker() {
15388 let mut history = canonical_selection_history();
15389 history.retain(|event| event.event_type != "SelectionResolved");
15390 let ctx = workflow_context(history);
15391 let mut selection = Box::pin(keyed_activity_selection(&ctx));
15392 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15393
15394 assert!(matches!(
15395 selection.as_mut().poll(&mut task_context),
15396 Poll::Pending
15397 ));
15398 assert!(ctx.take_commands().expect("commands").is_empty());
15399 assert!(
15400 ctx.matched_recorded_pending()
15401 .expect("selection pending state"),
15402 "terminal member history must keep the workflow durably pending until SelectionResolved commits"
15403 );
15404 }
15405
15406 #[test]
15407 fn selection_terminal_condition_history_waits_durably_for_its_winner_marker() {
15408 for (terminal_event, predicate_satisfied, timeout_seconds) in [
15409 ("ConditionWaitSatisfied", true, None),
15410 ("ConditionWaitTimedOut", false, Some(0)),
15411 ] {
15412 let member = SelectionMemberMetadata {
15413 key: SelectionKey::Name("condition".to_string()),
15414 index: 0,
15415 base_sequence: 1,
15416 size: 1,
15417 kind: "condition".to_string(),
15418 };
15419 let path = vec![selection_group_entry(1, 1, 0, "condition", &member)];
15420 let mut payload = json!({
15421 "sequence": 1,
15422 "condition_wait_id": "condition-1",
15423 "condition_wait_occurrence_id": "rust:condition-wait:0",
15424 "condition_key": "ready",
15425 "condition_definition_fingerprint": "sha256:ready-v1",
15426 "parallel_group_path": path,
15427 });
15428 payload
15429 .as_object_mut()
15430 .expect("condition history payload")
15431 .extend(
15432 serde_json::to_value(&path[0])
15433 .expect("condition selection metadata")
15434 .as_object()
15435 .expect("condition selection metadata object")
15436 .clone(),
15437 );
15438 if let Some(timeout_seconds) = timeout_seconds {
15439 payload["timeout_seconds"] = json!(timeout_seconds);
15440 }
15441 let history = vec![
15442 history_event("ConditionWaitOpened", payload.clone()),
15443 history_event(terminal_event, payload),
15444 ];
15445 let ctx = workflow_context(history);
15446 let mut options = ConditionWaitOptions::new("ready", "sha256:ready-v1");
15447 if timeout_seconds.is_some() {
15448 options = options.timeout(Duration::ZERO);
15449 }
15450 let mut selection = Box::pin(ctx.select_keyed(vec![(
15451 "condition",
15452 ParallelOperation::condition(options, move || Ok(predicate_satisfied)),
15453 )]));
15454 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15455
15456 assert!(matches!(
15457 selection.as_mut().poll(&mut task_context),
15458 Poll::Pending
15459 ));
15460 assert!(ctx.take_commands().expect("commands").is_empty());
15461 assert!(
15462 ctx.matched_recorded_pending()
15463 .expect("condition selection pending state"),
15464 "{terminal_event} must keep the workflow durably pending until SelectionResolved commits"
15465 );
15466 }
15467 }
15468
15469 #[test]
15470 fn selection_immediate_condition_members_open_a_durable_wait() {
15471 for predicate_satisfied in [true, false] {
15472 let ctx = workflow_context(Vec::new());
15473 let mut selection = Box::pin(ctx.select_keyed(vec![(
15474 "condition",
15475 ParallelOperation::condition(
15476 ConditionWaitOptions::new("ready", "sha256:ready-v1").timeout(Duration::ZERO),
15477 move || Ok(predicate_satisfied),
15478 ),
15479 )]));
15480 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15481
15482 assert!(matches!(
15483 selection.as_mut().poll(&mut task_context),
15484 Poll::Pending
15485 ));
15486 let commands = ctx.take_commands().expect("condition selection command");
15487 assert_eq!(commands.len(), 1);
15488 assert_eq!(commands[0]["type"], json!("open_condition_wait"));
15489 assert_eq!(commands[0]["timeout_seconds"], json!(0));
15490 assert_eq!(
15491 commands[0]["parallel_group_path"][0]["parallel_group_mode"],
15492 json!("select")
15493 );
15494 }
15495 }
15496
15497 #[test]
15498 fn selection_loser_cancellation_is_explicit_and_idempotent() {
15499 let history = vec![
15500 selection_activity_event("ActivityScheduled", 0, "slow", None),
15501 selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
15502 selection_winner_marker(),
15503 ];
15504 let ctx = workflow_context(history.clone());
15505 let mut call = Box::pin(keyed_activity_selection(&ctx));
15506 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15507 let Poll::Ready(Ok(selected)) = call.as_mut().poll(&mut task_context) else {
15508 panic!("winner must replay");
15509 };
15510 let slow = selected
15511 .handle(&SelectionKey::Name("slow".to_string()))
15512 .expect("slow handle")
15513 .clone();
15514 let mut cancel = Box::pin(slow.cancel());
15515 assert!(matches!(
15516 cancel.as_mut().poll(&mut task_context),
15517 Poll::Pending
15518 ));
15519 assert!(matches!(
15520 cancel.as_mut().poll(&mut task_context),
15521 Poll::Pending
15522 ));
15523 let commands = ctx.take_commands().expect("cancel command");
15524 assert_eq!(commands.len(), 1);
15525 assert_eq!(commands[0]["type"], json!("cancel_selection_operation"));
15526 assert_eq!(commands[0]["member_key"], json!("slow"));
15527
15528 let mut cancelled_history = history;
15529 cancelled_history.push(history_event(
15530 "SelectionOperationCancelled",
15531 json!({
15532 "selection_group_id": "select-calls:1:2",
15533 "member_key": "slow",
15534 "member_index": 0,
15535 "member_base_sequence": 1,
15536 "member_size": 1,
15537 "operation_kind": "activity",
15538 "operation_identity": "activity-slow",
15539 "cancelled_at": "2026-08-27T00:00:00Z",
15540 }),
15541 ));
15542 let replayed = workflow_context(cancelled_history);
15543 let mut call = Box::pin(keyed_activity_selection(&replayed));
15544 let Poll::Ready(Ok(selected)) = call.as_mut().poll(&mut task_context) else {
15545 panic!("winner must replay after cancellation");
15546 };
15547 let slow = selected
15548 .handle(&SelectionKey::Name("slow".to_string()))
15549 .expect("slow handle")
15550 .clone();
15551 let mut cancel = Box::pin(slow.cancel());
15552 assert!(matches!(
15553 cancel.as_mut().poll(&mut task_context),
15554 Poll::Ready(Ok(()))
15555 ));
15556 assert!(replayed.take_commands().expect("commands").is_empty());
15557 }
15558
15559 #[test]
15560 fn selection_cancellation_marker_is_bound_to_every_authored_handle_field() {
15561 let base_history = vec![
15562 selection_activity_event("ActivityScheduled", 0, "slow", None),
15563 selection_activity_event("ActivityScheduled", 1, "fast", None),
15564 selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
15565 selection_winner_marker(),
15566 ];
15567 for (field, corrupt) in [
15568 ("member_key", json!("fast")),
15569 ("member_index", json!(1)),
15570 ("member_base_sequence", json!(3)),
15571 ("member_size", json!(2)),
15572 ("operation_kind", json!("timer")),
15573 ("operation_identity", json!("forged")),
15574 ] {
15575 let mut cancellation = json!({
15576 "selection_group_id": "select-calls:1:2",
15577 "member_key": "slow",
15578 "member_index": 0,
15579 "member_base_sequence": 1,
15580 "member_size": 1,
15581 "operation_kind": "activity",
15582 "operation_identity": "activity-slow",
15583 });
15584 cancellation[field] = corrupt;
15585 let mut history = base_history.clone();
15586 history.push(history_event("SelectionOperationCancelled", cancellation));
15587 let ctx = workflow_context(history);
15588 let mut selection = Box::pin(keyed_activity_selection(&ctx));
15589 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15590
15591 assert!(matches!(
15592 selection.as_mut().poll(&mut task_context),
15593 Poll::Ready(Err(Error::NonDeterministicReplay(_)))
15594 ));
15595 }
15596 }
15597
15598 #[test]
15599 fn selection_child_identity_prefers_the_durable_run_id() {
15600 let ctx = workflow_context(vec![history_event(
15601 "ChildWorkflowScheduled",
15602 json!({
15603 "sequence": 1,
15604 "child_workflow_type": "child",
15605 "child_workflow_instance_id": "child-instance",
15606 "child_workflow_run_id": "child-run",
15607 }),
15608 )]);
15609 let state = ctx.state.lock().expect("workflow state");
15610
15611 assert_eq!(
15612 selection_operation_identity(&state, "child", 1, 1),
15613 "child-run"
15614 );
15615 }
15616
15617 #[test]
15618 fn selection_activity_identity_requires_canonical_execution_id() {
15619 let slow = selection_activity_event("ActivityScheduled", 0, "slow", None);
15620 let mut fast_open = selection_activity_event("ActivityScheduled", 1, "fast", None);
15621 let mut fast_completed =
15622 selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner")));
15623 for event in [&mut fast_open, &mut fast_completed] {
15624 event
15625 .payload
15626 .as_object_mut()
15627 .expect("activity payload")
15628 .remove("activity_execution_id");
15629 event.payload["activity_id"] = json!("forged-activity-id");
15630 }
15631 let mut marker = selection_winner_marker();
15632 marker.payload["operation_identity"] = json!("forged-activity-id");
15633 let ctx = workflow_context(vec![slow, fast_open, fast_completed, marker]);
15634 let mut selection = Box::pin(keyed_activity_selection(&ctx));
15635 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15636
15637 assert!(matches!(
15638 selection.as_mut().poll(&mut task_context),
15639 Poll::Ready(Err(Error::NonDeterministicReplay(_)))
15640 ));
15641 }
15642
15643 #[test]
15644 fn selection_completion_before_cancellation_remains_awaitable() {
15645 let history = vec![
15646 selection_activity_event("ActivityScheduled", 0, "slow", None),
15647 selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
15648 selection_winner_marker(),
15649 selection_activity_event(
15650 "ActivityCompleted",
15651 0,
15652 "slow",
15653 Some(json!("completed-first")),
15654 ),
15655 ];
15656 let ctx = workflow_context(history);
15657 let mut selection = Box::pin(keyed_activity_selection(&ctx));
15658 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15659 let Poll::Ready(Ok(selected)) = selection.as_mut().poll(&mut task_context) else {
15660 panic!("winner must replay");
15661 };
15662 let slow = selected
15663 .handle(&SelectionKey::Name("slow".to_string()))
15664 .expect("slow handle")
15665 .clone();
15666 let mut cancel = Box::pin(slow.cancel());
15667 assert!(matches!(
15668 cancel.as_mut().poll(&mut task_context),
15669 Poll::Ready(Ok(()))
15670 ));
15671 let mut await_slow = Box::pin(slow.await_result());
15672 assert!(matches!(
15673 await_slow.as_mut().poll(&mut task_context),
15674 Poll::Ready(Ok(ParallelResult::Activity(value))) if value == json!("completed-first")
15675 ));
15676 let commands = ctx.take_commands().expect("commands");
15677 assert!(commands.is_empty());
15678 }
15679
15680 #[test]
15681 fn selection_nested_later_failure_before_cancel_remains_the_awaited_failure() {
15682 let nested_member = SelectionMemberMetadata {
15683 key: SelectionKey::Name("nested".to_string()),
15684 index: 0,
15685 base_sequence: 1,
15686 size: 2,
15687 kind: "group".to_string(),
15688 };
15689 let deadline_member = SelectionMemberMetadata {
15690 key: SelectionKey::Name("deadline".to_string()),
15691 index: 1,
15692 base_sequence: 3,
15693 size: 1,
15694 kind: "timer".to_string(),
15695 };
15696 let nested_paths = [
15697 vec![
15698 selection_group_entry(1, 3, 0, "mixed", &nested_member),
15699 parallel_group_entry(1, 2, 0, "activity"),
15700 ],
15701 vec![
15702 selection_group_entry(1, 3, 1, "mixed", &nested_member),
15703 parallel_group_entry(1, 2, 1, "activity"),
15704 ],
15705 ];
15706 let deadline_path = vec![selection_group_entry(1, 3, 2, "mixed", &deadline_member)];
15707 let mut timer_fired = parallel_history_event(
15708 "TimerFired",
15709 3,
15710 "timer_id",
15711 "timer-3",
15712 deadline_path.clone(),
15713 None,
15714 );
15715 timer_fired.payload["delay_seconds"] = json!(0);
15716 timer_fired
15717 .raw
15718 .insert("id".to_string(), json!("timer-fired"));
15719 let mut timer_scheduled = parallel_history_event(
15720 "TimerScheduled",
15721 3,
15722 "timer_id",
15723 "timer-3",
15724 deadline_path,
15725 None,
15726 );
15727 timer_scheduled.payload["delay_seconds"] = json!(0);
15728 let history = vec![
15729 parallel_history_event(
15730 "ActivityScheduled",
15731 1,
15732 "activity_type",
15733 "nested-first",
15734 nested_paths[0].clone(),
15735 None,
15736 ),
15737 parallel_history_event(
15738 "ActivityScheduled",
15739 2,
15740 "activity_type",
15741 "nested-second",
15742 nested_paths[1].clone(),
15743 None,
15744 ),
15745 timer_scheduled,
15746 timer_fired,
15747 history_event(
15748 "SelectionResolved",
15749 json!({
15750 "selection_group_id": "select-calls:1:3",
15751 "selection_group_base_sequence": 1,
15752 "selection_group_size": 3,
15753 "member_key": "deadline",
15754 "member_index": 1,
15755 "member_base_sequence": 3,
15756 "member_size": 1,
15757 "operation_kind": "timer",
15758 "operation_identity": "timer-3",
15759 "outcome": "completed",
15760 "resolution_event_id": "timer-fired",
15761 "resolution_event_type": "TimerFired",
15762 }),
15763 ),
15764 parallel_history_event(
15765 "ActivityFailed",
15766 2,
15767 "activity_type",
15768 "nested-second",
15769 nested_paths[1].clone(),
15770 None,
15771 ),
15772 ];
15773 let ctx = workflow_context(history);
15774 let mut selection = Box::pin(ctx.select_keyed(vec![
15775 (
15776 "nested",
15777 ParallelOperation::group(vec![
15778 ParallelOperation::activity("nested-first", json!([])),
15779 ParallelOperation::activity("nested-second", json!([])),
15780 ]),
15781 ),
15782 ("deadline", ParallelOperation::timer(Duration::ZERO)),
15783 ]));
15784 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15785 let Poll::Ready(Ok(selected)) = selection.as_mut().poll(&mut task_context) else {
15786 panic!("deadline winner must replay");
15787 };
15788 let nested = selected
15789 .handle(&SelectionKey::Name("nested".to_string()))
15790 .expect("nested handle")
15791 .clone();
15792 let mut cancel = Box::pin(nested.cancel());
15793 assert!(matches!(
15794 cancel.as_mut().poll(&mut task_context),
15795 Poll::Ready(Ok(()))
15796 ));
15797 let mut await_nested = Box::pin(nested.await_result());
15798
15799 assert!(matches!(
15800 await_nested.as_mut().poll(&mut task_context),
15801 Poll::Ready(Err(Error::ActivityFailed(_)))
15802 ));
15803 assert!(ctx.take_commands().expect("commands").is_empty());
15804 }
15805
15806 #[test]
15807 fn selection_supports_child_timer_signal_condition_and_nested_groups() {
15808 let ctx = workflow_context(Vec::new());
15809 let mut call = Box::pin(ctx.select(vec![
15810 ParallelOperation::child_workflow(
15811 "child",
15812 ChildWorkflowOptions::new("children"),
15813 json!([]),
15814 ),
15815 ParallelOperation::timer(Duration::from_secs(30)),
15816 ParallelOperation::signal("approval"),
15817 ParallelOperation::condition(
15818 ConditionWaitOptions::new("ready", "sha256:ready"),
15819 || Ok(false),
15820 ),
15821 ParallelOperation::group(vec![
15822 ParallelOperation::activity("nested-one", json!([])),
15823 ParallelOperation::activity("nested-two", json!([])),
15824 ]),
15825 ]));
15826 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15827 assert!(matches!(
15828 call.as_mut().poll(&mut task_context),
15829 Poll::Pending
15830 ));
15831 let commands = ctx.take_commands().expect("selection commands");
15832 assert_eq!(
15833 commands
15834 .iter()
15835 .map(|command| command["type"].as_str().unwrap_or_default())
15836 .collect::<Vec<_>>(),
15837 [
15838 "start_child_workflow",
15839 "start_timer",
15840 "open_signal_wait",
15841 "open_condition_wait",
15842 "schedule_activity",
15843 "schedule_activity",
15844 ]
15845 );
15846 assert!(commands.iter().all(|command| {
15847 command["parallel_group_path"][0]["parallel_group_mode"] == json!("select")
15848 }));
15849 assert_eq!(
15850 commands[4]["parallel_group_path"].as_array().map(Vec::len),
15851 Some(2)
15852 );
15853 assert_eq!(
15854 commands[4]["parallel_group_path"][0]["selection_member_kind"],
15855 json!("group")
15856 );
15857 assert_eq!(
15858 commands[5]["parallel_group_path"][0]["selection_member_kind"],
15859 json!("group")
15860 );
15861
15862 let one_leaf_ctx = workflow_context(Vec::new());
15863 let mut one_leaf = Box::pin(one_leaf_ctx.select(vec![ParallelOperation::group(vec![
15864 ParallelOperation::activity("nested-only", json!([])),
15865 ])]));
15866 assert!(matches!(
15867 one_leaf.as_mut().poll(&mut task_context),
15868 Poll::Pending
15869 ));
15870 let one_leaf_commands = one_leaf_ctx.take_commands().expect("one-leaf commands");
15871 assert_eq!(one_leaf_commands.len(), 1);
15872 assert_eq!(
15873 one_leaf_commands[0]["parallel_group_path"][0]["selection_member_kind"],
15874 json!("group")
15875 );
15876 assert_eq!(
15877 one_leaf_commands[0]["parallel_group_path"][0]["selection_member_size"],
15878 json!(1)
15879 );
15880 }
15881
15882 async fn trip_saga(ctx: WorkflowContext) -> Result<Value> {
15883 let mut saga = ctx.saga();
15884 let outcome = async {
15885 let flight = ctx.activity("trip.reserve-flight", json!([])).await?;
15886 saga.add_compensation("trip.cancel-flight", json!([flight]))?;
15887 let hotel = ctx.activity("trip.reserve-hotel", json!([])).await?;
15888 saga.add_compensation("trip.cancel-hotel", json!([hotel]))?;
15889 ctx.activity("trip.charge", json!([])).await?;
15890 Ok(json!({"status": "booked"}))
15891 }
15892 .await;
15893 saga.finish(outcome).await
15894 }
15895
15896 fn saga_activity(
15897 event_type: &str,
15898 sequence: u64,
15899 activity_type: &str,
15900 result: Option<Value>,
15901 ) -> HistoryEvent {
15902 let mut payload = json!({
15903 "sequence": sequence,
15904 "activity_type": activity_type,
15905 "message": format!("{activity_type} failed"),
15906 "exception_type": "PlannedFailure",
15907 "non_retryable": true,
15908 });
15909 if let Some(result) = result {
15910 payload["result"] = fixture_envelope(result);
15911 }
15912 history_event(event_type, payload)
15913 }
15914
15915 #[test]
15916 fn saga_replays_reverse_compensation_across_restart_and_duplicate_delivery() {
15917 let completed_hotel_compensation = saga_activity(
15918 "ActivityCompleted",
15919 4,
15920 "trip.cancel-hotel",
15921 Some(Value::Null),
15922 );
15923 let history = vec![
15924 saga_activity(
15925 "ActivityCompleted",
15926 1,
15927 "trip.reserve-flight",
15928 Some(json!("flight-1")),
15929 ),
15930 saga_activity(
15931 "ActivityCompleted",
15932 2,
15933 "trip.reserve-hotel",
15934 Some(json!("hotel-1")),
15935 ),
15936 saga_activity("ActivityFailed", 3, "trip.charge", None),
15937 completed_hotel_compensation.clone(),
15938 completed_hotel_compensation,
15939 ];
15940
15941 for _restart in 0..2 {
15942 let ctx = workflow_context(history.clone());
15943 let mut future = Box::pin(trip_saga(ctx.clone()));
15944 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15945 assert!(matches!(
15946 future.as_mut().poll(&mut task_context),
15947 Poll::Pending
15948 ));
15949 let commands = ctx.take_commands().expect("compensation command");
15950 assert_eq!(commands.len(), 1);
15951 assert_eq!(commands[0]["activity_type"], "trip.cancel-flight");
15952 }
15953 }
15954
15955 #[test]
15956 fn saga_compensation_failure_preserves_both_typed_failures() {
15957 let history = vec![
15958 saga_activity(
15959 "ActivityCompleted",
15960 1,
15961 "trip.reserve-flight",
15962 Some(json!("flight-1")),
15963 ),
15964 saga_activity(
15965 "ActivityCompleted",
15966 2,
15967 "trip.reserve-hotel",
15968 Some(json!("hotel-1")),
15969 ),
15970 saga_activity("ActivityFailed", 3, "trip.charge", None),
15971 saga_activity("ActivityFailed", 4, "trip.cancel-hotel", None),
15972 ];
15973 let ctx = workflow_context(history);
15974 let mut future = Box::pin(trip_saga(ctx));
15975 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15976 let Poll::Ready(Err(Error::SagaCompensationFailed(failure))) =
15977 future.as_mut().poll(&mut task_context)
15978 else {
15979 panic!("compensation failure must remain structured");
15980 };
15981 assert!(matches!(
15982 *failure.initiating_failure,
15983 Error::ActivityFailed(_)
15984 ));
15985 assert!(matches!(
15986 *failure.compensation_failure,
15987 Error::ActivityFailed(_)
15988 ));
15989 assert_eq!(failure.compensation_activity_type, "trip.cancel-hotel");
15990 assert_eq!(failure.compensation_registration_order, 2);
15991 }
15992
15993 #[test]
15994 fn saga_compensates_cooperative_cancellation() {
15995 let ctx = workflow_context(vec![saga_activity(
15996 "ActivityCompleted",
15997 1,
15998 "trip.reserve-flight",
15999 Some(json!("flight-1")),
16000 )]);
16001 ctx.state.lock().expect("state").cancel_requested = true;
16002 let run = {
16003 let ctx = ctx.clone();
16004 async move {
16005 let mut saga = ctx.saga();
16006 let outcome = async {
16007 let flight = ctx.activity("trip.reserve-flight", json!([])).await?;
16008 saga.add_compensation("trip.cancel-flight", json!([flight]))?;
16009 ctx.throw_if_cancellation_requested()?;
16010 Ok(json!("unexpected"))
16011 }
16012 .await;
16013 saga.finish(outcome).await
16014 }
16015 };
16016 let mut future = Box::pin(run);
16017 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16018 assert!(matches!(
16019 future.as_mut().poll(&mut task_context),
16020 Poll::Pending
16021 ));
16022 let commands = ctx.take_commands().expect("cancellation compensation");
16023 assert_eq!(commands[0]["activity_type"], "trip.cancel-flight");
16024 }
16025
16026 fn workflow_task(
16027 workflow_type: &str,
16028 history_events: Vec<HistoryEvent>,
16029 payload_codec: &str,
16030 ) -> WorkflowTask {
16031 WorkflowTask {
16032 task_id: format!("wft-{workflow_type}"),
16033 workflow_command_id: None,
16034 workflow_id: Some(format!("wf-{workflow_type}")),
16035 run_id: Some(format!("run-{workflow_type}")),
16036 workflow_type: workflow_type.to_string(),
16037 cancel_requested: false,
16038 payload_codec: payload_codec.to_string(),
16039 arguments: Some(
16040 encode_value_envelope(&json!([]), payload_codec).expect("workflow arguments"),
16041 ),
16042 total_history_events: Some(history_events.len() as u64),
16043 history_size_bytes: None,
16044 continue_as_new_recommended: None,
16045 history_budget_pressure: None,
16046 history_events,
16047 next_history_page_token: None,
16048 workflow_task_attempt: 1,
16049 workflow_signal_id: None,
16050 signal_name: None,
16051 signal_arguments: None,
16052 workflow_update_id: None,
16053 update_name: None,
16054 lease_owner: Some("rust-worker".to_string()),
16055 }
16056 }
16057
16058 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
16059 struct SideEffectProbe {
16060 request_id: String,
16061 attempt: u32,
16062 }
16063
16064 #[test]
16065 fn typed_side_effect_runs_callback_once_and_replay_skips_it() {
16066 let calls = AtomicUsize::new(0);
16067 let ctx = workflow_context(Vec::new());
16068 let value = ctx
16069 .side_effect(|| {
16070 calls.fetch_add(1, Ordering::SeqCst);
16071 SideEffectProbe {
16072 request_id: "request-42".to_string(),
16073 attempt: 3,
16074 }
16075 })
16076 .expect("first side effect");
16077 assert_eq!(value.attempt, 3);
16078 assert_eq!(calls.load(Ordering::SeqCst), 1);
16079 let commands = ctx.take_commands().expect("commands");
16080 assert_eq!(commands.len(), 1);
16081 assert_eq!(commands[0]["type"], "record_side_effect");
16082 assert_eq!(
16083 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
16084 serde_json::to_value(&value).expect("value")
16085 );
16086
16087 let replay = workflow_context(vec![history_event(
16088 "SideEffectRecorded",
16089 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16090 )]);
16091 let replayed: SideEffectProbe = replay
16092 .side_effect(|| {
16093 calls.fetch_add(1, Ordering::SeqCst);
16094 panic!("committed side-effect callbacks must not run during replay")
16095 })
16096 .expect("replayed side effect");
16097 assert_eq!(replayed, value);
16098 assert_eq!(calls.load(Ordering::SeqCst), 1);
16099 assert!(replay.take_commands().expect("commands").is_empty());
16100 replay.ensure_history_consumed().expect("history consumed");
16101 }
16102
16103 #[test]
16104 fn side_effect_uses_avro_envelope_and_uuid_is_replay_stable() {
16105 let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
16106 let value = ctx
16107 .side_effect(|| SideEffectProbe {
16108 request_id: "avro-request".to_string(),
16109 attempt: 1,
16110 })
16111 .expect("Avro side effect");
16112 let uuid = ctx.uuid_v4().expect("deterministic UUID");
16113 let commands = ctx.take_commands().expect("commands");
16114 assert_eq!(commands.len(), 2);
16115 assert_eq!(commands[0]["result"]["codec"], DEFAULT_CODEC);
16116 assert_eq!(commands[1]["result"]["codec"], DEFAULT_CODEC);
16117 assert_eq!(
16118 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
16119 serde_json::to_value(&value).expect("value")
16120 );
16121
16122 let replay = workflow_context_with_codec(
16123 vec![
16124 history_event(
16125 "SideEffectRecorded",
16126 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16127 ),
16128 history_event(
16129 "SideEffectRecorded",
16130 json!({"sequence": 2, "result": commands[1]["result"].clone()}),
16131 ),
16132 ],
16133 DEFAULT_CODEC,
16134 );
16135 let replayed: SideEffectProbe = replay
16136 .side_effect(|| panic!("Avro callback must not run"))
16137 .expect("replayed Avro value");
16138 let replayed_uuid = replay.uuid_v4().expect("replayed UUID");
16139 assert_eq!(replayed, value);
16140 assert_eq!(replayed_uuid, uuid);
16141 assert!(replay.take_commands().expect("commands").is_empty());
16142 }
16143
16144 #[test]
16145 fn typed_side_effect_replay_preserves_bytes_and_maps() {
16146 let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
16147 let value = ctx
16148 .side_effect_avro_value(typed_fidelity_probe)
16149 .expect("typed side effect");
16150 let commands = ctx.take_commands().expect("side-effect command");
16151 assert_eq!(
16152 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
16153 .expect("recorded side effect"),
16154 value
16155 );
16156
16157 let replay = workflow_context_with_codec(
16158 vec![history_event(
16159 "SideEffectRecorded",
16160 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16161 )],
16162 DEFAULT_CODEC,
16163 );
16164 assert_eq!(
16165 replay
16166 .side_effect_avro_value(|| panic!("replay must not invoke callback"))
16167 .expect("replayed typed side effect"),
16168 value
16169 );
16170 }
16171
16172 #[test]
16173 fn ordered_side_effects_share_the_durable_command_stream() {
16174 let first = encode_value_envelope(&json!("first"), DEFAULT_CODEC).expect("first");
16175 let second = encode_value_envelope(&json!(29), DEFAULT_CODEC).expect("second");
16176 let ctx = workflow_context(vec![
16177 history_event(
16178 "SideEffectRecorded",
16179 json!({"sequence": 1, "result": first}),
16180 ),
16181 history_event(
16182 "SideEffectRecorded",
16183 json!({"sequence": 2, "result": second}),
16184 ),
16185 ]);
16186 let first: String = ctx
16187 .side_effect(|| panic!("first callback must not run"))
16188 .expect("first replay");
16189 let second: i32 = ctx
16190 .side_effect(|| panic!("second callback must not run"))
16191 .expect("second replay");
16192 assert_eq!(first, "first");
16193 assert_eq!(second, 29);
16194 ctx.ensure_history_consumed().expect("ordered history");
16195
16196 let reordered = workflow_context(vec![history_event(
16197 "VersionMarkerRecorded",
16198 json!({
16199 "sequence": 1,
16200 "change_id": "before-side-effect",
16201 "version": 1,
16202 "min_supported": 1,
16203 "max_supported": 1,
16204 }),
16205 )]);
16206 let error = reordered
16207 .side_effect(|| "new".to_string())
16208 .expect_err("command reordering must fail");
16209 assert!(matches!(
16210 error,
16211 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16212 if reason == "recorded_command_mismatch"
16213 ));
16214 }
16215
16216 #[test]
16217 fn version_markers_replay_across_upgrades_and_do_not_duplicate() {
16218 let ctx = workflow_context(Vec::new());
16219 assert_eq!(ctx.get_version("checkout-v2", 1, 2).expect("version"), 2);
16220 assert_eq!(ctx.get_version("checkout-v2", 1, 3).expect("cached"), 2);
16221 assert!(ctx.patched("new-search").expect("patch"));
16222 ctx.deprecate_patch("new-search").expect("deprecate patch");
16223 let commands = ctx.take_commands().expect("commands");
16224 assert_eq!(commands.len(), 2);
16225 assert_eq!(commands[0]["type"], "record_version_marker");
16226 assert_eq!(commands[0]["version"], 2);
16227 assert_eq!(commands[1]["change_id"], "new-search");
16228
16229 let replay = workflow_context(vec![history_event(
16230 "VersionMarkerRecorded",
16231 json!({
16232 "sequence": 1,
16233 "change_id": "checkout-v2",
16234 "version": 2,
16235 "min_supported": 1,
16236 "max_supported": 2,
16237 }),
16238 )]);
16239 assert_eq!(replay.get_version("checkout-v2", 1, 4).expect("upgrade"), 2);
16240 assert_eq!(replay.get_version("checkout-v2", 2, 5).expect("repeat"), 2);
16241 assert!(replay.take_commands().expect("commands").is_empty());
16242 replay.ensure_history_consumed().expect("history consumed");
16243 }
16244
16245 #[test]
16246 fn version_markers_reject_incompatible_or_malformed_history() {
16247 let incompatible = workflow_context(vec![history_event(
16248 "VersionMarkerRecorded",
16249 json!({
16250 "sequence": 1,
16251 "change_id": "checkout-v2",
16252 "version": 1,
16253 "min_supported": 1,
16254 "max_supported": 2,
16255 }),
16256 )]);
16257 let error = incompatible
16258 .get_version("checkout-v2", 2, 3)
16259 .expect_err("old version is unsupported");
16260 assert!(matches!(
16261 error,
16262 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16263 if reason == "version_marker_incompatible_range"
16264 ));
16265
16266 for (history, reason) in [
16267 (
16268 vec![history_event("SideEffectRecorded", json!({"sequence": 1}))],
16269 "side_effect_result_missing",
16270 ),
16271 (
16272 vec![history_event(
16273 "SideEffectRecorded",
16274 json!({
16275 "sequence": 1,
16276 "result": {"codec": "avro", "blob": "not-base64"},
16277 }),
16278 )],
16279 "side_effect_payload_incompatible",
16280 ),
16281 (
16282 vec![history_event(
16283 "SideEffectRecorded",
16284 json!({"sequence": 1, "result": {"unwrapped": true}}),
16285 )],
16286 "side_effect_payload_malformed",
16287 ),
16288 (
16289 vec![history_event(
16290 "VersionMarkerRecorded",
16291 json!({
16292 "sequence": 1,
16293 "change_id": "change",
16294 "version": 1,
16295 "min_supported": 2,
16296 "max_supported": 1,
16297 }),
16298 )],
16299 "version_marker_history_range_invalid",
16300 ),
16301 ] {
16302 let error = WorkflowState::new(
16303 history,
16304 "rust-workers".to_string(),
16305 DEFAULT_CODEC.to_string(),
16306 None,
16307 )
16308 .expect_err("malformed history must fail");
16309 assert!(matches!(
16310 error,
16311 Error::NonDeterministicReplay(ReplayFailure { reason: actual, .. })
16312 if actual == reason
16313 ));
16314 }
16315 }
16316
16317 #[test]
16318 fn typed_search_attributes_replay_value_and_type_identity_after_restart() {
16319 let history = vec![history_event(
16320 "SearchAttributesUpserted",
16321 json!({
16322 "sequence": 1,
16323 "attributes": {"customer_tier": "gold"},
16324 "attribute_types": {"customer_tier": "keyword"},
16325 "merged": {"customer_tier": "gold"}
16326 }),
16327 )];
16328
16329 let matching = workflow_context(history.clone());
16330 matching
16331 .upsert_search_attributes(
16332 SearchAttributeUpdate::new()
16333 .keyword("customer_tier", "gold")
16334 .expect("keyword update"),
16335 )
16336 .expect("matching typed update must replay");
16337 matching
16338 .ensure_history_consumed()
16339 .expect("history consumed");
16340
16341 let changed_type = workflow_context(history.clone());
16342 let error = changed_type
16343 .upsert_search_attributes(
16344 SearchAttributeUpdate::new()
16345 .string("customer_tier", "gold")
16346 .expect("string update"),
16347 )
16348 .expect_err("same JSON value with a different declaration must be nondeterministic");
16349 let Error::NonDeterministicReplay(failure) = error else {
16350 panic!("typed identity drift must be a replay failure");
16351 };
16352 assert_eq!(failure.reason, "search_attribute_type_mismatch");
16353 assert_eq!(failure.sequence, Some(1));
16354
16355 let changed_value = workflow_context(history);
16356 let error = changed_value
16357 .upsert_search_attributes(
16358 SearchAttributeUpdate::new()
16359 .keyword("customer_tier", "platinum")
16360 .expect("keyword update"),
16361 )
16362 .expect_err("changed values must be nondeterministic");
16363 let Error::NonDeterministicReplay(failure) = error else {
16364 panic!("value drift must be a replay failure");
16365 };
16366 assert_eq!(failure.reason, "search_attribute_value_mismatch");
16367 }
16368
16369 #[test]
16370 fn legacy_search_attribute_history_keeps_type_identity_unknown() {
16371 let history = vec![history_event(
16372 "SearchAttributesUpserted",
16373 json!({
16374 "sequence": 1,
16375 "attributes": {"customer_tier": "gold"},
16376 "merged": {"customer_tier": "gold"}
16377 }),
16378 )];
16379
16380 for update in [
16381 SearchAttributeUpdate::new()
16382 .keyword("customer_tier", "gold")
16383 .expect("keyword update"),
16384 SearchAttributeUpdate::new()
16385 .string("customer_tier", "gold")
16386 .expect("string update"),
16387 ] {
16388 let restarted = workflow_context(history.clone());
16389 restarted
16390 .upsert_search_attributes(update)
16391 .expect("legacy history constrains values but has unknown type identity");
16392 restarted
16393 .ensure_history_consumed()
16394 .expect("history consumed");
16395 }
16396 }
16397
16398 #[test]
16399 fn search_attribute_command_emits_canonical_types() {
16400 let ctx = workflow_context(Vec::new());
16401 ctx.upsert_search_attributes(
16402 SearchAttributeUpdate::new()
16403 .keyword("customer_tier", "gold")
16404 .expect("keyword update")
16405 .int("attempts", 3)
16406 .expect("int update")
16407 .delete("obsolete")
16408 .expect("delete update"),
16409 )
16410 .expect("valid search attributes");
16411
16412 assert_eq!(
16413 ctx.take_commands().expect("commands"),
16414 vec![json!({
16415 "type": "upsert_search_attributes",
16416 "attributes": {
16417 "attempts": 3,
16418 "customer_tier": "gold",
16419 "obsolete": null
16420 },
16421 "attribute_types": {
16422 "attempts": "int",
16423 "customer_tier": "keyword"
16424 }
16425 })]
16426 );
16427 }
16428
16429 #[test]
16430 fn duplicate_side_effects_and_version_markers_are_rejected() {
16431 let duplicate_side_effect = WorkflowState::new(
16432 vec![
16433 history_event(
16434 "SideEffectRecorded",
16435 json!({"sequence": 1, "result": fixture_envelope(json!(1))}),
16436 ),
16437 history_event(
16438 "SideEffectRecorded",
16439 json!({"sequence": 1, "result": fixture_envelope(json!(2))}),
16440 ),
16441 ],
16442 "rust-workers".to_string(),
16443 DEFAULT_CODEC.to_string(),
16444 None,
16445 )
16446 .expect_err("duplicate side effect");
16447 assert!(matches!(
16448 duplicate_side_effect,
16449 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16450 if reason == "duplicate_side_effect_record"
16451 ));
16452
16453 let marker = |sequence| {
16454 history_event(
16455 "VersionMarkerRecorded",
16456 json!({
16457 "sequence": sequence,
16458 "change_id": "same-change",
16459 "version": 1,
16460 "min_supported": 1,
16461 "max_supported": 1,
16462 }),
16463 )
16464 };
16465 let duplicate_marker = WorkflowState::new(
16466 vec![marker(1), marker(3)],
16467 "rust-workers".to_string(),
16468 DEFAULT_CODEC.to_string(),
16469 None,
16470 )
16471 .expect_err("duplicate marker");
16472 assert!(matches!(
16473 duplicate_marker,
16474 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16475 if reason == "duplicate_version_marker"
16476 ));
16477 }
16478
16479 #[test]
16480 fn workflow_stream_authoring_derives_identity_and_replay_skips_duplicate_append() {
16481 let mut state = WorkflowState::new(
16482 Vec::new(),
16483 "rust-workers".to_string(),
16484 DEFAULT_CODEC.to_string(),
16485 None,
16486 )
16487 .expect("workflow state");
16488 state.workflow_command_identity = "command-7".to_string();
16489 let context = WorkflowContext {
16490 state: Arc::new(Mutex::new(state)),
16491 };
16492 let item =
16493 WorkflowStreamAppendItem::from_reference("s3://bucket/item.avro").item_type("receipt");
16494
16495 context
16496 .append_workflow_stream("output", &[item], Some(10))
16497 .expect("append command");
16498 context
16499 .error_workflow_stream("output", "producer failed", None)
16500 .expect("error command");
16501 let commands = context.take_commands().expect("commands");
16502
16503 assert_eq!(commands[0]["type"], "record_side_effect");
16504 assert_eq!(
16505 commands[0]["workflow_stream"]["command_identity"],
16506 "command-7"
16507 );
16508 assert_eq!(commands[0]["workflow_stream"]["command_ordinal"], 0);
16509 assert_eq!(
16510 commands[0]["workflow_stream"]["items"][0]["idempotency_key"],
16511 "dw-stream:command-7:0:0"
16512 );
16513 assert_eq!(commands[1]["workflow_stream"]["operation"], "error");
16514
16515 let recorded = history_event(
16516 "SideEffectRecorded",
16517 json!({"sequence": 1, "result": fixture_envelope(Value::Null)}),
16518 );
16519 let mut replay_state = WorkflowState::new(
16520 vec![recorded],
16521 "rust-workers".to_string(),
16522 DEFAULT_CODEC.to_string(),
16523 None,
16524 )
16525 .expect("replay state");
16526 replay_state.workflow_command_identity = "command-7".to_string();
16527 let replay_context = WorkflowContext {
16528 state: Arc::new(Mutex::new(replay_state)),
16529 };
16530 replay_context
16531 .append_workflow_stream(
16532 "output",
16533 &[WorkflowStreamAppendItem::from_reference(
16534 "s3://bucket/item.avro",
16535 )],
16536 Some(10),
16537 )
16538 .expect("replayed append");
16539 assert!(replay_context
16540 .take_commands()
16541 .expect("replayed commands")
16542 .is_empty());
16543 }
16544
16545 #[test]
16546 fn workflow_stream_authoring_requires_server_durable_command_identity() {
16547 let context = workflow_context(Vec::new());
16548 let error = context
16549 .append_workflow_stream(
16550 "output",
16551 &[WorkflowStreamAppendItem::from_reference(
16552 "s3://bucket/item.avro",
16553 )],
16554 None,
16555 )
16556 .expect_err("stream append without durable command identity must fail closed");
16557
16558 assert!(matches!(error, Error::MissingWorkflowCommandIdentity));
16559 assert!(context.take_commands().expect("commands").is_empty());
16560 }
16561
16562 #[test]
16563 fn cold_worker_replay_does_not_repeat_committed_side_effects_or_markers() {
16564 fn worker(calls: Arc<AtomicUsize>) -> Worker {
16565 let client = Client::new("http://127.0.0.1:8080").expect("client");
16566 let mut worker = Worker::new(client, "rust-workers");
16567 worker.register_workflow("rust.side-effect-version", move |ctx, _input| {
16568 let calls = Arc::clone(&calls);
16569 async move {
16570 let captured = ctx.side_effect(|| {
16571 calls.fetch_add(1, Ordering::SeqCst);
16572 "captured-once".to_string()
16573 })?;
16574 let version = ctx.get_version("cold-restart", 1, 2)?;
16575 Ok(json!({"captured": captured, "version": version}))
16576 }
16577 });
16578 worker
16579 }
16580
16581 fn task(history_events: Vec<HistoryEvent>) -> WorkflowTask {
16582 WorkflowTask {
16583 task_id: "wft-side-effect-version".to_string(),
16584 workflow_command_id: None,
16585 workflow_id: Some("wf-side-effect-version".to_string()),
16586 run_id: Some("run-side-effect-version".to_string()),
16587 workflow_type: "rust.side-effect-version".to_string(),
16588 cancel_requested: false,
16589 payload_codec: DEFAULT_CODEC.to_string(),
16590 arguments: Some(
16591 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("arguments"),
16592 ),
16593 history_events,
16594 total_history_events: None,
16595 history_size_bytes: None,
16596 continue_as_new_recommended: None,
16597 history_budget_pressure: None,
16598 next_history_page_token: None,
16599 workflow_task_attempt: 1,
16600 workflow_signal_id: None,
16601 signal_name: None,
16602 signal_arguments: None,
16603 workflow_update_id: None,
16604 update_name: None,
16605 lease_owner: Some("rust-worker".to_string()),
16606 }
16607 }
16608
16609 let calls = Arc::new(AtomicUsize::new(0));
16610 let initial = worker(Arc::clone(&calls))
16611 .execute_workflow_task(task(Vec::new()))
16612 .expect("initial execution");
16613 assert_eq!(
16614 initial
16615 .iter()
16616 .map(|command| &command["type"])
16617 .collect::<Vec<_>>(),
16618 vec![
16619 "record_side_effect",
16620 "record_version_marker",
16621 "complete_workflow"
16622 ]
16623 );
16624 assert_eq!(calls.load(Ordering::SeqCst), 1);
16625
16626 let restarted = worker(Arc::clone(&calls));
16627 let replayed = restarted
16628 .execute_workflow_task(task(vec![
16629 history_event(
16630 "SideEffectRecorded",
16631 json!({"sequence": 1, "result": initial[0]["result"].clone()}),
16632 ),
16633 history_event(
16634 "VersionMarkerRecorded",
16635 json!({
16636 "sequence": 2,
16637 "change_id": "cold-restart",
16638 "version": 2,
16639 "min_supported": 1,
16640 "max_supported": 2,
16641 }),
16642 ),
16643 ]))
16644 .expect("cold replay");
16645 assert_eq!(replayed.len(), 1);
16646 assert_eq!(replayed[0]["type"], "complete_workflow");
16647 assert_eq!(calls.load(Ordering::SeqCst), 1);
16648 }
16649
16650 #[test]
16651 fn side_effect_replay_rejects_changed_rust_value_type() {
16652 let result = encode_value_envelope(&json!({"value": 42}), DEFAULT_CODEC).expect("result");
16653 let ctx = workflow_context(vec![history_event(
16654 "SideEffectRecorded",
16655 json!({"sequence": 1, "result": result}),
16656 )]);
16657 let error = ctx
16658 .side_effect::<Vec<String>, _>(|| panic!("callback must not run"))
16659 .expect_err("changed type must fail replay");
16660 assert!(matches!(
16661 error,
16662 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16663 if reason == "side_effect_type_mismatch"
16664 ));
16665 }
16666
16667 fn completed_retry_activity_history() -> Vec<HistoryEvent> {
16668 vec![
16669 history_event(
16670 "ActivityScheduled",
16671 json!({
16672 "sequence": 1,
16673 "activity_type": "flaky",
16674 "activity_execution_id": "act-1",
16675 "activity": {
16676 "id": "act-1",
16677 "sequence": 1,
16678 "type": "flaky",
16679 "queue": "critical-activities",
16680 "execution_mode": null,
16681 "retry_policy": {
16682 "snapshot_version": 1,
16683 "max_attempts": 3,
16684 "backoff_seconds": [2, 4],
16685 "start_to_close_timeout": 30,
16686 "schedule_to_start_timeout": 5,
16687 "schedule_to_close_timeout": 90,
16688 "heartbeat_timeout": 10,
16689 "non_retryable_error_types": ["PermanentError"]
16690 }
16691 }
16692 }),
16693 ),
16694 history_event(
16695 "ActivityStarted",
16696 json!({
16697 "sequence": 1,
16698 "activity_type": "flaky",
16699 "activity_execution_id": "act-1",
16700 "activity_attempt_id": "attempt-1",
16701 "attempt_number": 1
16702 }),
16703 ),
16704 history_event(
16705 "ActivityRetryScheduled",
16706 json!({
16707 "sequence": 1,
16708 "activity_type": "flaky",
16709 "activity_execution_id": "act-1",
16710 "activity_attempt_id": "attempt-1",
16711 "attempt_number": 1,
16712 "retry_after_attempt": 1,
16713 "retry_backoff_seconds": 2,
16714 "failure_category": "activity",
16715 "exception_type": "TransientError"
16716 }),
16717 ),
16718 history_event(
16719 "ActivityStarted",
16720 json!({
16721 "sequence": 1,
16722 "activity_type": "flaky",
16723 "activity_execution_id": "act-1",
16724 "activity_attempt_id": "attempt-2",
16725 "attempt_number": 2
16726 }),
16727 ),
16728 history_event(
16729 "ActivityCompleted",
16730 json!({
16731 "sequence": 1,
16732 "activity_type": "flaky",
16733 "activity_execution_id": "act-1",
16734 "activity_attempt_id": "attempt-2",
16735 "attempt_number": 2,
16736 "payload_codec": DEFAULT_CODEC,
16737 "result": fixture_envelope(json!({"status":"recovered"}))
16738 }),
16739 ),
16740 ]
16741 }
16742
16743 fn retry_activity_options() -> ActivityOptions {
16744 ActivityOptions::new()
16745 .task_queue("critical-activities")
16746 .retry_policy(
16747 ActivityRetryPolicy::new(3)
16748 .backoff_intervals([Duration::from_secs(2), Duration::from_secs(4)])
16749 .non_retryable_error_type("PermanentError"),
16750 )
16751 .start_to_close_timeout(Duration::from_secs(30))
16752 .schedule_to_start_timeout(Duration::from_secs(5))
16753 .schedule_to_close_timeout(Duration::from_secs(90))
16754 .heartbeat_timeout(Duration::from_secs(10))
16755 }
16756
16757 #[test]
16758 fn fixed_avro_value_round_trips_json_values() {
16759 let value = json!({"greeting": "hello", "count": 3, "ok": true});
16760 let envelope = PayloadEnvelope::avro(&value).expect("encode");
16761 assert_eq!(envelope.codec, DEFAULT_CODEC);
16762 assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
16763 }
16764
16765 #[tokio::test]
16766 async fn typed_handler_adapters_round_trip_serde_contracts_on_the_fixed_wire() {
16767 let client = Client::new("http://127.0.0.1:8080").expect("client");
16768 let mut worker = Worker::new(client, "rust-workers");
16769 worker.register_typed_workflow(
16770 "typed.contract.workflow",
16771 |_ctx, input: TypedContract| async move { Ok(input) },
16772 );
16773 worker.register_typed_activity(
16774 "typed.contract.activity",
16775 |_ctx, input: TypedContract| async move { Ok(input) },
16776 );
16777
16778 let expected = typed_contract();
16779 let arguments = AvroValue::Array(vec![
16780 AvroValue::from_serialize(&expected).expect("typed request")
16781 ]);
16782 let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("arguments");
16783 let mut workflow = workflow_task("typed.contract.workflow", Vec::new(), DEFAULT_CODEC);
16784 workflow.arguments = Some(envelope.clone());
16785 let commands = worker
16786 .execute_workflow_task(workflow)
16787 .expect("typed workflow task");
16788 let workflow_result: TypedContract =
16789 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
16790 .expect("workflow result envelope")
16791 .deserialize()
16792 .expect("workflow result type");
16793 assert_eq!(workflow_result, expected);
16794
16795 let activity = ActivityTask {
16796 task_id: "typed-contract-activity".to_string(),
16797 activity_attempt_id: Some("typed-contract-attempt".to_string()),
16798 attempt_id: None,
16799 activity_type: "typed.contract.activity".to_string(),
16800 payload_codec: DEFAULT_CODEC.to_string(),
16801 arguments: Some(envelope),
16802 attempt_number: 1,
16803 lease_owner: Some("rust-worker".to_string()),
16804 };
16805 let activity_result: TypedContract = worker
16806 .execute_activity_task(activity)
16807 .await
16808 .expect("typed activity task")
16809 .deserialize()
16810 .expect("activity result type");
16811 assert_eq!(activity_result, expected);
16812 }
16813
16814 #[tokio::test]
16815 async fn typed_handler_errors_include_handler_name_direction_and_rust_type() {
16816 let client = Client::new("http://127.0.0.1:8080").expect("client");
16817 let mut worker = Worker::new(client, "rust-workers");
16818 worker.register_typed_workflow(
16819 "typed.shape.workflow",
16820 |_ctx, input: TypedContract| async move { Ok(input) },
16821 );
16822 worker.register_typed_activity("typed.unsupported.activity", |_ctx, (): ()| async move {
16823 Ok(f64::NAN)
16824 });
16825
16826 let mut workflow = workflow_task("typed.shape.workflow", Vec::new(), DEFAULT_CODEC);
16827 workflow.arguments = Some(
16828 encode_typed_envelope(
16829 &AvroValue::Array(vec![
16830 AvroValue::String("first".to_string()),
16831 AvroValue::String("second".to_string()),
16832 ]),
16833 DEFAULT_CODEC,
16834 )
16835 .expect("malformed typed arguments"),
16836 );
16837 let commands = worker
16838 .execute_workflow_task(workflow)
16839 .expect("shape mismatch becomes a workflow failure");
16840 let message = commands[0]["message"].as_str().expect("failure message");
16841 assert!(message.contains("workflow handler \"typed.shape.workflow\" input type"));
16842 assert!(message.contains(type_name::<TypedContract>()));
16843 assert!(message.contains("task carried 2 arguments"));
16844
16845 let activity = ActivityTask {
16846 task_id: "typed-unsupported-activity".to_string(),
16847 activity_attempt_id: Some("typed-unsupported-attempt".to_string()),
16848 attempt_id: None,
16849 activity_type: "typed.unsupported.activity".to_string(),
16850 payload_codec: DEFAULT_CODEC.to_string(),
16851 arguments: Some(
16852 encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
16853 .expect("unit arguments"),
16854 ),
16855 attempt_number: 1,
16856 lease_owner: Some("rust-worker".to_string()),
16857 };
16858 let Error::HandlerType {
16859 handler_kind,
16860 handler_name,
16861 value_kind,
16862 rust_type,
16863 message,
16864 } = worker
16865 .execute_activity_task(activity)
16866 .await
16867 .expect_err("non-finite handler output must fail")
16868 else {
16869 panic!("expected contextual handler type failure");
16870 };
16871 assert_eq!(handler_kind, HandlerKind::Activity);
16872 assert_eq!(handler_name, "typed.unsupported.activity");
16873 assert_eq!(value_kind, HandlerValueKind::Result);
16874 assert_eq!(rust_type, type_name::<f64>());
16875 assert!(message.contains("non_finite_float"));
16876 }
16877
16878 #[tokio::test]
16879 async fn typed_replayed_workflow_decodes_input_and_activity_result_losslessly() {
16880 #[derive(Clone, Default)]
16881 struct State {
16882 observed: Option<TypedContract>,
16883 }
16884
16885 let client = Client::new("http://127.0.0.1:8080").expect("client");
16886 let mut worker = Worker::new(client, "rust-workers");
16887 worker.register_typed_replayed_workflow(
16888 "typed.contract.replayed",
16889 State::default,
16890 |ctx, input: TypedContract, state| async move {
16891 let result: TypedContract =
16892 ctx.activity_typed("typed.contract.activity", input).await?;
16893 state.update(|current| current.observed = Some(result.clone()))?;
16894 Ok(result)
16895 },
16896 );
16897 worker.register_replayed_query::<State, _, _>(
16898 "typed.contract.replayed",
16899 "observed",
16900 |_ctx, state, _args| async move {
16901 Ok(json!(state.observed.as_ref().map(|value| value.signed)))
16902 },
16903 );
16904
16905 let expected = typed_contract();
16906 let typed_value = AvroValue::from_serialize(&expected).expect("typed value");
16907 let workflow_arguments =
16908 encode_typed_envelope(&AvroValue::Array(vec![typed_value.clone()]), DEFAULT_CODEC)
16909 .expect("workflow arguments");
16910 let result = encode_typed_envelope(&typed_value, DEFAULT_CODEC).expect("activity result");
16911 let task = QueryTask {
16912 query_task_id: "typed-replay-query".to_string(),
16913 query_task_attempt: 1,
16914 lease_owner: Some("rust-worker".to_string()),
16915 workflow_id: Some("typed-replay".to_string()),
16916 run_id: Some("typed-replay-run".to_string()),
16917 workflow_type: "typed.contract.replayed".to_string(),
16918 query_name: "observed".to_string(),
16919 payload_codec: DEFAULT_CODEC.to_string(),
16920 workflow_arguments: Some(workflow_arguments),
16921 query_arguments: Some(
16922 encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
16923 .expect("query arguments"),
16924 ),
16925 history_events: vec![
16926 history_event(
16927 "ActivityScheduled",
16928 json!({
16929 "sequence": 1,
16930 "activity_type": "typed.contract.activity"
16931 }),
16932 ),
16933 history_event(
16934 "ActivityCompleted",
16935 json!({
16936 "sequence": 1,
16937 "activity_type": "typed.contract.activity",
16938 "payload_codec": DEFAULT_CODEC,
16939 "result": result
16940 }),
16941 ),
16942 ],
16943 history_export: None,
16944 run_status: Some("completed".to_string()),
16945 };
16946
16947 assert_eq!(
16948 worker
16949 .execute_query_task(task)
16950 .await
16951 .expect("typed replay query")
16952 .deserialize::<i64>()
16953 .expect("query result"),
16954 expected.signed
16955 );
16956 }
16957
16958 #[tokio::test]
16959 async fn typed_worker_surfaces_preserve_bytes_and_map_list_identity() {
16960 let client = Client::new("http://127.0.0.1:8080").expect("client");
16961 let mut worker = Worker::new(client, "rust-workers");
16962 worker.register_workflow_avro_value("typed.echo", |_ctx, input| async move { Ok(input) });
16963 worker
16964 .register_activity_avro_value("typed.activity", |_ctx, input| async move { Ok(input) });
16965 worker.register_query_avro_value("typed.echo", "inspect", |_ctx, input| async move {
16966 Ok(input)
16967 });
16968 worker.register_update_avro_value("typed.echo", "replace", |_ctx, input| async move {
16969 Ok(input)
16970 });
16971 worker.register_workflow_avro_value("typed.signal", |ctx, _input| async move {
16972 Ok(AvroValue::Array(
16973 ctx.wait_signal_avro_value("changed").await?,
16974 ))
16975 });
16976
16977 let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
16978 let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed envelope");
16979
16980 let mut workflow = workflow_task("typed.echo", Vec::new(), DEFAULT_CODEC);
16981 workflow.arguments = Some(envelope.clone());
16982 let commands = worker
16983 .execute_workflow_task(workflow)
16984 .expect("typed workflow task");
16985 assert_eq!(commands[0]["type"], "complete_workflow");
16986 assert_eq!(
16987 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
16988 .expect("typed workflow result"),
16989 arguments
16990 );
16991
16992 let activity = ActivityTask {
16993 task_id: "activity-typed".to_string(),
16994 activity_attempt_id: Some("attempt-typed".to_string()),
16995 attempt_id: None,
16996 activity_type: "typed.activity".to_string(),
16997 payload_codec: DEFAULT_CODEC.to_string(),
16998 arguments: Some(envelope.clone()),
16999 attempt_number: 1,
17000 lease_owner: Some("rust-worker".to_string()),
17001 };
17002 assert_eq!(
17003 worker
17004 .execute_activity_task(activity)
17005 .await
17006 .expect("typed activity result"),
17007 arguments
17008 );
17009
17010 let query = QueryTask {
17011 query_task_id: "query-typed".to_string(),
17012 query_task_attempt: 1,
17013 lease_owner: Some("rust-worker".to_string()),
17014 workflow_id: Some("typed-1".to_string()),
17015 run_id: Some("run-typed".to_string()),
17016 workflow_type: "typed.echo".to_string(),
17017 query_name: "inspect".to_string(),
17018 payload_codec: DEFAULT_CODEC.to_string(),
17019 workflow_arguments: Some(
17020 encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
17021 .expect("workflow input"),
17022 ),
17023 query_arguments: Some(envelope.clone()),
17024 history_events: Vec::new(),
17025 history_export: None,
17026 run_status: Some("running".to_string()),
17027 };
17028 assert_eq!(
17029 worker
17030 .execute_query_task(query)
17031 .await
17032 .expect("typed query result"),
17033 arguments
17034 );
17035
17036 let mut update = workflow_task(
17037 "typed.echo",
17038 vec![history_event(
17039 "UpdateAccepted",
17040 json!({
17041 "update_id": "update-typed",
17042 "update_name": "replace",
17043 "arguments": envelope.clone(),
17044 }),
17045 )],
17046 DEFAULT_CODEC,
17047 );
17048 update.workflow_update_id = Some("update-typed".to_string());
17049 update.update_name = Some("replace".to_string());
17050 let commands = worker
17051 .execute_workflow_task(update)
17052 .expect("typed update task");
17053 assert_eq!(commands[0]["type"], "complete_update");
17054 assert_eq!(
17055 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17056 .expect("typed update result"),
17057 arguments
17058 );
17059
17060 let mut signal = workflow_task(
17061 "typed.signal",
17062 vec![history_event(
17063 "SignalReceived",
17064 json!({
17065 "signal_id": "signal-typed",
17066 "signal_name": "changed",
17067 "arguments": envelope.clone(),
17068 }),
17069 )],
17070 DEFAULT_CODEC,
17071 );
17072 signal.workflow_signal_id = Some("signal-typed".to_string());
17073 signal.signal_name = Some("changed".to_string());
17074 signal.signal_arguments = Some(envelope);
17075 let commands = worker
17076 .execute_workflow_task(signal)
17077 .expect("typed signal resume");
17078 assert_eq!(
17079 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17080 .expect("typed signal result"),
17081 arguments
17082 );
17083 }
17084
17085 #[tokio::test]
17086 async fn typed_helpers_never_parse_json_inspection_projection() {
17087 let collision_values = projection_collision_probe();
17088 let expected = AvroValue::Array(collision_values.clone());
17089 let envelope = encode_typed_envelope(&expected, DEFAULT_CODEC).expect("collision envelope");
17090
17091 let activity_context = workflow_context_with_codec(
17092 vec![history_event(
17093 "ActivityCompleted",
17094 json!({
17095 "sequence": 1,
17096 "activity_type": "collision.activity",
17097 "payload_codec": DEFAULT_CODEC,
17098 "result": envelope.clone(),
17099 }),
17100 )],
17101 DEFAULT_CODEC,
17102 );
17103 assert_eq!(
17104 activity_context
17105 .activity_avro_value("collision.activity", AvroValue::Array(Vec::new()))
17106 .await
17107 .expect("typed activity collision result"),
17108 expected
17109 );
17110
17111 let signal_context = workflow_context_with_codec(
17112 vec![
17113 history_event(
17114 "SignalWaitOpened",
17115 json!({"sequence": 1, "signal_name": "collision"}),
17116 ),
17117 history_event(
17118 "SignalApplied",
17119 json!({
17120 "sequence": 1,
17121 "signal_name": "collision",
17122 "payload_codec": DEFAULT_CODEC,
17123 "value": envelope.clone(),
17124 }),
17125 ),
17126 ],
17127 DEFAULT_CODEC,
17128 );
17129 assert_eq!(
17130 signal_context
17131 .wait_signal_avro_value("collision")
17132 .await
17133 .expect("typed signal collision arguments"),
17134 collision_values
17135 );
17136
17137 let child_context = workflow_context_with_codec(
17138 vec![
17139 history_event(
17140 "ChildWorkflowScheduled",
17141 json!({
17142 "sequence": 1,
17143 "child_workflow_instance_id": "collision-child",
17144 "child_workflow_run_id": "collision-run",
17145 "child_workflow_type": "collision.child",
17146 }),
17147 ),
17148 history_event(
17149 "ChildRunCompleted",
17150 json!({
17151 "sequence": 1,
17152 "child_workflow_instance_id": "collision-child",
17153 "child_workflow_run_id": "collision-run",
17154 "child_workflow_type": "collision.child",
17155 "payload_codec": DEFAULT_CODEC,
17156 "result": envelope,
17157 }),
17158 ),
17159 ],
17160 DEFAULT_CODEC,
17161 );
17162 let child = child_context
17163 .start_child_workflow_avro_value(
17164 "collision.child",
17165 ChildWorkflowOptions::new("collision-workers"),
17166 AvroValue::Array(Vec::new()),
17167 )
17168 .await
17169 .expect("typed child collision result");
17170 assert_eq!(child.result, expected);
17171 }
17172
17173 #[tokio::test]
17174 async fn replayed_typed_query_keeps_lossless_workflow_and_query_inputs() {
17175 let client = Client::new("http://127.0.0.1:8080").expect("client");
17176 let mut worker = Worker::new(client, "rust-workers");
17177 worker.register_replayed_workflow_avro_value(
17178 "typed.replayed",
17179 || (),
17180 |_ctx, input, _state| async move { Ok(input) },
17181 );
17182 worker.register_replayed_query_avro_value::<(), _, _>(
17183 "typed.replayed",
17184 "inspect",
17185 |ctx, _state, args| async move {
17186 let mut signals = ctx.signals_avro_value("collision");
17187 let signal = signals
17188 .pop()
17189 .map(AvroValue::Array)
17190 .unwrap_or_else(|| AvroValue::Array(Vec::new()));
17191 Ok(AvroValue::Array(vec![
17192 ctx.workflow_input_avro_value().clone(),
17193 signal,
17194 args,
17195 ]))
17196 },
17197 );
17198 let arguments = AvroValue::Array(projection_collision_probe());
17199 let signal_arguments =
17200 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed query signal arguments");
17201 let task = QueryTask {
17202 query_task_id: "query-typed-replay".to_string(),
17203 query_task_attempt: 1,
17204 lease_owner: Some("rust-worker".to_string()),
17205 workflow_id: Some("typed-replay".to_string()),
17206 run_id: Some("run-typed-replay".to_string()),
17207 workflow_type: "typed.replayed".to_string(),
17208 query_name: "inspect".to_string(),
17209 payload_codec: DEFAULT_CODEC.to_string(),
17210 workflow_arguments: Some(
17211 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("workflow arguments"),
17212 ),
17213 query_arguments: Some(
17214 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("query arguments"),
17215 ),
17216 history_events: vec![history_event(
17217 "SignalReceived",
17218 json!({
17219 "signal_id": "collision-signal",
17220 "signal_name": "collision",
17221 "workflow_sequence": 1,
17222 "payload_codec": DEFAULT_CODEC,
17223 "arguments": signal_arguments,
17224 }),
17225 )],
17226 history_export: None,
17227 run_status: Some("completed".to_string()),
17228 };
17229
17230 assert_eq!(
17231 worker
17232 .execute_query_task(task)
17233 .await
17234 .expect("typed replay query"),
17235 AvroValue::Array(vec![arguments.clone(), arguments.clone(), arguments])
17236 );
17237 }
17238
17239 #[test]
17240 fn public_avro_adapter_rejects_non_string_map_keys_before_json_conversion() {
17241 let value = BTreeMap::from([(1_i32, "integer key")]);
17242 let error = PayloadEnvelope::avro(&value)
17243 .expect_err("integer map keys must fail")
17244 .to_string();
17245
17246 assert!(error.contains("invalid_map_key"));
17247 }
17248
17249 #[test]
17250 fn json_tagged_payload_fails_closed_with_actionable_diagnostic() {
17251 let envelope = PayloadEnvelope {
17252 codec: "json".to_string(),
17253 blob: r#"{"greeting":"hello"}"#.to_string(),
17254 };
17255
17256 let error = decode_payload::<Value>(&envelope).expect_err("JSON payload must fail");
17257 let diagnostic = error.to_string();
17258 assert!(diagnostic.contains("unsupported_payload_codec"));
17259 assert!(diagnostic.contains("codec=\"avro\""));
17260 assert!(diagnostic.contains("HTTP document transport"));
17261 }
17262
17263 #[test]
17264 fn untagged_json_payload_value_fails_closed() {
17265 let error = decode_wire_value(&json!({"stale": true}), DEFAULT_CODEC)
17266 .expect_err("untagged JSON payload values must fail");
17267 let diagnostic = error.to_string();
17268 assert!(diagnostic.contains("unsupported_payload_codec"));
17269 assert!(diagnostic.contains("untagged durable payload"));
17270 assert!(diagnostic.contains("HTTP document transport"));
17271 }
17272
17273 #[test]
17274 fn prerelease_avro_payload_without_single_object_frame_is_rejected() {
17275 let envelope = PayloadEnvelope {
17276 codec: DEFAULT_CODEC.to_string(),
17277 blob: BASE64.encode([0x01]),
17278 };
17279
17280 let error = decode_payload::<Value>(&envelope).expect_err("prerelease payload must fail");
17281 assert!(error.to_string().contains("invalid_payload_framing"));
17282 }
17283
17284 #[tokio::test]
17285 async fn workflow_completion_rejects_invalid_payload_slots_without_transport() {
17286 let server = MockWorkerServer::start();
17287 let client = Client::builder(server.base_url())
17288 .timeout(Duration::from_secs(2))
17289 .build()
17290 .expect("client");
17291 let invalid_commands = [
17292 json!({
17293 "type": "complete_workflow",
17294 "result": {"codec": "json", "blob": null}
17295 }),
17296 json!({
17297 "type": "schedule_activity",
17298 "arguments": {"codec": "yaml", "blob": "ignored"}
17299 }),
17300 json!({
17301 "type": "start_child_workflow",
17302 "arguments": {"codec": DEFAULT_CODEC, "blob": null}
17303 }),
17304 json!({"type": "continue_as_new", "arguments": []}),
17305 json!({"type": "complete_update"}),
17306 json!({"type": "record_side_effect", "result": null}),
17307 json!({
17308 "type": "start_service_operation",
17309 "payload_codec": DEFAULT_CODEC,
17310 "request_payload": "raw-avro-bytes"
17311 }),
17312 ];
17313
17314 for command in invalid_commands {
17315 let error = client
17316 .complete_workflow_task("invalid-codec", "rust-worker", 1, vec![command])
17317 .await
17318 .expect_err("invalid durable payload must fail locally");
17319 let diagnostic = error.to_string();
17320 assert!(
17321 diagnostic.contains("unsupported_payload_codec")
17322 || diagnostic.contains("invalid_payload_envelope")
17323 || diagnostic.contains("untagged durable payload"),
17324 "unexpected validation diagnostic: {diagnostic}"
17325 );
17326 }
17327
17328 assert_eq!(
17329 server.request_count("/api/worker/workflow-tasks/invalid-codec/complete"),
17330 0,
17331 "invalid command payloads must not reach HTTP transport"
17332 );
17333 }
17334
17335 #[test]
17336 fn workflow_completion_validates_only_protocol_owned_payload_slots() {
17337 let envelope = fixture_envelope(json!({"codec": "customer-value"}));
17338 let commands = [
17339 json!({"type": "complete_workflow", "result": envelope.clone()}),
17340 json!({"type": "schedule_activity", "arguments": envelope.clone()}),
17341 json!({"type": "start_child_workflow", "arguments": envelope.clone()}),
17342 json!({"type": "continue_as_new", "arguments": envelope.clone()}),
17343 json!({"type": "complete_update", "result": envelope.clone()}),
17344 json!({"type": "record_side_effect", "result": envelope.clone()}),
17345 json!({
17346 "type": "start_service_operation",
17347 "payload_codec": DEFAULT_CODEC,
17348 "request_payload": envelope.clone()
17349 }),
17350 json!({
17351 "type": "complete_workflow",
17352 "result": envelope,
17353 "metadata": {
17354 "codec": "json",
17355 "payload_codec": "customer-codec",
17356 "result": {"codec": "yaml", "blob": null}
17357 }
17358 }),
17359 ];
17360
17361 validate_workflow_task_commands(&commands)
17362 .expect("customer metadata must not become a protocol codec declaration");
17363 }
17364
17365 #[test]
17366 fn valid_avro_tasks_normalize_absent_and_null_arguments_to_empty_lists() {
17367 assert_eq!(
17368 decode_task_avro_arguments(None, DEFAULT_CODEC).expect("absent arguments"),
17369 AvroValue::Array(Vec::new())
17370 );
17371 assert_eq!(
17372 decode_task_avro_arguments(Some(&Value::Null), DEFAULT_CODEC).expect("null arguments"),
17373 AvroValue::Array(Vec::new())
17374 );
17375
17376 let mut signal = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
17377 signal.signal_name = Some("empty-signal".to_string());
17378 signal.signal_arguments = None;
17379 let decoded = decode_resume_signal(&signal)
17380 .expect("valid Avro signal")
17381 .expect("named signal resumes the workflow");
17382 assert!(decoded.arguments.is_empty());
17383 }
17384
17385 #[tokio::test]
17386 async fn malformed_task_level_codecs_become_pre_handler_failures() {
17387 let client = Client::new("http://127.0.0.1:8080").expect("client");
17388 let mut worker = Worker::new(client, "rust-workers");
17389 let handler_calls = Arc::new(AtomicUsize::new(0));
17390
17391 let calls = Arc::clone(&handler_calls);
17392 worker.register_workflow("codec.workflow", move |_ctx, _args| {
17393 calls.fetch_add(1, Ordering::SeqCst);
17394 async move { Ok(Value::Null) }
17395 });
17396 let calls = Arc::clone(&handler_calls);
17397 worker.register_activity("codec.activity", move |_ctx, _args| {
17398 calls.fetch_add(1, Ordering::SeqCst);
17399 async move { Ok(Value::Null) }
17400 });
17401 let calls = Arc::clone(&handler_calls);
17402 worker.register_query("codec.workflow", "known", move |_ctx, _args| {
17403 calls.fetch_add(1, Ordering::SeqCst);
17404 async move { Ok(Value::Null) }
17405 });
17406
17407 let mut failures = Vec::new();
17408 for codec_case in [
17409 InvalidTaskPayloadCodec::Missing,
17410 InvalidTaskPayloadCodec::Null,
17411 InvalidTaskPayloadCodec::NonString,
17412 ] {
17413 let mut workflow = json!({
17414 "task_id": format!("workflow-{}", codec_case.label()),
17415 "workflow_type": "codec.workflow"
17416 });
17417 codec_case.apply(&mut workflow);
17418 match serde_json::from_value::<WorkflowTask>(workflow) {
17419 Ok(task) => match worker.execute_workflow_task(task) {
17420 Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
17421 outcome => failures.push(format!(
17422 "workflow {} codec returned {outcome:?}",
17423 codec_case.label()
17424 )),
17425 },
17426 Err(error) => failures.push(format!(
17427 "workflow {} codec failed transport deserialization: {error}",
17428 codec_case.label()
17429 )),
17430 }
17431
17432 let mut activity = json!({
17433 "task_id": format!("activity-{}", codec_case.label()),
17434 "activity_attempt_id": format!("attempt-{}", codec_case.label()),
17435 "activity_type": "codec.activity",
17436 "attempt_number": 1
17437 });
17438 codec_case.apply(&mut activity);
17439 match serde_json::from_value::<ActivityTask>(activity) {
17440 Ok(task) => match worker.execute_activity_task(task).await {
17441 Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
17442 outcome => failures.push(format!(
17443 "activity {} codec returned {outcome:?}",
17444 codec_case.label()
17445 )),
17446 },
17447 Err(error) => failures.push(format!(
17448 "activity {} codec failed transport deserialization: {error}",
17449 codec_case.label()
17450 )),
17451 }
17452
17453 let mut query = json!({
17454 "query_task_id": format!("query-{}", codec_case.label()),
17455 "workflow_type": "codec.workflow",
17456 "query_name": "known"
17457 });
17458 codec_case.apply(&mut query);
17459 match serde_json::from_value::<QueryTask>(query) {
17460 Ok(task) => match worker.execute_query_task(task).await {
17461 Err(failure) if failure.message.contains("unsupported_payload_codec") => {}
17462 outcome => failures.push(format!(
17463 "query {} codec returned {outcome:?}",
17464 codec_case.label()
17465 )),
17466 },
17467 Err(error) => failures.push(format!(
17468 "query {} codec failed transport deserialization: {error}",
17469 codec_case.label()
17470 )),
17471 }
17472 }
17473
17474 assert!(failures.is_empty(), "{}", failures.join("\n"));
17475 assert_eq!(
17476 handler_calls.load(Ordering::SeqCst),
17477 0,
17478 "invalid task codecs must not invoke a handler"
17479 );
17480 }
17481
17482 #[tokio::test]
17483 async fn polled_malformed_task_codecs_are_settled_without_handler_execution() {
17484 for codec_case in [
17485 InvalidTaskPayloadCodec::Missing,
17486 InvalidTaskPayloadCodec::Null,
17487 InvalidTaskPayloadCodec::NonString,
17488 ] {
17489 let server = MockWorkerServer::invalid_task_payload_codec(codec_case);
17490 let client = Client::builder(server.base_url())
17491 .timeout(Duration::from_secs(2))
17492 .build()
17493 .expect("client");
17494 let mut worker = Worker::new(client, "rust-workers")
17495 .worker_id("codec-worker")
17496 .poll_timeout(Duration::from_millis(10));
17497 let handler_calls = Arc::new(AtomicUsize::new(0));
17498
17499 let calls = Arc::clone(&handler_calls);
17500 worker.register_workflow("codec.workflow", move |_ctx, _args| {
17501 calls.fetch_add(1, Ordering::SeqCst);
17502 async move { Ok(Value::Null) }
17503 });
17504 let calls = Arc::clone(&handler_calls);
17505 worker.register_activity("codec.activity", move |_ctx, _args| {
17506 calls.fetch_add(1, Ordering::SeqCst);
17507 async move { Ok(Value::Null) }
17508 });
17509 let calls = Arc::clone(&handler_calls);
17510 worker.register_query("codec.workflow", "known", move |_ctx, _args| {
17511 calls.fetch_add(1, Ordering::SeqCst);
17512 async move { Ok(Value::Null) }
17513 });
17514
17515 assert_eq!(
17516 worker.run_once().await.expect("invalid tasks are settled"),
17517 3,
17518 "all {} codec tasks must be handled",
17519 codec_case.label()
17520 );
17521 assert_eq!(
17522 handler_calls.load(Ordering::SeqCst),
17523 0,
17524 "{} task codecs must fail before every handler",
17525 codec_case.label()
17526 );
17527
17528 for path in [
17529 "/api/worker/workflow-tasks/codec-workflow/fail",
17530 "/api/worker/activity-tasks/codec-activity/fail",
17531 "/api/worker/query-tasks/codec-query/fail",
17532 ] {
17533 let body = server.request_body(path);
17534 assert!(
17535 body["failure"]["message"]
17536 .as_str()
17537 .is_some_and(|message| message.contains("unsupported_payload_codec")),
17538 "{path} must receive the stable codec diagnostic for the {} case: {body}",
17539 codec_case.label()
17540 );
17541 }
17542 assert_eq!(
17543 server.request_body("/api/worker/query-tasks/codec-query/fail")["failure"]
17544 ["reason"],
17545 "query_payload_decode_failed"
17546 );
17547 for path in [
17548 "/api/worker/workflow-tasks/codec-workflow/complete",
17549 "/api/worker/activity-tasks/codec-activity/complete",
17550 "/api/worker/query-tasks/codec-query/complete",
17551 ] {
17552 assert_eq!(
17553 server.request_count(path),
17554 0,
17555 "invalid {} codec task reached {path}",
17556 codec_case.label()
17557 );
17558 }
17559 }
17560 }
17561
17562 #[tokio::test]
17563 async fn invalid_inbound_codecs_precede_handlers_and_unrelated_outcomes() {
17564 let client = Client::new("http://127.0.0.1:8080").expect("client");
17565 let mut worker = Worker::new(client, "rust-workers");
17566 let handler_calls = Arc::new(AtomicUsize::new(0));
17567
17568 let calls = Arc::clone(&handler_calls);
17569 worker.register_workflow("codec.workflow", move |_ctx, _args| {
17570 calls.fetch_add(1, Ordering::SeqCst);
17571 async move { Ok(Value::Null) }
17572 });
17573 let calls = Arc::clone(&handler_calls);
17574 worker.register_activity("codec.activity", move |_ctx, _args| {
17575 calls.fetch_add(1, Ordering::SeqCst);
17576 async move { Ok(Value::Null) }
17577 });
17578 let calls = Arc::clone(&handler_calls);
17579 worker.register_update("codec.workflow", "known", move |_ctx, _args| {
17580 calls.fetch_add(1, Ordering::SeqCst);
17581 async move { Ok(Value::Null) }
17582 });
17583 let calls = Arc::clone(&handler_calls);
17584 worker.register_query("codec.workflow", "known", move |_ctx, _args| {
17585 calls.fetch_add(1, Ordering::SeqCst);
17586 async move { Ok(Value::Null) }
17587 });
17588
17589 let mut workflow = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
17590 workflow.payload_codec = "json".to_string();
17591 workflow.arguments = None;
17592 let error = worker
17593 .execute_workflow_task(workflow)
17594 .expect_err("task codec must be checked before workflow invocation");
17595 assert!(error.to_string().contains("unsupported_payload_codec"));
17596
17597 let activity = ActivityTask {
17598 task_id: "activity-invalid-codec".to_string(),
17599 activity_attempt_id: None,
17600 attempt_id: None,
17601 activity_type: "codec.activity".to_string(),
17602 payload_codec: "unknown".to_string(),
17603 arguments: None,
17604 attempt_number: 1,
17605 lease_owner: None,
17606 };
17607 let error = worker
17608 .execute_activity_task(activity)
17609 .await
17610 .expect_err("task codec must be checked before activity invocation");
17611 assert!(error.to_string().contains("unsupported_payload_codec"));
17612
17613 let mut update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
17614 update.workflow_update_id = Some("update-invalid-codec".to_string());
17615 update.update_name = Some("known".to_string());
17616 update.history_events.push(history_event(
17617 "UpdateAccepted",
17618 json!({
17619 "update_id": "update-invalid-codec",
17620 "update_name": "known",
17621 "arguments": {"codec": "json", "blob": null}
17622 }),
17623 ));
17624 let error = worker
17625 .execute_workflow_task(update)
17626 .expect_err("nested update codec must be checked before handler lookup");
17627 assert!(error.to_string().contains("unsupported_payload_codec"));
17628
17629 let query: QueryTask = serde_json::from_value(json!({
17630 "query_task_id": "query-invalid-codec",
17631 "workflow_type": "codec.workflow",
17632 "query_name": "known",
17633 "payload_codec": DEFAULT_CODEC,
17634 "workflow_arguments": null,
17635 "query_arguments": null,
17636 "history_export": {
17637 "payloads": {"codec": DEFAULT_CODEC},
17638 "signals": [{
17639 "name": "empty",
17640 "payload_codec": "json",
17641 "arguments": null
17642 }]
17643 }
17644 }))
17645 .expect("query task");
17646 let failure = worker
17647 .execute_query_task(query)
17648 .await
17649 .expect_err("exported signal codec must be checked before query invocation");
17650 assert_eq!(failure.reason, "query_payload_decode_failed");
17651 assert!(failure.message.contains("unsupported_payload_codec"));
17652
17653 let exported_history: QueryTask = serde_json::from_value(json!({
17654 "query_task_id": "query-invalid-history-codec",
17655 "workflow_type": "codec.workflow",
17656 "query_name": "known",
17657 "payload_codec": DEFAULT_CODEC,
17658 "history_export": {
17659 "payloads": {"codec": DEFAULT_CODEC},
17660 "history_events": [{
17661 "type": "ActivityCompleted",
17662 "payload": {"payload_codec": "unknown", "result": null}
17663 }]
17664 }
17665 }))
17666 .expect("query task");
17667 let failure = worker
17668 .execute_query_task(exported_history)
17669 .await
17670 .expect_err("exported history codec must be checked before query invocation");
17671 assert_eq!(failure.reason, "query_payload_decode_failed");
17672 assert!(failure.message.contains("unsupported_payload_codec"));
17673 assert_eq!(handler_calls.load(Ordering::SeqCst), 0);
17674
17675 let mut unknown_workflow = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
17676 unknown_workflow.arguments = None;
17677 unknown_workflow.history_events.push(history_event(
17678 "SignalReceived",
17679 json!({
17680 "signal_name": "empty",
17681 "payload_codec": "json",
17682 "arguments": null
17683 }),
17684 ));
17685 let error = worker
17686 .execute_workflow_task(unknown_workflow)
17687 .expect_err("history codec must precede unknown workflow outcome");
17688 assert!(error.to_string().contains("unsupported_payload_codec"));
17689
17690 let unknown_activity = ActivityTask {
17691 task_id: "activity-unknown".to_string(),
17692 activity_attempt_id: None,
17693 attempt_id: None,
17694 activity_type: "missing".to_string(),
17695 payload_codec: "json".to_string(),
17696 arguments: None,
17697 attempt_number: 1,
17698 lease_owner: None,
17699 };
17700 let error = worker
17701 .execute_activity_task(unknown_activity)
17702 .await
17703 .expect_err("codec must precede unknown activity outcome");
17704 assert!(error.to_string().contains("unsupported_payload_codec"));
17705
17706 let mut unknown_update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
17707 unknown_update.payload_codec = "json".to_string();
17708 unknown_update.arguments = None;
17709 unknown_update.workflow_update_id = Some("update-unknown".to_string());
17710 unknown_update.update_name = Some("missing".to_string());
17711 let error = worker
17712 .execute_workflow_task(unknown_update)
17713 .expect_err("codec must precede fail_update shortcut");
17714 assert!(error.to_string().contains("unsupported_payload_codec"));
17715
17716 let unknown_query: QueryTask = serde_json::from_value(json!({
17717 "query_task_id": "query-unknown",
17718 "workflow_type": "missing",
17719 "query_name": "missing",
17720 "payload_codec": "json",
17721 "workflow_arguments": null,
17722 "query_arguments": null
17723 }))
17724 .expect("query task");
17725 let failure = worker
17726 .execute_query_task(unknown_query)
17727 .await
17728 .expect_err("codec must precede unknown query outcome");
17729 assert_eq!(failure.reason, "query_payload_decode_failed");
17730 assert!(failure.message.contains("unsupported_payload_codec"));
17731 }
17732
17733 #[tokio::test]
17734 async fn invalid_signal_history_payload_aliases_precede_shortcuts() {
17735 let client = Client::new("http://127.0.0.1:8080").expect("client");
17736 let worker = Worker::new(client, "rust-workers");
17737
17738 for event_type in ["SignalReceived", "SignalApplied"] {
17739 for (payload_field, codec) in [
17740 ("value", "json"),
17741 ("input", "unknown"),
17742 ("arguments", "json"),
17743 ] {
17744 let payload = json!({
17745 "signal_name": "empty",
17746 payload_field: {"codec": codec, "blob": null}
17747 });
17748 let workflow = workflow_task(
17749 "missing",
17750 vec![history_event(event_type, payload.clone())],
17751 DEFAULT_CODEC,
17752 );
17753 let error = worker
17754 .execute_workflow_task(workflow)
17755 .expect_err("signal payload codec must precede unknown workflow outcome");
17756 assert!(
17757 error.to_string().contains("unsupported_payload_codec"),
17758 "{event_type}.{payload_field} returned an unrelated workflow error: {error}"
17759 );
17760
17761 let query: QueryTask = serde_json::from_value(json!({
17762 "query_task_id": format!("query-{event_type}-{payload_field}"),
17763 "workflow_type": "missing",
17764 "query_name": "missing",
17765 "payload_codec": DEFAULT_CODEC,
17766 "workflow_arguments": null,
17767 "query_arguments": null,
17768 "history_events": [{
17769 "event_type": event_type,
17770 "payload": payload
17771 }]
17772 }))
17773 .expect("query task");
17774 let failure = worker
17775 .execute_query_task(query)
17776 .await
17777 .expect_err("signal payload codec must precede unknown query outcome");
17778 assert_eq!(
17779 failure.reason, "query_payload_decode_failed",
17780 "{event_type}.{payload_field} returned an unrelated query outcome"
17781 );
17782 assert!(
17783 failure.message.contains("unsupported_payload_codec"),
17784 "{event_type}.{payload_field} returned an unrelated query error: {}",
17785 failure.message
17786 );
17787 }
17788 }
17789 }
17790
17791 #[test]
17792 fn workflow_context_schedules_activity_until_completion_is_in_history() {
17793 let ctx = WorkflowContext {
17794 state: Arc::new(Mutex::new(
17795 WorkflowState::new_with_identity(
17796 Vec::new(),
17797 Some("wf-parent".to_string()),
17798 Some("run-parent".to_string()),
17799 "rust-workers".to_string(),
17800 DEFAULT_CODEC.to_string(),
17801 None,
17802 )
17803 .expect("workflow state"),
17804 )),
17805 };
17806
17807 let mut call = Box::pin(ctx.activity("hello.activity", json!(["Ada"])));
17808 let mut task_context = TaskContext::from_waker(noop_waker_ref());
17809 assert!(matches!(
17810 call.as_mut().poll(&mut task_context),
17811 Poll::Pending
17812 ));
17813
17814 let commands = ctx.take_commands().expect("commands");
17815 assert_eq!(commands[0]["type"], "schedule_activity");
17816 assert_eq!(commands[0]["activity_type"], "hello.activity");
17817 }
17818
17819 #[test]
17820 fn activity_options_encode_retry_policy_queue_and_every_timeout() {
17821 let ctx = workflow_context(Vec::new());
17822 let options = ActivityOptions::new()
17823 .task_queue("payments")
17824 .retry_policy(
17825 ActivityRetryPolicy::new(4)
17826 .exponential_backoff(Duration::from_secs(1), 3, Some(Duration::from_secs(10)))
17827 .non_retryable_error_type("ValidationError"),
17828 )
17829 .start_to_close_timeout(Duration::from_secs(120))
17830 .schedule_to_start_timeout(Duration::from_secs(10))
17831 .schedule_to_close_timeout(Duration::from_secs(300))
17832 .heartbeat_timeout(Duration::from_secs(15));
17833 let mut call = Box::pin(ctx.activity_with_options(
17834 "charge-card",
17835 options,
17836 json!([{"order_id": "o-1"}]),
17837 ));
17838 let mut task_context = TaskContext::from_waker(noop_waker_ref());
17839
17840 assert!(matches!(
17841 call.as_mut().poll(&mut task_context),
17842 Poll::Pending
17843 ));
17844 assert!(matches!(
17845 call.as_mut().poll(&mut task_context),
17846 Poll::Pending
17847 ));
17848
17849 let commands = ctx.take_commands().expect("activity command");
17850 assert_eq!(commands.len(), 1, "one future emits one logical schedule");
17851 assert_eq!(commands[0]["queue"], "payments");
17852 assert_eq!(
17853 commands[0]["retry_policy"],
17854 json!({
17855 "max_attempts": 4,
17856 "backoff_seconds": [1, 3, 9],
17857 "non_retryable_error_types": ["ValidationError"],
17858 })
17859 );
17860 assert_eq!(commands[0]["start_to_close_timeout"], 120);
17861 assert_eq!(commands[0]["schedule_to_start_timeout"], 10);
17862 assert_eq!(commands[0]["schedule_to_close_timeout"], 300);
17863 assert_eq!(commands[0]["heartbeat_timeout"], 15);
17864 }
17865
17866 #[test]
17867 fn activity_options_encode_explicit_and_rounded_backoff_intervals() {
17868 let ctx = workflow_context(Vec::new());
17869 let options = ActivityOptions::new().retry_policy(
17870 ActivityRetryPolicy::new(3)
17871 .backoff_intervals([Duration::from_millis(1), Duration::from_millis(1_001)]),
17872 );
17873 let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
17874 let mut task_context = TaskContext::from_waker(noop_waker_ref());
17875
17876 assert!(matches!(
17877 call.as_mut().poll(&mut task_context),
17878 Poll::Pending
17879 ));
17880 assert_eq!(
17881 ctx.take_commands().expect("command")[0]["retry_policy"]["backoff_seconds"],
17882 json!([1, 2])
17883 );
17884 }
17885
17886 #[test]
17887 fn invalid_activity_options_return_typed_errors_before_emitting_commands() {
17888 let cases = [
17889 (
17890 ActivityOptions::new().task_queue(" "),
17891 ActivityOptionsErrorKind::EmptyTaskQueue,
17892 ),
17893 (
17894 ActivityOptions::new().retry_policy(ActivityRetryPolicy::default()),
17895 ActivityOptionsErrorKind::EmptyRetryPolicy,
17896 ),
17897 (
17898 ActivityOptions::new().retry_policy(ActivityRetryPolicy::new(0)),
17899 ActivityOptionsErrorKind::InvalidMaxAttempts,
17900 ),
17901 (
17902 ActivityOptions::new().retry_policy(ActivityRetryPolicy {
17903 max_attempts: None,
17904 backoff: Some(ActivityBackoff::Explicit(vec![Duration::from_secs(1)])),
17905 non_retryable_error_types: Vec::new(),
17906 }),
17907 ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
17908 ),
17909 (
17910 ActivityOptions::new().retry_policy(
17911 ActivityRetryPolicy::new(2)
17912 .backoff_intervals([Duration::from_secs(1), Duration::from_secs(2)]),
17913 ),
17914 ActivityOptionsErrorKind::TooManyBackoffIntervals,
17915 ),
17916 (
17917 ActivityOptions::new().retry_policy(
17918 ActivityRetryPolicy::new(2).exponential_backoff(
17919 Duration::from_secs(1),
17920 0,
17921 None,
17922 ),
17923 ),
17924 ActivityOptionsErrorKind::InvalidBackoffCoefficient,
17925 ),
17926 (
17927 ActivityOptions::new()
17928 .retry_policy(ActivityRetryPolicy::new(2).non_retryable_error_type(" ")),
17929 ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
17930 ),
17931 (
17932 ActivityOptions::new().retry_policy(
17933 ActivityRetryPolicy::new(10_002).exponential_backoff(
17934 Duration::from_secs(1),
17935 1,
17936 None,
17937 ),
17938 ),
17939 ActivityOptionsErrorKind::BackoffGenerationTooLarge,
17940 ),
17941 (
17942 ActivityOptions::new().retry_policy(
17943 ActivityRetryPolicy::new(2)
17944 .backoff_intervals([Duration::from_secs(i64::MAX as u64 + 1)]),
17945 ),
17946 ActivityOptionsErrorKind::BackoffOverflow,
17947 ),
17948 ];
17949
17950 for (options, expected_kind) in cases {
17951 let ctx = workflow_context(Vec::new());
17952 let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
17953 let mut task_context = TaskContext::from_waker(noop_waker_ref());
17954 let Poll::Ready(Err(Error::InvalidActivityOptions(error))) =
17955 call.as_mut().poll(&mut task_context)
17956 else {
17957 panic!("expected typed activity validation error");
17958 };
17959 assert_eq!(error.kind, expected_kind);
17960 assert!(ctx.take_commands().expect("commands").is_empty());
17961 }
17962 }
17963
17964 #[test]
17965 fn activity_options_validate_positive_and_ordered_timeouts() {
17966 let zero_timeout_cases = [
17967 ActivityOptions::new().start_to_close_timeout(Duration::ZERO),
17968 ActivityOptions::new().schedule_to_start_timeout(Duration::ZERO),
17969 ActivityOptions::new().schedule_to_close_timeout(Duration::ZERO),
17970 ActivityOptions::new().heartbeat_timeout(Duration::ZERO),
17971 ];
17972 for options in zero_timeout_cases {
17973 assert_eq!(
17974 options.validate().expect_err("zero timeout").kind,
17975 ActivityOptionsErrorKind::TimeoutNotPositive
17976 );
17977 }
17978
17979 let ordering_cases = [
17980 ActivityOptions::new()
17981 .heartbeat_timeout(Duration::from_secs(11))
17982 .start_to_close_timeout(Duration::from_secs(10)),
17983 ActivityOptions::new()
17984 .start_to_close_timeout(Duration::from_secs(31))
17985 .schedule_to_close_timeout(Duration::from_secs(30)),
17986 ActivityOptions::new()
17987 .schedule_to_start_timeout(Duration::from_secs(31))
17988 .schedule_to_close_timeout(Duration::from_secs(30)),
17989 ];
17990 for options in ordering_cases {
17991 assert_eq!(
17992 options.validate().expect_err("timeout order").kind,
17993 ActivityOptionsErrorKind::TimeoutOrder
17994 );
17995 }
17996
17997 assert_eq!(
17998 ActivityOptions::new()
17999 .start_to_close_timeout(Duration::from_secs(i64::MAX as u64 + 1))
18000 .validate()
18001 .expect_err("protocol integer overflow")
18002 .kind,
18003 ActivityOptionsErrorKind::TimeoutOverflow
18004 );
18005 }
18006
18007 #[test]
18008 fn replayed_activity_retry_history_completes_without_duplicate_schedule() {
18009 let ctx = workflow_context(completed_retry_activity_history());
18010 let mut call =
18011 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18012 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18013
18014 assert!(matches!(
18015 call.as_mut().poll(&mut task_context),
18016 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18017 ));
18018 assert!(ctx.take_commands().expect("commands").is_empty());
18019 ctx.ensure_history_consumed().expect("history consumed");
18020 }
18021
18022 #[test]
18023 fn duplicate_non_retryable_types_use_one_command_and_replay_representation() {
18024 let mut options = retry_activity_options();
18025 options
18026 .retry_policy
18027 .as_mut()
18028 .expect("retry policy")
18029 .non_retryable_error_types
18030 .extend([" PermanentError ".to_string(), "PermanentError".to_string()]);
18031
18032 let new_ctx = workflow_context(Vec::new());
18033 let mut new_call =
18034 Box::pin(new_ctx.activity_with_options("flaky", options.clone(), json!([])));
18035 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18036 assert!(matches!(
18037 new_call.as_mut().poll(&mut task_context),
18038 Poll::Pending
18039 ));
18040 let commands = new_ctx.take_commands().expect("commands");
18041 assert_eq!(commands.len(), 1);
18042 assert_eq!(
18043 commands[0]["retry_policy"]["non_retryable_error_types"],
18044 json!(["PermanentError"])
18045 );
18046
18047 let replay_ctx = workflow_context(completed_retry_activity_history());
18048 let mut replay_call =
18049 Box::pin(replay_ctx.activity_with_options("flaky", options, json!([])));
18050 assert!(matches!(
18051 replay_call.as_mut().poll(&mut task_context),
18052 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18053 ));
18054 assert!(replay_ctx.take_commands().expect("commands").is_empty());
18055 replay_ctx
18056 .ensure_history_consumed()
18057 .expect("history consumed");
18058 }
18059
18060 #[test]
18061 fn replayed_intermediate_retry_remains_pending_across_restarts() {
18062 let history = completed_retry_activity_history()
18063 .into_iter()
18064 .take(3)
18065 .collect::<Vec<_>>();
18066
18067 for _restart in 0..2 {
18068 let ctx = workflow_context(history.clone());
18069 let mut call =
18070 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18071 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18072 assert!(matches!(
18073 call.as_mut().poll(&mut task_context),
18074 Poll::Pending
18075 ));
18076 assert!(ctx.take_commands().expect("commands").is_empty());
18077 }
18078 }
18079
18080 #[test]
18081 fn replayed_activity_rejects_changed_queue_retry_and_every_timeout_field() {
18082 let mut changed_queue = retry_activity_options();
18083 changed_queue.task_queue = Some("different-queue".to_string());
18084
18085 let mut changed_max_attempts = retry_activity_options();
18086 let retry_policy = changed_max_attempts
18087 .retry_policy
18088 .as_mut()
18089 .expect("retry policy");
18090 retry_policy.max_attempts = Some(4);
18091
18092 let mut changed_backoff = retry_activity_options();
18093 let retry_policy = changed_backoff.retry_policy.as_mut().expect("retry policy");
18094 retry_policy.backoff = Some(ActivityBackoff::Explicit(vec![
18095 Duration::from_secs(3),
18096 Duration::from_secs(4),
18097 ]));
18098
18099 let mut changed_non_retryable_types = retry_activity_options();
18100 let retry_policy = changed_non_retryable_types
18101 .retry_policy
18102 .as_mut()
18103 .expect("retry policy");
18104 retry_policy.non_retryable_error_types = vec!["AnotherPermanentError".to_string()];
18105
18106 let mut changed_start_to_close = retry_activity_options();
18107 changed_start_to_close.start_to_close_timeout = Some(Duration::from_secs(31));
18108 let mut changed_schedule_to_start = retry_activity_options();
18109 changed_schedule_to_start.schedule_to_start_timeout = Some(Duration::from_secs(6));
18110 let mut changed_schedule_to_close = retry_activity_options();
18111 changed_schedule_to_close.schedule_to_close_timeout = Some(Duration::from_secs(91));
18112 let mut changed_heartbeat = retry_activity_options();
18113 changed_heartbeat.heartbeat_timeout = Some(Duration::from_secs(11));
18114
18115 let cases = [
18116 (changed_queue, "activity_task_queue_mismatch"),
18117 (changed_max_attempts, "activity_retry_policy_mismatch"),
18118 (changed_backoff, "activity_retry_policy_mismatch"),
18119 (
18120 changed_non_retryable_types,
18121 "activity_retry_policy_mismatch",
18122 ),
18123 (changed_start_to_close, "activity_retry_policy_mismatch"),
18124 (changed_schedule_to_start, "activity_retry_policy_mismatch"),
18125 (changed_schedule_to_close, "activity_retry_policy_mismatch"),
18126 (changed_heartbeat, "activity_retry_policy_mismatch"),
18127 ];
18128
18129 for (options, expected_reason) in cases {
18130 let ctx = workflow_context(completed_retry_activity_history());
18131 let mut call = Box::pin(ctx.activity_with_options("flaky", options, json!([])));
18132 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18133 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18134 call.as_mut().poll(&mut task_context)
18135 else {
18136 panic!("changed activity options must fail replay");
18137 };
18138 assert_eq!(failure.reason, expected_reason);
18139 assert_eq!(failure.sequence, Some(1));
18140 assert!(ctx.take_commands().expect("commands").is_empty());
18141 }
18142 }
18143
18144 #[test]
18145 fn replayed_activity_rejects_changed_execution_mode_and_snapshot_version() {
18146 let cases = [
18147 (
18148 "execution_mode",
18149 json!("local"),
18150 "activity_execution_mode_mismatch",
18151 ),
18152 (
18153 "snapshot_version",
18154 json!(2),
18155 "activity_retry_policy_mismatch",
18156 ),
18157 ];
18158
18159 for (field, value, expected_reason) in cases {
18160 let mut history = completed_retry_activity_history();
18161 let activity = history[0].payload["activity"]
18162 .as_object_mut()
18163 .expect("activity snapshot");
18164 if field == "execution_mode" {
18165 activity.insert(field.to_string(), value);
18166 } else {
18167 activity["retry_policy"]
18168 .as_object_mut()
18169 .expect("retry snapshot")
18170 .insert(field.to_string(), value);
18171 }
18172
18173 let ctx = workflow_context(history);
18174 let mut call =
18175 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18176 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18177 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18178 call.as_mut().poll(&mut task_context)
18179 else {
18180 panic!("changed {field} must fail replay");
18181 };
18182 assert_eq!(failure.reason, expected_reason);
18183 assert_eq!(failure.sequence, Some(1));
18184 assert!(ctx.take_commands().expect("commands").is_empty());
18185 }
18186 }
18187
18188 #[test]
18189 fn replayed_legacy_activity_treats_missing_option_snapshot_as_unknown() {
18190 let mut history = completed_retry_activity_history();
18191 let activity = history[0].payload["activity"]
18192 .as_object_mut()
18193 .expect("activity snapshot");
18194 activity.remove("execution_mode");
18195 activity.remove("retry_policy");
18196
18197 let mut current = retry_activity_options();
18198 current.start_to_close_timeout = Some(Duration::from_secs(45));
18199 current.schedule_to_start_timeout = Some(Duration::from_secs(8));
18200 current.schedule_to_close_timeout = Some(Duration::from_secs(120));
18201 current.heartbeat_timeout = Some(Duration::from_secs(12));
18202
18203 let ctx = workflow_context(history);
18204 let mut call = Box::pin(ctx.activity_with_options("flaky", current, json!([])));
18205 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18206 assert!(matches!(
18207 call.as_mut().poll(&mut task_context),
18208 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18209 ));
18210 assert!(ctx.take_commands().expect("commands").is_empty());
18211 ctx.ensure_history_consumed().expect("history consumed");
18212 }
18213
18214 #[test]
18215 fn terminal_activity_failed_after_start_returns_typed_failure() {
18216 let history = vec![
18217 history_event(
18218 "ActivityScheduled",
18219 json!({
18220 "sequence": 1,
18221 "activity_type": "flaky",
18222 "activity_execution_id": "act-terminal",
18223 "activity": {
18224 "id": "act-terminal",
18225 "sequence": 1,
18226 "type": "flaky",
18227 "queue": "critical-activities",
18228 "retry_policy": {
18229 "snapshot_version": 1,
18230 "max_attempts": 3,
18231 "backoff_seconds": [2, 4],
18232 "non_retryable_error_types": ["PermanentError"]
18233 }
18234 }
18235 }),
18236 ),
18237 history_event(
18238 "ActivityStarted",
18239 json!({
18240 "sequence": 1,
18241 "activity_type": "flaky",
18242 "activity_execution_id": "act-terminal",
18243 "activity_attempt_id": "attempt-1",
18244 "attempt_number": 1
18245 }),
18246 ),
18247 history_event(
18248 "ActivityFailed",
18249 json!({
18250 "sequence": 1,
18251 "activity_type": "flaky",
18252 "activity_execution_id": "act-terminal",
18253 "activity_attempt_id": "attempt-1",
18254 "attempt_number": 1,
18255 "failure_id": "failure-terminal",
18256 "failure_category": "activity",
18257 "exception_type": "PermanentError",
18258 "message": "cannot retry",
18259 "non_retryable": true
18260 }),
18261 ),
18262 ];
18263 let ctx = workflow_context(history);
18264 let mut call =
18265 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18266 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18267
18268 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18269 call.as_mut().poll(&mut task_context)
18270 else {
18271 panic!("terminal ActivityFailed must settle the activity future");
18272 };
18273 assert_eq!(failure.kind, ActivityFailureKind::Failed);
18274 assert_eq!(
18275 failure.activity_execution_id.as_deref(),
18276 Some("act-terminal")
18277 );
18278 assert_eq!(failure.exception_type.as_deref(), Some("PermanentError"));
18279 assert!(failure.non_retryable);
18280 assert!(ctx.take_commands().expect("commands").is_empty());
18281 ctx.ensure_history_consumed().expect("history consumed");
18282 }
18283
18284 #[test]
18285 fn activity_terminal_events_return_machine_readable_failures() {
18286 let cases = [
18287 (
18288 "ActivityFailed",
18289 json!({
18290 "sequence": 1,
18291 "activity_type": "charge-card",
18292 "activity_execution_id": "act-1",
18293 "activity_attempt_id": "attempt-2",
18294 "attempt_number": 2,
18295 "failure_id": "failure-1",
18296 "failure_category": "activity",
18297 "exception_type": "PaymentDeclined",
18298 "exception_class": "payments.PaymentDeclined",
18299 "message": "card declined",
18300 "non_retryable": true
18301 }),
18302 ActivityFailureKind::Failed,
18303 "activity",
18304 ),
18305 (
18306 "ActivityCancelled",
18307 json!({
18308 "sequence": 1,
18309 "activity_type": "charge-card",
18310 "activity_execution_id": "act-1",
18311 "activity_attempt_id": "attempt-1"
18312 }),
18313 ActivityFailureKind::Cancelled,
18314 "cancelled",
18315 ),
18316 ];
18317
18318 for (event_type, payload, expected_kind, expected_reason) in cases {
18319 let ctx = workflow_context(vec![history_event(event_type, payload)]);
18320 let mut call = Box::pin(ctx.activity("charge-card", json!([])));
18321 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18322 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18323 call.as_mut().poll(&mut task_context)
18324 else {
18325 panic!("expected terminal activity failure");
18326 };
18327 assert_eq!(failure.kind, expected_kind);
18328 assert_eq!(failure.reason, expected_reason);
18329 assert_eq!(failure.activity_execution_id.as_deref(), Some("act-1"));
18330 assert_eq!(failure.activity_type.as_deref(), Some("charge-card"));
18331 }
18332 }
18333
18334 #[test]
18335 fn every_activity_timeout_class_is_typed() {
18336 for timeout_kind in [
18337 "start_to_close",
18338 "schedule_to_start",
18339 "schedule_to_close",
18340 "heartbeat",
18341 ] {
18342 let ctx = workflow_context(vec![history_event(
18343 "ActivityTimedOut",
18344 json!({
18345 "sequence": 1,
18346 "activity_type": "slow",
18347 "activity_execution_id": "act-timeout",
18348 "activity_attempt_id": "attempt-timeout",
18349 "failure_category": "timeout",
18350 "timeout_kind": timeout_kind,
18351 "message": "deadline expired"
18352 }),
18353 )]);
18354 let mut call = Box::pin(ctx.activity("slow", json!([])));
18355 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18356 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18357 call.as_mut().poll(&mut task_context)
18358 else {
18359 panic!("expected timeout failure");
18360 };
18361 assert_eq!(failure.kind, ActivityFailureKind::TimedOut);
18362 assert_eq!(failure.reason, timeout_kind);
18363 assert_eq!(failure.timeout_kind.as_deref(), Some(timeout_kind));
18364 assert_eq!(failure.failure_category.as_deref(), Some("timeout"));
18365 }
18366 }
18367
18368 #[test]
18369 fn workflow_sleep_emits_one_durable_timer_and_rounds_up() {
18370 let ctx = workflow_context(Vec::new());
18371 let mut sleep = Box::pin(ctx.sleep(Duration::from_millis(1_001)));
18372 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18373
18374 assert!(matches!(
18375 sleep.as_mut().poll(&mut task_context),
18376 Poll::Pending
18377 ));
18378 assert!(matches!(
18379 sleep.as_mut().poll(&mut task_context),
18380 Poll::Pending
18381 ));
18382
18383 let commands = ctx.take_commands().expect("timer command");
18384 assert_eq!(
18385 commands,
18386 vec![json!({
18387 "type": "start_timer",
18388 "delay_seconds": 2,
18389 })]
18390 );
18391 }
18392
18393 #[test]
18394 fn workflow_sleep_replays_matching_schedule_and_fire_without_a_command() {
18395 let history = vec![
18396 history_event(
18397 "TimerScheduled",
18398 json!({
18399 "sequence": 1,
18400 "timer_id": "timer-1",
18401 "delay_seconds": 5,
18402 "fire_at": "2026-07-11T12:00:05Z",
18403 }),
18404 ),
18405 history_event(
18406 "TimerFired",
18407 json!({
18408 "sequence": 1,
18409 "timer_id": "timer-1",
18410 "delay_seconds": 5,
18411 "fire_at": "2026-07-11T12:00:05Z",
18412 "fired_at": "2026-07-11T12:00:05Z",
18413 }),
18414 ),
18415 ];
18416
18417 for _restart in 0..2 {
18418 let ctx = workflow_context(history.clone());
18419 let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(5)));
18420 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18421 assert!(matches!(
18422 sleep.as_mut().poll(&mut task_context),
18423 Poll::Ready(Ok(()))
18424 ));
18425 assert!(ctx.take_commands().expect("commands").is_empty());
18426 ctx.ensure_history_consumed().expect("history consumed");
18427 }
18428 }
18429
18430 #[test]
18431 fn workflow_sleep_rejects_changed_delay_during_replay() {
18432 let ctx = workflow_context(vec![
18433 history_event(
18434 "TimerScheduled",
18435 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
18436 ),
18437 history_event(
18438 "TimerFired",
18439 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
18440 ),
18441 ]);
18442 let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(500)));
18443 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18444
18445 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18446 sleep.as_mut().poll(&mut task_context)
18447 else {
18448 panic!("changed timer delay must be rejected");
18449 };
18450 assert_eq!(failure.reason, "timer_delay_mismatch");
18451 assert_eq!(failure.sequence, Some(1));
18452 }
18453
18454 #[test]
18455 fn workflow_condition_wait_emits_published_identity_and_timeout_contract() {
18456 let ctx = workflow_context(Vec::new());
18457 let mut wait = Box::pin(
18458 ctx.wait_condition(
18459 ConditionWaitOptions::new("approval.ready", "sha256:approval-v1")
18460 .timeout(Duration::from_millis(60_001)),
18461 || Ok(false),
18462 ),
18463 );
18464 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18465
18466 assert!(matches!(
18467 wait.as_mut().poll(&mut task_context),
18468 Poll::Pending
18469 ));
18470 assert!(matches!(
18471 wait.as_mut().poll(&mut task_context),
18472 Poll::Pending
18473 ));
18474 assert_eq!(
18475 ctx.take_commands().expect("condition command"),
18476 vec![json!({
18477 "type": "open_condition_wait",
18478 "condition_wait_occurrence_id": "rust:condition-wait:0",
18479 "condition_key": "approval.ready",
18480 "condition_definition_fingerprint": "sha256:approval-v1",
18481 "timeout_seconds": 61,
18482 })]
18483 );
18484 }
18485
18486 #[test]
18487 fn workflow_condition_wait_returns_explicit_immediate_results_without_commands() {
18488 let ctx = workflow_context(Vec::new());
18489 let mut satisfied = Box::pin(wait_condition!(ctx, "already-ready", || Ok(true)));
18490 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18491 assert!(matches!(
18492 satisfied.as_mut().poll(&mut task_context),
18493 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18494 ));
18495
18496 let mut timed_out = Box::pin(wait_condition!(
18497 ctx,
18498 "no-wait",
18499 timeout: Duration::ZERO,
18500 || Ok(false),
18501 ));
18502 assert!(matches!(
18503 timed_out.as_mut().poll(&mut task_context),
18504 Poll::Ready(Ok(ConditionWaitResult::TimedOut))
18505 ));
18506 assert!(ctx.take_commands().expect("commands").is_empty());
18507 }
18508
18509 #[test]
18510 fn signal_and_update_history_reevaluate_open_conditions_after_restart() {
18511 let signal_history = vec![
18512 history_event(
18513 "ConditionWaitOpened",
18514 json!({
18515 "sequence": 4,
18516 "condition_wait_id": "condition:4",
18517 "condition_wait_occurrence_id": "rust:condition-wait:0",
18518 "condition_key": "approval",
18519 "condition_definition_fingerprint": "sha256:approval-v1",
18520 "timeout_seconds": 30,
18521 }),
18522 ),
18523 history_event(
18524 "SignalReceived",
18525 json!({
18526 "workflow_sequence": 4,
18527 "signal_name": "approve",
18528 "arguments": fixture_envelope(json!(["Ada"])),
18529 }),
18530 ),
18531 ];
18532 for _worker_before_or_after_restart in 0..2 {
18533 let ctx = workflow_context(signal_history.clone());
18534 let predicate_ctx = ctx.clone();
18535 let mut wait = Box::pin(
18536 ctx.wait_condition(
18537 ConditionWaitOptions::new("approval", "sha256:approval-v1")
18538 .timeout(Duration::from_secs(30)),
18539 move || Ok(!predicate_ctx.signals("approve")?.is_empty()),
18540 ),
18541 );
18542 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18543 assert!(matches!(
18544 wait.as_mut().poll(&mut task_context),
18545 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18546 ));
18547 assert!(ctx.take_commands().expect("commands").is_empty());
18548 ctx.ensure_history_consumed().expect("condition consumed");
18549 }
18550
18551 let update_history = vec![
18552 history_event(
18553 "ConditionWaitOpened",
18554 json!({
18555 "sequence": 7,
18556 "condition_wait_id": "condition:7",
18557 "condition_wait_occurrence_id": "rust:condition-wait:0",
18558 "condition_key": "update-approval",
18559 "condition_definition_fingerprint": "sha256:update-approval-v1",
18560 }),
18561 ),
18562 history_event(
18563 "UpdateApplied",
18564 json!({
18565 "sequence": 7,
18566 "update_id": "update-1",
18567 "update_name": "approve",
18568 "arguments": fixture_envelope(json!([true])),
18569 }),
18570 ),
18571 ];
18572 let ctx = workflow_context(update_history);
18573 let predicate_ctx = ctx.clone();
18574 let mut wait = Box::pin(ctx.wait_condition(
18575 ConditionWaitOptions::new("update-approval", "sha256:update-approval-v1"),
18576 move || {
18577 Ok(predicate_ctx
18578 .updates("approve")?
18579 .first()
18580 .and_then(|arguments| arguments.first())
18581 .and_then(Value::as_bool)
18582 == Some(true))
18583 },
18584 ));
18585 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18586 assert!(matches!(
18587 wait.as_mut().poll(&mut task_context),
18588 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18589 ));
18590 assert!(ctx.take_commands().expect("commands").is_empty());
18591 ctx.ensure_history_consumed().expect("condition consumed");
18592 }
18593
18594 #[test]
18595 fn condition_wait_preserves_open_satisfied_and_timed_out_replay_states() {
18596 let open_history = vec![
18597 history_event(
18598 "ConditionWaitOpened",
18599 json!({
18600 "sequence": 3,
18601 "condition_wait_id": "condition:3",
18602 "condition_wait_occurrence_id": "rust:condition-wait:0",
18603 "condition_key": "two-votes",
18604 "condition_definition_fingerprint": "sha256:two-votes-v1",
18605 "timeout_seconds": 120,
18606 }),
18607 ),
18608 history_event(
18609 "SignalReceived",
18610 json!({
18611 "workflow_sequence": 3,
18612 "signal_name": "vote",
18613 "arguments": fixture_envelope(json!(["first"])),
18614 }),
18615 ),
18616 ];
18617 for _worker_before_or_after_restart in 0..2 {
18618 let ctx = workflow_context(open_history.clone());
18619 let predicate_ctx = ctx.clone();
18620 let mut wait = Box::pin(
18621 ctx.wait_condition(
18622 ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1")
18623 .timeout(Duration::from_secs(120)),
18624 move || Ok(predicate_ctx.signals("vote")?.len() >= 2),
18625 ),
18626 );
18627 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18628 assert!(matches!(
18629 wait.as_mut().poll(&mut task_context),
18630 Poll::Pending
18631 ));
18632 assert_eq!(
18633 ctx.take_commands().expect("reopened condition"),
18634 vec![json!({
18635 "type": "open_condition_wait",
18636 "condition_wait_occurrence_id": "rust:condition-wait:0",
18637 "condition_key": "two-votes",
18638 "condition_definition_fingerprint": "sha256:two-votes-v1",
18639 "timeout_seconds": 120,
18640 })]
18641 );
18642 }
18643
18644 let satisfied_ctx = workflow_context(vec![
18645 history_event(
18646 "ConditionWaitOpened",
18647 json!({
18648 "sequence": 5,
18649 "condition_wait_id": "condition:5",
18650 "condition_wait_occurrence_id": "rust:condition-wait:0",
18651 "condition_key": "approval",
18652 "condition_definition_fingerprint": "sha256:approval-v1",
18653 }),
18654 ),
18655 history_event(
18656 "ConditionWaitSatisfied",
18657 json!({
18658 "sequence": 5,
18659 "condition_wait_id": "condition:5",
18660 "condition_wait_occurrence_id": "rust:condition-wait:0",
18661 "condition_key": "approval",
18662 "condition_definition_fingerprint": "sha256:approval-v1",
18663 }),
18664 ),
18665 ]);
18666 let mut satisfied = Box::pin(satisfied_ctx.wait_condition(
18667 ConditionWaitOptions::new("approval", "sha256:approval-v1"),
18668 || Ok(false),
18669 ));
18670 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18671 assert!(matches!(
18672 satisfied.as_mut().poll(&mut task_context),
18673 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18674 ));
18675
18676 let timed_out_ctx = workflow_context(vec![
18677 history_event(
18678 "ConditionWaitOpened",
18679 json!({
18680 "sequence": 8,
18681 "condition_wait_id": "condition:8",
18682 "condition_wait_occurrence_id": "rust:condition-wait:0",
18683 "condition_key": "approval-timeout",
18684 "condition_definition_fingerprint": "sha256:approval-timeout-v1",
18685 "timeout_seconds": 5,
18686 }),
18687 ),
18688 history_event(
18689 "TimerScheduled",
18690 json!({
18691 "sequence": 9,
18692 "timer_id": "condition-timer:9",
18693 "timer_kind": "condition_timeout",
18694 "condition_wait_id": "condition:8",
18695 "delay_seconds": 5,
18696 }),
18697 ),
18698 history_event(
18699 "TimerFired",
18700 json!({
18701 "sequence": 9,
18702 "timer_id": "condition-timer:9",
18703 "timer_kind": "condition_timeout",
18704 "condition_wait_id": "condition:8",
18705 "delay_seconds": 5,
18706 }),
18707 ),
18708 ]);
18709 let mut timed_out = Box::pin(
18710 timed_out_ctx.wait_condition(
18711 ConditionWaitOptions::new("approval-timeout", "sha256:approval-timeout-v1")
18712 .timeout(Duration::from_secs(5)),
18713 || Ok(true),
18714 ),
18715 );
18716 assert!(matches!(
18717 timed_out.as_mut().poll(&mut task_context),
18718 Poll::Ready(Ok(ConditionWaitResult::TimedOut))
18719 ));
18720 }
18721
18722 #[test]
18723 fn condition_wait_replays_repeated_physical_opens_as_one_logical_wait() {
18724 let history = vec![
18725 history_event(
18726 "ConditionWaitOpened",
18727 json!({
18728 "sequence": 3,
18729 "condition_wait_id": "condition:3",
18730 "condition_wait_occurrence_id": "rust:condition-wait:0",
18731 "condition_key": "two-votes",
18732 "condition_definition_fingerprint": "sha256:two-votes-v1",
18733 }),
18734 ),
18735 history_event(
18736 "SignalReceived",
18737 json!({
18738 "workflow_sequence": 3,
18739 "signal_name": "vote",
18740 "arguments": fixture_envelope(json!(["first"])),
18741 }),
18742 ),
18743 history_event(
18744 "ConditionWaitSatisfied",
18745 json!({
18746 "sequence": 3,
18747 "condition_wait_id": "condition:3",
18748 "condition_wait_occurrence_id": "rust:condition-wait:0",
18749 "condition_key": "two-votes",
18750 "condition_definition_fingerprint": "sha256:two-votes-v1",
18751 }),
18752 ),
18753 history_event(
18754 "ConditionWaitOpened",
18755 json!({
18756 "sequence": 5,
18757 "condition_wait_id": "condition:5",
18758 "condition_wait_occurrence_id": "rust:condition-wait:0",
18759 "condition_key": "two-votes",
18760 "condition_definition_fingerprint": "sha256:two-votes-v1",
18761 }),
18762 ),
18763 history_event(
18764 "SignalReceived",
18765 json!({
18766 "workflow_sequence": 5,
18767 "signal_name": "vote",
18768 "arguments": fixture_envelope(json!(["second"])),
18769 }),
18770 ),
18771 history_event(
18772 "ConditionWaitSatisfied",
18773 json!({
18774 "sequence": 5,
18775 "condition_wait_id": "condition:5",
18776 "condition_wait_occurrence_id": "rust:condition-wait:0",
18777 "condition_key": "two-votes",
18778 "condition_definition_fingerprint": "sha256:two-votes-v1",
18779 }),
18780 ),
18781 ];
18782 for _cold_worker_or_restart in 0..2 {
18783 let ctx = workflow_context(history.clone());
18784 let predicate_ctx = ctx.clone();
18785 let mut wait = Box::pin(ctx.wait_condition(
18786 ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1"),
18787 move || Ok(predicate_ctx.signals("vote")?.len() >= 2),
18788 ));
18789 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18790
18791 assert!(matches!(
18792 wait.as_mut().poll(&mut task_context),
18793 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18794 ));
18795 assert!(ctx.take_commands().expect("commands").is_empty());
18796 ctx.ensure_history_consumed()
18797 .expect("every physical wait-open is consumed");
18798 }
18799 }
18800
18801 #[test]
18802 fn condition_wait_replays_update_driven_physical_opens_as_one_occurrence() {
18803 let history = vec![
18804 history_event(
18805 "ConditionWaitOpened",
18806 json!({
18807 "sequence": 3,
18808 "condition_wait_id": "condition:3",
18809 "condition_wait_occurrence_id": "rust:condition-wait:0",
18810 "condition_key": "approved",
18811 "condition_definition_fingerprint": "sha256:approved-v1",
18812 }),
18813 ),
18814 history_event(
18815 "UpdateApplied",
18816 json!({
18817 "sequence": 3,
18818 "update_id": "update-1",
18819 "update_name": "approve",
18820 "arguments": fixture_envelope(json!([false])),
18821 }),
18822 ),
18823 history_event(
18824 "ConditionWaitOpened",
18825 json!({
18826 "sequence": 5,
18827 "condition_wait_id": "condition:5",
18828 "condition_wait_occurrence_id": "rust:condition-wait:0",
18829 "condition_key": "approved",
18830 "condition_definition_fingerprint": "sha256:approved-v1",
18831 }),
18832 ),
18833 history_event(
18834 "UpdateApplied",
18835 json!({
18836 "sequence": 5,
18837 "update_id": "update-2",
18838 "update_name": "approve",
18839 "arguments": fixture_envelope(json!([true])),
18840 }),
18841 ),
18842 ];
18843
18844 for _cold_worker_or_restart in 0..2 {
18845 let ctx = workflow_context(history.clone());
18846 let predicate_ctx = ctx.clone();
18847 let mut wait = Box::pin(ctx.wait_condition(
18848 ConditionWaitOptions::new("approved", "sha256:approved-v1"),
18849 move || {
18850 Ok(predicate_ctx
18851 .updates("approve")?
18852 .last()
18853 .and_then(|arguments| arguments.first())
18854 .and_then(Value::as_bool)
18855 == Some(true))
18856 },
18857 ));
18858 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18859
18860 assert!(matches!(
18861 wait.as_mut().poll(&mut task_context),
18862 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18863 ));
18864 assert!(ctx.take_commands().expect("commands").is_empty());
18865 ctx.ensure_history_consumed()
18866 .expect("every update-driven reopen is consumed");
18867 }
18868 }
18869
18870 #[test]
18871 fn condition_wait_replay_keeps_every_adjacent_authored_occurrence_distinct() {
18872 for (first_key, first_fingerprint, second_key, second_fingerprint) in [
18873 ("shared", "sha256:first", "shared", "sha256:second"),
18874 ("first", "sha256:shared", "second", "sha256:shared"),
18875 ("shared", "sha256:shared", "shared", "sha256:shared"),
18876 ("first", "sha256:first", "second", "sha256:second"),
18877 ] {
18878 let history = vec![
18879 history_event(
18880 "ConditionWaitOpened",
18881 json!({
18882 "sequence": 3,
18883 "condition_wait_id": "condition:3",
18884 "condition_wait_occurrence_id": "rust:condition-wait:0",
18885 "condition_key": first_key,
18886 "condition_definition_fingerprint": first_fingerprint,
18887 }),
18888 ),
18889 history_event(
18890 "ConditionWaitSatisfied",
18891 json!({
18892 "sequence": 3,
18893 "condition_wait_id": "condition:3",
18894 "condition_wait_occurrence_id": "rust:condition-wait:0",
18895 "condition_key": first_key,
18896 "condition_definition_fingerprint": first_fingerprint,
18897 }),
18898 ),
18899 history_event(
18900 "ConditionWaitOpened",
18901 json!({
18902 "sequence": 4,
18903 "condition_wait_id": "condition:4",
18904 "condition_wait_occurrence_id": "rust:condition-wait:1",
18905 "condition_key": second_key,
18906 "condition_definition_fingerprint": second_fingerprint,
18907 }),
18908 ),
18909 history_event(
18910 "ConditionWaitSatisfied",
18911 json!({
18912 "sequence": 4,
18913 "condition_wait_id": "condition:4",
18914 "condition_wait_occurrence_id": "rust:condition-wait:1",
18915 "condition_key": second_key,
18916 "condition_definition_fingerprint": second_fingerprint,
18917 }),
18918 ),
18919 ];
18920 for _cold_worker_or_restart in 0..2 {
18921 let ctx = workflow_context(history.clone());
18922 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18923 let mut first = Box::pin(ctx.wait_condition(
18924 ConditionWaitOptions::new(first_key, first_fingerprint),
18925 || Ok(false),
18926 ));
18927 assert!(matches!(
18928 first.as_mut().poll(&mut task_context),
18929 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18930 ));
18931
18932 let mut second = Box::pin(ctx.wait_condition(
18933 ConditionWaitOptions::new(second_key, second_fingerprint),
18934 || Ok(false),
18935 ));
18936 assert!(matches!(
18937 second.as_mut().poll(&mut task_context),
18938 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18939 ));
18940 assert!(ctx.take_commands().expect("commands").is_empty());
18941 ctx.ensure_history_consumed()
18942 .expect("each authored wait consumes one occurrence");
18943 }
18944 }
18945 }
18946
18947 #[test]
18948 fn cold_workers_replay_adjacent_condition_waits_from_one_loop_call_site() {
18949 fn worker() -> Worker {
18950 let client = Client::new("http://127.0.0.1:8080").expect("client");
18951 let mut worker = Worker::new(client, "rust-workers");
18952 worker.register_workflow("rust.condition-loop", |ctx, _input| async move {
18953 let mut outcomes = Vec::new();
18954 for _ in 0..2 {
18955 outcomes.push(
18956 ctx.wait_condition(
18957 ConditionWaitOptions::new("shared", "sha256:shared"),
18958 || Ok(false),
18959 )
18960 .await?,
18961 );
18962 }
18963 Ok(json!(outcomes))
18964 });
18965 worker
18966 }
18967
18968 let task = workflow_task(
18969 "rust.condition-loop",
18970 vec![
18971 history_event(
18972 "ConditionWaitOpened",
18973 json!({
18974 "sequence": 1,
18975 "condition_wait_id": "condition:1",
18976 "condition_wait_occurrence_id": "rust:condition-wait:0",
18977 "condition_key": "shared",
18978 "condition_definition_fingerprint": "sha256:shared",
18979 }),
18980 ),
18981 history_event(
18982 "ConditionWaitSatisfied",
18983 json!({
18984 "sequence": 1,
18985 "condition_wait_id": "condition:1",
18986 "condition_wait_occurrence_id": "rust:condition-wait:0",
18987 "condition_key": "shared",
18988 "condition_definition_fingerprint": "sha256:shared",
18989 }),
18990 ),
18991 history_event(
18992 "ConditionWaitOpened",
18993 json!({
18994 "sequence": 2,
18995 "condition_wait_id": "condition:2",
18996 "condition_wait_occurrence_id": "rust:condition-wait:1",
18997 "condition_key": "shared",
18998 "condition_definition_fingerprint": "sha256:shared",
18999 }),
19000 ),
19001 history_event(
19002 "ConditionWaitSatisfied",
19003 json!({
19004 "sequence": 2,
19005 "condition_wait_id": "condition:2",
19006 "condition_wait_occurrence_id": "rust:condition-wait:1",
19007 "condition_key": "shared",
19008 "condition_definition_fingerprint": "sha256:shared",
19009 }),
19010 ),
19011 ],
19012 DEFAULT_CODEC,
19013 );
19014
19015 for _cold_worker_or_restart in 0..2 {
19016 let commands = worker()
19017 .execute_workflow_task(task.clone())
19018 .expect("adjacent loop waits replay deterministically");
19019 assert_eq!(commands.len(), 1);
19020 assert_eq!(commands[0]["type"], "complete_workflow");
19021 assert_eq!(
19022 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
19023 json!(["satisfied", "satisfied"])
19024 );
19025 }
19026 }
19027
19028 #[test]
19029 fn condition_wait_replay_rejects_identity_predicate_and_timeout_changes() {
19030 let history = vec![history_event(
19031 "ConditionWaitOpened",
19032 json!({
19033 "sequence": 12,
19034 "condition_wait_id": "condition:12",
19035 "condition_wait_occurrence_id": "rust:condition-wait:0",
19036 "condition_key": "approval",
19037 "condition_definition_fingerprint": "sha256:approval-v1",
19038 "timeout_seconds": 30,
19039 }),
19040 )];
19041 for (options, expected_reason) in [
19042 (
19043 ConditionWaitOptions::new("changed", "sha256:approval-v1")
19044 .timeout(Duration::from_secs(30)),
19045 "condition_wait_key_mismatch",
19046 ),
19047 (
19048 ConditionWaitOptions::new("approval", "sha256:approval-v2")
19049 .timeout(Duration::from_secs(30)),
19050 "condition_wait_predicate_mismatch",
19051 ),
19052 (
19053 ConditionWaitOptions::new("approval", "sha256:approval-v1")
19054 .timeout(Duration::from_secs(29)),
19055 "condition_wait_timeout_mismatch",
19056 ),
19057 ] {
19058 let ctx = workflow_context(history.clone());
19059 let mut wait = Box::pin(ctx.wait_condition(options, || Ok(false)));
19060 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19061 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
19062 wait.as_mut().poll(&mut task_context)
19063 else {
19064 panic!("changed condition definition must fail replay");
19065 };
19066 assert_eq!(failure.reason, expected_reason);
19067 assert_eq!(failure.sequence, Some(12));
19068 }
19069 }
19070
19071 #[test]
19072 fn condition_wait_history_requires_the_canonical_predicate_fingerprint() {
19073 let error = WorkflowState::new(
19074 vec![history_event(
19075 "ConditionWaitOpened",
19076 json!({
19077 "sequence": 12,
19078 "condition_wait_id": "condition:12",
19079 "condition_wait_occurrence_id": "rust:condition-wait:0",
19080 "condition_key": "approval",
19081 }),
19082 )],
19083 "rust-workers".to_string(),
19084 DEFAULT_CODEC.to_string(),
19085 None,
19086 )
19087 .expect_err("condition history without a predicate fingerprint must fail");
19088
19089 assert!(matches!(
19090 error,
19091 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19092 if reason == "condition_wait_predicate_fingerprint_missing"
19093 ));
19094 }
19095
19096 #[test]
19097 fn condition_wait_history_requires_authored_occurrence_identity() {
19098 let error = WorkflowState::new(
19099 vec![history_event(
19100 "ConditionWaitOpened",
19101 json!({
19102 "sequence": 12,
19103 "condition_wait_id": "condition:12",
19104 "condition_key": "approval",
19105 "condition_definition_fingerprint": "sha256:approval-v1",
19106 }),
19107 )],
19108 "rust-workers".to_string(),
19109 DEFAULT_CODEC.to_string(),
19110 None,
19111 )
19112 .expect_err("condition history without occurrence identity must fail");
19113
19114 assert!(matches!(
19115 error,
19116 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19117 if reason == "condition_wait_occurrence_id_missing"
19118 ));
19119 }
19120
19121 #[test]
19122 fn typed_search_attribute_updates_validate_emit_and_replay() {
19123 let update = SearchAttributeUpdate::new()
19124 .keyword("OrderStatus", " waiting ")
19125 .expect("keyword")
19126 .int("Attempt", 3)
19127 .expect("int")
19128 .bool("Escalated", false)
19129 .expect("bool")
19130 .keyword_list("Regions", ["us-east", "eu-west"])
19131 .expect("list")
19132 .datetime("UpdatedAt", "2026-08-22T04:00:00Z")
19133 .expect("datetime")
19134 .delete("LegacyStatus")
19135 .expect("delete");
19136 let ctx = workflow_context(Vec::new());
19137 ctx.upsert_search_attributes(update.clone())
19138 .expect("typed update");
19139 assert_eq!(
19140 ctx.take_commands().expect("search-attribute command"),
19141 vec![json!({
19142 "type": "upsert_search_attributes",
19143 "attributes": {
19144 "Attempt": 3,
19145 "Escalated": false,
19146 "LegacyStatus": null,
19147 "OrderStatus": "waiting",
19148 "Regions": ["us-east", "eu-west"],
19149 "UpdatedAt": "2026-08-22T04:00:00Z",
19150 },
19151 "attribute_types": {
19152 "Attempt": "int",
19153 "Escalated": "bool",
19154 "OrderStatus": "keyword",
19155 "Regions": "keyword_list",
19156 "UpdatedAt": "datetime",
19157 },
19158 })]
19159 );
19160
19161 let replay = workflow_context(vec![history_event(
19162 "SearchAttributesUpserted",
19163 json!({
19164 "sequence": 6,
19165 "attributes": {
19166 "Attempt": 3,
19167 "Escalated": false,
19168 "LegacyStatus": null,
19169 "OrderStatus": "waiting",
19170 "Regions": ["us-east", "eu-west"],
19171 "UpdatedAt": "2026-08-22T04:00:00Z",
19172 },
19173 "attribute_types": {
19174 "Attempt": "int",
19175 "Escalated": "bool",
19176 "OrderStatus": "keyword",
19177 "Regions": "keyword_list",
19178 "UpdatedAt": "datetime",
19179 },
19180 "merged": {},
19181 }),
19182 )]);
19183 replay
19184 .upsert_search_attributes(update)
19185 .expect("matching update replays");
19186 assert!(replay.take_commands().expect("commands").is_empty());
19187 replay.ensure_history_consumed().expect("history consumed");
19188
19189 let type_drift = workflow_context(vec![history_event(
19190 "SearchAttributesUpserted",
19191 json!({
19192 "sequence": 7,
19193 "attributes": {"OrderStatus": "waiting"},
19194 "attribute_types": {"OrderStatus": "keyword"},
19195 "merged": {"OrderStatus": "waiting"},
19196 }),
19197 )]);
19198 let error = type_drift
19199 .upsert_search_attributes(
19200 SearchAttributeUpdate::new()
19201 .string("OrderStatus", "waiting")
19202 .expect("string update"),
19203 )
19204 .expect_err("same JSON value with a changed type must fail replay");
19205 assert!(matches!(
19206 error,
19207 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19208 if reason == "search_attribute_type_mismatch"
19209 ));
19210
19211 let malformed_types = WorkflowState::new(
19212 vec![history_event(
19213 "SearchAttributesUpserted",
19214 json!({
19215 "sequence": 8,
19216 "attributes": {"OrderStatus": "waiting"},
19217 "attribute_types": {"OrderStatus": "unsupported"},
19218 "merged": {"OrderStatus": "waiting"},
19219 }),
19220 )],
19221 "rust-workers".to_string(),
19222 DEFAULT_CODEC.to_string(),
19223 None,
19224 )
19225 .expect_err("unsupported search-attribute type metadata must fail");
19226 assert!(matches!(
19227 malformed_types,
19228 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19229 if reason == "search_attribute_types_malformed"
19230 ));
19231
19232 assert!(matches!(
19233 SearchAttributeUpdate::new().keyword("bad key", "value"),
19234 Err(SearchAttributeUpdateError::InvalidKey(_))
19235 ));
19236 assert!(matches!(
19237 SearchAttributeUpdate::new().float("Ratio", f64::NAN),
19238 Err(SearchAttributeUpdateError::NonFiniteFloat(_))
19239 ));
19240 assert!(matches!(
19241 SearchAttributeUpdate::new().keyword("UnicodeKeyword", "é".repeat(128)),
19242 Err(SearchAttributeUpdateError::ValueTooLong { .. })
19243 ));
19244 assert!(matches!(
19245 SearchAttributeUpdate::new().datetime("UpdatedAt", "2026-02-30T04:00:00Z"),
19246 Err(SearchAttributeUpdateError::InvalidDateTime(_))
19247 ));
19248 assert!(matches!(
19249 workflow_context(Vec::new()).upsert_search_attributes(SearchAttributeUpdate::new()),
19250 Err(Error::InvalidSearchAttributeUpdate(
19251 SearchAttributeUpdateError::Empty
19252 ))
19253 ));
19254 }
19255
19256 #[test]
19257 fn workflow_history_rejects_unpaired_or_mismatched_timer_events() {
19258 let lone_fire = WorkflowState::new(
19259 vec![history_event(
19260 "TimerFired",
19261 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19262 )],
19263 "rust-workers".to_string(),
19264 DEFAULT_CODEC.to_string(),
19265 None,
19266 )
19267 .expect_err("TimerFired requires TimerScheduled");
19268 assert!(matches!(
19269 lone_fire,
19270 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19271 if reason == "timer_schedule_missing_or_duplicate"
19272 ));
19273
19274 let wrong_identity = WorkflowState::new(
19275 vec![
19276 history_event(
19277 "TimerScheduled",
19278 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19279 ),
19280 history_event(
19281 "TimerFired",
19282 json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
19283 ),
19284 ],
19285 "rust-workers".to_string(),
19286 DEFAULT_CODEC.to_string(),
19287 None,
19288 )
19289 .expect_err("fire must match scheduled timer identity");
19290 assert!(matches!(
19291 wrong_identity,
19292 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19293 if reason == "timer_identity_mismatch"
19294 ));
19295
19296 let duplicate_fire = WorkflowState::new(
19297 vec![
19298 history_event(
19299 "TimerScheduled",
19300 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19301 ),
19302 history_event(
19303 "TimerFired",
19304 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19305 ),
19306 history_event(
19307 "TimerFired",
19308 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19309 ),
19310 ],
19311 "rust-workers".to_string(),
19312 DEFAULT_CODEC.to_string(),
19313 None,
19314 )
19315 .expect_err("a durable timer cannot fire twice");
19316 assert!(matches!(
19317 duplicate_fire,
19318 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19319 if reason == "duplicate_timer_fire"
19320 ));
19321
19322 let wrong_fired_delay = WorkflowState::new(
19323 vec![
19324 history_event(
19325 "TimerScheduled",
19326 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19327 ),
19328 history_event(
19329 "TimerFired",
19330 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 6}),
19331 ),
19332 ],
19333 "rust-workers".to_string(),
19334 DEFAULT_CODEC.to_string(),
19335 None,
19336 )
19337 .expect_err("timer schedule and fire delays must agree");
19338 assert!(matches!(
19339 wrong_fired_delay,
19340 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19341 if reason == "timer_history_delay_mismatch"
19342 ));
19343 }
19344
19345 #[test]
19346 fn replay_rejects_activity_moved_before_recorded_timer() {
19347 let ctx = workflow_context(vec![
19348 history_event(
19349 "TimerScheduled",
19350 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19351 ),
19352 history_event(
19353 "TimerFired",
19354 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19355 ),
19356 history_event(
19357 "ActivityCompleted",
19358 json!({
19359 "sequence": 2,
19360 "activity_type": "after-timer",
19361 "payload_codec": DEFAULT_CODEC,
19362 "result": fixture_envelope(json!("done")),
19363 }),
19364 ),
19365 ]);
19366 let mut activity = Box::pin(ctx.activity("after-timer", json!([])));
19367 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19368
19369 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
19370 activity.as_mut().poll(&mut task_context)
19371 else {
19372 panic!("reordered durable command must be rejected");
19373 };
19374 assert_eq!(failure.reason, "recorded_command_mismatch");
19375 assert_eq!(failure.sequence, Some(1));
19376 assert_eq!(failure.expected.as_deref(), Some("timer"));
19377 assert_eq!(failure.actual.as_deref(), Some("activity:after-timer"));
19378 }
19379
19380 #[test]
19381 fn workflow_context_emits_a_typed_named_signal_wait() {
19382 let ctx = workflow_context(Vec::new());
19383 let mut signal = Box::pin(ctx.wait_signal("finish"));
19384 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19385
19386 assert!(matches!(
19387 signal.as_mut().poll(&mut task_context),
19388 Poll::Pending
19389 ));
19390 assert_eq!(
19391 ctx.take_commands().expect("signal-wait command"),
19392 vec![json!({
19393 "type": "open_signal_wait",
19394 "signal_name": "finish",
19395 })]
19396 );
19397 }
19398
19399 #[test]
19400 fn runtime_message_stream_transport_cannot_be_opened_as_a_user_signal() {
19401 let ctx = workflow_context(Vec::new());
19402 let mut signal = Box::pin(ctx.wait_signal(MESSAGE_STREAM_SIGNAL));
19403 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19404
19405 let Poll::Ready(Err(Error::Codec(message))) = signal.as_mut().poll(&mut task_context)
19406 else {
19407 panic!("runtime-reserved signal should be rejected");
19408 };
19409 assert!(message.contains("reserved by the workflow runtime"));
19410 assert!(ctx.take_commands().expect("commands").is_empty());
19411 }
19412
19413 #[tokio::test]
19414 async fn runtime_message_stream_transport_cannot_be_sent_as_a_user_signal() {
19415 let client = Client::builder("http://127.0.0.1:9")
19416 .build()
19417 .expect("client");
19418 let error = client
19419 .signal_workflow("workflow-1", MESSAGE_STREAM_SIGNAL, json!(["forged"]))
19420 .await
19421 .expect_err("runtime-reserved signal should be rejected before transport");
19422
19423 assert!(
19424 matches!(error, Error::Codec(ref message) if message.contains("reserved by the workflow runtime"))
19425 );
19426 }
19427
19428 #[test]
19429 fn message_stream_worker_task_consumes_current_contiguous_bounded_batch() {
19430 fn delivery(message_id: &str, position: u64, value: &str) -> Value {
19431 let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
19432 value.to_string(),
19433 )]))
19434 .expect("message payload");
19435 json!({
19436 "schema": MESSAGE_STREAM_SCHEMA,
19437 "stream_name": "orders",
19438 "message_id": message_id,
19439 "position": position,
19440 "payload_envelope": payload,
19441 })
19442 }
19443
19444 fn opened(sequence: u64) -> HistoryEvent {
19445 history_event(
19446 "SignalWaitOpened",
19447 json!({
19448 "sequence": sequence,
19449 "signal_name": MESSAGE_STREAM_SIGNAL,
19450 }),
19451 )
19452 }
19453
19454 fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
19455 history_event(
19456 "SignalApplied",
19457 json!({
19458 "sequence": sequence,
19459 "signal_name": MESSAGE_STREAM_SIGNAL,
19460 "value": fixture_envelope(json!([delivery])),
19461 }),
19462 )
19463 }
19464
19465 fn received(delivery: Value) -> HistoryEvent {
19466 history_event(
19467 "SignalReceived",
19468 json!({
19469 "signal_name": MESSAGE_STREAM_SIGNAL,
19470 "arguments": fixture_envelope(json!([delivery])),
19471 "payload_codec": DEFAULT_CODEC,
19472 }),
19473 )
19474 }
19475
19476 let client = Client::new("http://127.0.0.1:8080").expect("client");
19477 let mut worker = Worker::new(client, "rust-workers");
19478 worker.register_workflow("rust.message-stream-batch", |ctx, _input| async move {
19479 let messages = ctx.message_stream("orders")?.receive(2).await?;
19480 Ok(json!(messages
19481 .into_iter()
19482 .map(|message| message.message_id)
19483 .collect::<Vec<_>>()))
19484 });
19485
19486 let first = delivery("message-1", 1, "one");
19487 let second = delivery("message-2", 2, "two");
19488 let batch = worker
19489 .execute_workflow_task_decision(workflow_task(
19490 "rust.message-stream-batch",
19491 vec![
19492 opened(1),
19493 received(first.clone()),
19494 applied(1, first.clone()),
19495 received(first.clone()),
19496 received(second),
19497 ],
19498 DEFAULT_CODEC,
19499 ))
19500 .expect("worker task consumes the available batch");
19501
19502 assert_eq!(batch.commands.len(), 1);
19503 assert_eq!(batch.commands[0]["type"], "complete_workflow");
19504 assert_eq!(
19505 decode_wire_value(&batch.commands[0]["result"], DEFAULT_CODEC)
19506 .expect("workflow result"),
19507 json!(["message-1", "message-2"])
19508 );
19509 assert_eq!(
19510 batch.message_stream_cursors,
19511 vec![json!({"stream_name": "orders", "through_position": 2})]
19512 );
19513 assert!(batch.message_stream_waits.is_empty());
19514
19515 let partial = worker
19516 .execute_workflow_task_decision(workflow_task(
19517 "rust.message-stream-batch",
19518 vec![opened(1), received(first.clone()), applied(1, first)],
19519 DEFAULT_CODEC,
19520 ))
19521 .expect("worker task returns without waiting for a missing second item");
19522 assert_eq!(partial.commands.len(), 1);
19523 assert_eq!(partial.commands[0]["type"], "complete_workflow");
19524 assert_eq!(
19525 decode_wire_value(&partial.commands[0]["result"], DEFAULT_CODEC)
19526 .expect("workflow result"),
19527 json!(["message-1"])
19528 );
19529 assert_eq!(
19530 partial.message_stream_cursors,
19531 vec![json!({"stream_name": "orders", "through_position": 1})]
19532 );
19533 assert!(partial.message_stream_waits.is_empty());
19534 }
19535
19536 #[test]
19537 fn message_stream_replay_preserves_partial_batch_boundary_before_later_wait() {
19538 fn delivery(message_id: &str, position: u64, value: &str) -> Value {
19539 let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
19540 value.to_string(),
19541 )]))
19542 .expect("message payload");
19543 json!({
19544 "schema": MESSAGE_STREAM_SCHEMA,
19545 "stream_name": "orders",
19546 "message_id": message_id,
19547 "position": position,
19548 "payload_envelope": payload,
19549 })
19550 }
19551
19552 fn opened(sequence: u64) -> HistoryEvent {
19553 history_event(
19554 "SignalWaitOpened",
19555 json!({
19556 "sequence": sequence,
19557 "signal_name": MESSAGE_STREAM_SIGNAL,
19558 }),
19559 )
19560 }
19561
19562 fn received(delivery: Value) -> HistoryEvent {
19563 history_event(
19564 "SignalReceived",
19565 json!({
19566 "signal_name": MESSAGE_STREAM_SIGNAL,
19567 "arguments": fixture_envelope(json!([delivery])),
19568 "payload_codec": DEFAULT_CODEC,
19569 }),
19570 )
19571 }
19572
19573 fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
19574 history_event(
19575 "SignalApplied",
19576 json!({
19577 "sequence": sequence,
19578 "signal_name": MESSAGE_STREAM_SIGNAL,
19579 "value": fixture_envelope(json!([delivery])),
19580 }),
19581 )
19582 }
19583
19584 let client = Client::new("http://127.0.0.1:8080").expect("client");
19585 let mut worker = Worker::new(client, "rust-workers");
19586 worker.register_workflow(
19587 "rust.message-stream-partial-batches",
19588 |ctx, _input| async move {
19589 let stream = ctx.message_stream("orders")?;
19590 let first = stream.receive(10).await?;
19591 let second = stream.receive(10).await?;
19592 Ok(json!([
19593 first
19594 .into_iter()
19595 .map(|message| message.message_id)
19596 .collect::<Vec<_>>(),
19597 second
19598 .into_iter()
19599 .map(|message| message.message_id)
19600 .collect::<Vec<_>>(),
19601 ]))
19602 },
19603 );
19604
19605 let first = delivery("message-1", 1, "one");
19606 let second = delivery("message-2", 2, "two");
19607 let decision = worker
19608 .execute_workflow_task_decision(workflow_task(
19609 "rust.message-stream-partial-batches",
19610 vec![
19611 opened(1),
19612 received(first.clone()),
19613 applied(1, first),
19614 opened(2),
19615 received(second.clone()),
19616 applied(2, second),
19617 ],
19618 DEFAULT_CODEC,
19619 ))
19620 .expect("cold replay preserves both authored receive boundaries");
19621
19622 assert_eq!(decision.commands.len(), 1);
19623 assert_eq!(decision.commands[0]["type"], "complete_workflow");
19624 assert_eq!(
19625 decode_wire_value(&decision.commands[0]["result"], DEFAULT_CODEC)
19626 .expect("workflow result"),
19627 json!([["message-1"], ["message-2"]])
19628 );
19629 assert_eq!(
19630 decision.message_stream_cursors,
19631 vec![json!({"stream_name": "orders", "through_position": 2})]
19632 );
19633 assert!(decision.message_stream_waits.is_empty());
19634 }
19635
19636 #[test]
19637 fn empty_message_stream_opens_internal_signal_wait_and_reports_position() {
19638 let ctx = workflow_context(Vec::new());
19639 let stream = ctx.message_stream("orders").expect("message stream");
19640 let mut receive = Box::pin(stream.receive(10));
19641 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19642
19643 assert!(matches!(
19644 receive.as_mut().poll(&mut task_context),
19645 Poll::Pending
19646 ));
19647 assert_eq!(
19648 ctx.take_commands().expect("message-stream wait command"),
19649 vec![json!({
19650 "type": "open_signal_wait",
19651 "signal_name": MESSAGE_STREAM_SIGNAL,
19652 })]
19653 );
19654 let (cursors, waits) = ctx.message_stream_metadata().expect("stream metadata");
19655 assert!(cursors.is_empty());
19656 assert_eq!(
19657 waits,
19658 vec![json!({"stream_name": "orders", "after_position": 0})]
19659 );
19660 }
19661
19662 #[test]
19663 fn continue_as_new_cursor_checkpoint_preserves_global_pending_position() {
19664 let ctx = workflow_context(vec![history_event(
19665 "SignalReceived",
19666 json!({
19667 "signal_name": MESSAGE_STREAM_SIGNAL,
19668 "arguments": fixture_envelope(json!([{
19669 "schema": MESSAGE_STREAM_CURSOR_SCHEMA,
19670 "stream_name": "orders",
19671 "through_position": 2,
19672 }])),
19673 "payload_codec": DEFAULT_CODEC,
19674 }),
19675 )]);
19676 let stream = ctx.message_stream("orders").expect("message stream");
19677 let mut receive = Box::pin(stream.receive(10));
19678 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19679
19680 assert!(matches!(
19681 receive.as_mut().poll(&mut task_context),
19682 Poll::Pending
19683 ));
19684 let (cursors, waits) = ctx.message_stream_metadata().expect("stream metadata");
19685 assert_eq!(
19686 cursors,
19687 vec![json!({"stream_name": "orders", "through_position": 2})]
19688 );
19689 assert_eq!(
19690 waits,
19691 vec![json!({"stream_name": "orders", "after_position": 2})]
19692 );
19693 }
19694
19695 #[test]
19696 fn message_stream_delivery_preserves_typed_avro_arguments_across_replay() {
19697 let mut empty_map = BTreeMap::new();
19698 let mut nested = BTreeMap::new();
19699 nested.insert(
19700 "value".to_string(),
19701 AvroValue::Array(vec![AvroValue::Bytes(b"nested".to_vec())]),
19702 );
19703 let values = vec![
19704 AvroValue::Bytes(vec![0, 255]),
19705 AvroValue::Long(1),
19706 AvroValue::Double(1.0),
19707 AvroValue::Array(Vec::new()),
19708 AvroValue::Map(std::mem::take(&mut empty_map)),
19709 AvroValue::Map(nested),
19710 ];
19711 let payload = encode_avro_value(&AvroValue::Array(values.clone())).expect("payload");
19712 let transport = vec![json!({
19713 "schema": MESSAGE_STREAM_SCHEMA,
19714 "stream_name": "orders",
19715 "message_id": "message-1",
19716 "position": 1,
19717 "payload_envelope": payload,
19718 })];
19719
19720 for _ in 0..2 {
19721 let Some(MessageStreamDelivery::Message(message)) =
19722 decode_message_stream_delivery(transport.clone()).expect("delivery")
19723 else {
19724 panic!("message delivery expected");
19725 };
19726 assert_eq!(message.arguments, values);
19727 assert!(matches!(message.arguments[1], AvroValue::Long(1)));
19728 assert!(matches!(message.arguments[2], AvroValue::Double(1.0)));
19729 }
19730 }
19731
19732 #[test]
19733 fn cold_worker_replacement_consumes_message_stream_wait_arrivals_once_in_order() {
19734 fn delivery(message_id: &str, position: u64, value: &str) -> Value {
19735 let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
19736 value.to_string(),
19737 )]))
19738 .expect("message payload");
19739 json!({
19740 "schema": MESSAGE_STREAM_SCHEMA,
19741 "stream_name": "orders",
19742 "message_id": message_id,
19743 "position": position,
19744 "payload_envelope": payload,
19745 })
19746 }
19747
19748 fn opened(sequence: u64) -> HistoryEvent {
19749 history_event(
19750 "SignalWaitOpened",
19751 json!({
19752 "sequence": sequence,
19753 "signal_name": MESSAGE_STREAM_SIGNAL,
19754 }),
19755 )
19756 }
19757
19758 fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
19759 history_event(
19760 "SignalApplied",
19761 json!({
19762 "sequence": sequence,
19763 "signal_name": MESSAGE_STREAM_SIGNAL,
19764 "value": fixture_envelope(json!([delivery])),
19765 }),
19766 )
19767 }
19768
19769 fn worker() -> Worker {
19770 let client = Client::new("http://127.0.0.1:8080").expect("client");
19771 let mut worker = Worker::new(client, "rust-workers");
19772 worker.register_workflow("rust.message-stream", |ctx, _input| async move {
19773 let stream = ctx.message_stream("orders")?;
19774 let first = stream.receive_one().await?;
19775 let second = stream.receive_one().await?;
19776 Ok(json!([first.message_id, second.message_id]))
19777 });
19778 worker
19779 }
19780
19781 fn task_with_resume(history: Vec<HistoryEvent>, delivery: Value) -> WorkflowTask {
19782 let mut task = workflow_task("rust.message-stream", history, DEFAULT_CODEC);
19783 task.signal_name = Some(MESSAGE_STREAM_SIGNAL.to_string());
19784 task.signal_arguments = Some(fixture_envelope(json!([delivery])));
19785 task
19786 }
19787
19788 let waiting = worker()
19789 .execute_workflow_task_decision(workflow_task(
19790 "rust.message-stream",
19791 Vec::new(),
19792 DEFAULT_CODEC,
19793 ))
19794 .expect("first worker opens the stream wait");
19795 assert_eq!(
19796 waiting.commands,
19797 vec![json!({
19798 "type": "open_signal_wait",
19799 "signal_name": MESSAGE_STREAM_SIGNAL,
19800 })]
19801 );
19802 assert!(waiting.message_stream_cursors.is_empty());
19803 assert_eq!(
19804 waiting.message_stream_waits,
19805 vec![json!({"stream_name": "orders", "after_position": 0})]
19806 );
19807
19808 let first_delivery = delivery("message-1", 1, "one");
19809 let first_arrival = worker()
19810 .execute_workflow_task_decision(task_with_resume(
19811 vec![opened(1)],
19812 first_delivery.clone(),
19813 ))
19814 .expect("replacement worker consumes the first arrival");
19815 assert_eq!(
19816 first_arrival.commands,
19817 vec![json!({
19818 "type": "open_signal_wait",
19819 "signal_name": MESSAGE_STREAM_SIGNAL,
19820 })]
19821 );
19822 assert_eq!(
19823 first_arrival.message_stream_cursors,
19824 vec![json!({"stream_name": "orders", "through_position": 1})]
19825 );
19826 assert_eq!(
19827 first_arrival.message_stream_waits,
19828 vec![json!({"stream_name": "orders", "after_position": 1})]
19829 );
19830
19831 let second_delivery = delivery("message-2", 2, "two");
19832 let first_applied = applied(1, first_delivery);
19833 let completed = worker()
19834 .execute_workflow_task_decision(task_with_resume(
19835 vec![opened(1), first_applied.clone(), opened(2)],
19836 second_delivery.clone(),
19837 ))
19838 .expect("next replacement worker consumes the second arrival");
19839 assert_eq!(completed.commands.len(), 1);
19840 assert_eq!(completed.commands[0]["type"], "complete_workflow");
19841 assert_eq!(
19842 decode_wire_value(&completed.commands[0]["result"], DEFAULT_CODEC)
19843 .expect("workflow result"),
19844 json!(["message-1", "message-2"])
19845 );
19846 assert_eq!(
19847 completed.message_stream_cursors,
19848 vec![json!({"stream_name": "orders", "through_position": 2})]
19849 );
19850 assert!(completed.message_stream_waits.is_empty());
19851
19852 let replay_history = vec![
19853 opened(1),
19854 first_applied,
19855 opened(2),
19856 applied(2, second_delivery),
19857 ];
19858 for _cold_worker_or_restart in 0..2 {
19859 let replayed = worker()
19860 .execute_workflow_task_decision(workflow_task(
19861 "rust.message-stream",
19862 replay_history.clone(),
19863 DEFAULT_CODEC,
19864 ))
19865 .expect("cold worker replays each logical message exactly once");
19866 assert_eq!(replayed.commands.len(), 1);
19867 assert_eq!(
19868 decode_wire_value(&replayed.commands[0]["result"], DEFAULT_CODEC)
19869 .expect("replayed workflow result"),
19870 json!(["message-1", "message-2"])
19871 );
19872 assert_eq!(
19873 replayed.message_stream_cursors,
19874 vec![json!({"stream_name": "orders", "through_position": 2})]
19875 );
19876 assert!(replayed.message_stream_waits.is_empty());
19877 }
19878 }
19879
19880 #[test]
19881 fn message_stream_capability_and_completion_require_protocol_one_fifteen() {
19882 assert!(!worker_protocol_supports_message_streams("1.14"));
19883 assert!(worker_protocol_supports_message_streams("1.15"));
19884 assert!(worker_protocol_supports_message_streams("1.16"));
19885 assert!(worker_protocol_supports_message_streams(
19886 WORKER_PROTOCOL_VERSION
19887 ));
19888 assert_eq!(MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION, "1.15");
19889 }
19890
19891 #[test]
19892 fn condition_wait_history_cannot_be_consumed_as_a_typed_signal_wait() {
19893 let ctx = workflow_context(vec![
19894 history_event(
19895 "ConditionWaitOpened",
19896 json!({
19897 "sequence": 1,
19898 "condition_wait_id": "condition:1",
19899 "condition_wait_occurrence_id": "rust:condition-wait:0",
19900 "condition_key": "signal:finish",
19901 "condition_definition_fingerprint": "sha256:signal-finish-v1",
19902 }),
19903 ),
19904 history_event(
19905 "ConditionWaitSatisfied",
19906 json!({
19907 "sequence": 1,
19908 "condition_wait_id": "condition:1",
19909 "condition_wait_occurrence_id": "rust:condition-wait:0",
19910 "condition_key": "signal:finish",
19911 "condition_definition_fingerprint": "sha256:signal-finish-v1",
19912 }),
19913 ),
19914 history_event(
19915 "SignalReceived",
19916 json!({"signal_name": "finish", "arguments": []}),
19917 ),
19918 ]);
19919 let mut signal = Box::pin(ctx.wait_signal("finish"));
19920 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19921
19922 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
19923 signal.as_mut().poll(&mut task_context)
19924 else {
19925 panic!("condition history must not resolve as a typed signal wait");
19926 };
19927 assert_eq!(failure.reason, "recorded_command_mismatch");
19928 assert_eq!(failure.expected.as_deref(), Some("condition wait"));
19929 }
19930
19931 #[test]
19932 fn replay_orders_signal_waits_and_timers_in_one_command_stream() {
19933 let signal_then_timer = vec![
19934 history_event(
19935 "SignalWaitOpened",
19936 json!({"sequence": 1, "signal_name": "go"}),
19937 ),
19938 history_event(
19939 "SignalApplied",
19940 json!({
19941 "sequence": 1,
19942 "signal_name": "go",
19943 "value": fixture_envelope(json!(["now"])),
19944 }),
19945 ),
19946 history_event(
19947 "TimerScheduled",
19948 json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
19949 ),
19950 history_event(
19951 "TimerFired",
19952 json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
19953 ),
19954 ];
19955
19956 let ctx = workflow_context(signal_then_timer.clone());
19957 let mut signal = Box::pin(ctx.wait_signal("go"));
19958 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19959 assert!(matches!(
19960 signal.as_mut().poll(&mut task_context),
19961 Poll::Ready(Ok(arguments)) if arguments == vec![json!("now")]
19962 ));
19963 let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5)));
19964 assert!(matches!(
19965 timer.as_mut().poll(&mut task_context),
19966 Poll::Ready(Ok(()))
19967 ));
19968 ctx.ensure_history_consumed()
19969 .expect("signal and timer history consumed in order");
19970
19971 let reordered = workflow_context(signal_then_timer);
19972 let mut timer_first = Box::pin(reordered.sleep(Duration::from_secs(5)));
19973 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
19974 timer_first.as_mut().poll(&mut task_context)
19975 else {
19976 panic!("timer cannot consume signal-wait-first history");
19977 };
19978 assert_eq!(failure.reason, "recorded_command_mismatch");
19979 assert_eq!(failure.sequence, Some(1));
19980 assert_eq!(failure.expected.as_deref(), Some("signal wait"));
19981
19982 let timer_then_signal = vec![
19983 history_event(
19984 "TimerScheduled",
19985 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19986 ),
19987 history_event(
19988 "TimerFired",
19989 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19990 ),
19991 history_event(
19992 "SignalWaitOpened",
19993 json!({"sequence": 2, "signal_name": "go"}),
19994 ),
19995 history_event(
19996 "SignalApplied",
19997 json!({
19998 "sequence": 2,
19999 "signal_name": "go",
20000 "value": fixture_envelope(json!([])),
20001 }),
20002 ),
20003 ];
20004 let reordered = workflow_context(timer_then_signal);
20005 let mut signal_first = Box::pin(reordered.wait_signal("go"));
20006 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
20007 signal_first.as_mut().poll(&mut task_context)
20008 else {
20009 panic!("signal wait cannot consume timer-first history");
20010 };
20011 assert_eq!(failure.reason, "recorded_command_mismatch");
20012 assert_eq!(failure.sequence, Some(1));
20013 assert_eq!(failure.expected.as_deref(), Some("timer"));
20014 }
20015
20016 #[test]
20017 fn workflow_history_rejects_duplicate_or_colliding_command_sequences() {
20018 let duplicate_timer = WorkflowState::new(
20019 vec![
20020 history_event(
20021 "TimerScheduled",
20022 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20023 ),
20024 history_event(
20025 "TimerScheduled",
20026 json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
20027 ),
20028 ],
20029 "rust-workers".to_string(),
20030 DEFAULT_CODEC.to_string(),
20031 None,
20032 )
20033 .expect_err("one workflow sequence cannot schedule two timers");
20034 assert!(matches!(
20035 duplicate_timer,
20036 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20037 if reason == "timer_schedule_missing_or_duplicate"
20038 ));
20039
20040 let colliding_kinds = WorkflowState::new(
20041 vec![
20042 history_event(
20043 "TimerScheduled",
20044 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20045 ),
20046 history_event(
20047 "ActivityCompleted",
20048 json!({"sequence": 1, "activity_type": "same-sequence"}),
20049 ),
20050 ],
20051 "rust-workers".to_string(),
20052 DEFAULT_CODEC.to_string(),
20053 None,
20054 )
20055 .expect_err("one workflow sequence cannot identify two command kinds");
20056 assert!(matches!(
20057 colliding_kinds,
20058 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20059 if reason == "durable_command_sequence_collision"
20060 ));
20061
20062 let duplicate_signal_wait = WorkflowState::new(
20063 vec![
20064 history_event(
20065 "SignalWaitOpened",
20066 json!({"sequence": 1, "signal_name": "go"}),
20067 ),
20068 history_event(
20069 "SignalWaitOpened",
20070 json!({"sequence": 1, "signal_name": "go"}),
20071 ),
20072 ],
20073 "rust-workers".to_string(),
20074 DEFAULT_CODEC.to_string(),
20075 None,
20076 )
20077 .expect_err("one workflow sequence cannot open two signal waits");
20078 assert!(matches!(
20079 duplicate_signal_wait,
20080 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20081 if reason == "signal_wait_open_missing_or_duplicate"
20082 ));
20083 }
20084
20085 #[test]
20086 fn workflow_history_accepts_a_first_command_after_global_sequence_gaps() {
20087 let result = encode_value_envelope(&json!({"captured": true}), DEFAULT_CODEC)
20088 .expect("side-effect result");
20089 let ctx = workflow_context(vec![history_event(
20090 "SideEffectRecorded",
20091 json!({"sequence": 99, "result": result}),
20092 )]);
20093
20094 let replayed: Value = ctx
20095 .side_effect(|| panic!("recorded side effect must not run"))
20096 .expect("positive global workflow sequence is valid");
20097 assert_eq!(replayed, json!({"captured": true}));
20098 ctx.ensure_history_consumed().expect("history consumed");
20099 }
20100
20101 #[test]
20102 fn workflow_history_rejects_zero_and_descending_command_sequences() {
20103 let result =
20104 encode_value_envelope(&json!("captured"), DEFAULT_CODEC).expect("side-effect result");
20105 let zero = WorkflowState::new(
20106 vec![history_event(
20107 "SideEffectRecorded",
20108 json!({"sequence": 0, "result": result.clone()}),
20109 )],
20110 "rust-workers".to_string(),
20111 DEFAULT_CODEC.to_string(),
20112 None,
20113 )
20114 .expect_err("durable command sequences must be positive");
20115 assert!(matches!(
20116 zero,
20117 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20118 if reason == "durable_command_sequence_invalid"
20119 ));
20120
20121 let descending = WorkflowState::new(
20122 vec![
20123 history_event(
20124 "SideEffectRecorded",
20125 json!({"sequence": 3, "result": result}),
20126 ),
20127 history_event(
20128 "VersionMarkerRecorded",
20129 json!({
20130 "sequence": 2,
20131 "change_id": "descending-marker",
20132 "version": 1,
20133 "min_supported": 1,
20134 "max_supported": 1,
20135 }),
20136 ),
20137 ],
20138 "rust-workers".to_string(),
20139 DEFAULT_CODEC.to_string(),
20140 None,
20141 )
20142 .expect_err("new durable commands must remain strictly ordered");
20143 let Error::NonDeterministicReplay(failure) = descending else {
20144 panic!("expected typed replay failure");
20145 };
20146 assert_eq!(failure.reason, "durable_command_sequence_mismatch");
20147 assert_eq!(failure.sequence, Some(2));
20148 assert_eq!(
20149 failure.expected.as_deref(),
20150 Some("workflow sequence greater than 3")
20151 );
20152 assert_eq!(failure.actual.as_deref(), Some("2"));
20153 }
20154
20155 #[test]
20156 fn workflow_task_replay_completes_after_signals_create_sequence_gaps() {
20157 fn worker() -> Worker {
20158 let client = Client::new("http://127.0.0.1:8080").expect("client");
20159 let mut worker = Worker::new(client, "rust-workers");
20160 worker.register_workflow("rust.finish-after-gaps", |ctx, _input| async move {
20161 ctx.wait_signal("finish").await?;
20162 let marker: String =
20163 ctx.side_effect(|| panic!("recorded side effect must not run"))?;
20164 assert_eq!(marker, "after-finish");
20165 Ok(json!("finished"))
20166 });
20167 worker
20168 }
20169
20170 let marker = encode_value_envelope(&json!("after-finish"), DEFAULT_CODEC)
20171 .expect("side-effect result");
20172 let task = workflow_task(
20173 "rust.finish-after-gaps",
20174 vec![
20175 history_event(
20176 "SignalWaitOpened",
20177 json!({"sequence": 1, "signal_name": "finish"}),
20178 ),
20179 history_event(
20180 "SignalReceived",
20181 json!({
20182 "signal_id": "increment-3",
20183 "signal_name": "increment",
20184 "workflow_sequence": 2,
20185 "payload_codec": DEFAULT_CODEC,
20186 "arguments": fixture_envelope(json!([3])),
20187 }),
20188 ),
20189 history_event(
20190 "SignalReceived",
20191 json!({
20192 "signal_id": "increment-5",
20193 "signal_name": "increment",
20194 "workflow_sequence": 3,
20195 "payload_codec": DEFAULT_CODEC,
20196 "arguments": fixture_envelope(json!([5])),
20197 }),
20198 ),
20199 history_event(
20200 "SignalReceived",
20201 json!({
20202 "signal_id": "finish",
20203 "signal_name": "finish",
20204 "workflow_sequence": 4,
20205 "payload_codec": DEFAULT_CODEC,
20206 "arguments": fixture_envelope(json!([])),
20207 }),
20208 ),
20209 history_event(
20210 "SignalApplied",
20211 json!({
20212 "sequence": 1,
20213 "signal_id": "finish",
20214 "signal_name": "finish",
20215 "payload_codec": DEFAULT_CODEC,
20216 "value": fixture_envelope(json!([])),
20217 }),
20218 ),
20219 history_event(
20220 "SideEffectRecorded",
20221 json!({"sequence": 5, "result": marker}),
20222 ),
20223 ],
20224 DEFAULT_CODEC,
20225 );
20226
20227 for _original_or_cold_worker in 0..2 {
20228 let commands = worker()
20229 .execute_workflow_task(task.clone())
20230 .expect("signal gaps preserve deterministic replay");
20231 assert_eq!(commands.len(), 1, "replay emits only terminal completion");
20232 assert_eq!(commands[0]["type"], "complete_workflow");
20233 assert_eq!(
20234 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
20235 json!("finished")
20236 );
20237 }
20238 }
20239
20240 #[test]
20241 fn workflow_sleep_rejects_unrepresentable_rounded_duration() {
20242 let ctx = workflow_context(Vec::new());
20243 let mut sleep = Box::pin(ctx.start_timer(Duration::new(u64::MAX, 1)));
20244 let mut task_context = TaskContext::from_waker(noop_waker_ref());
20245 assert!(matches!(
20246 sleep.as_mut().poll(&mut task_context),
20247 Poll::Ready(Err(Error::TimerDurationOverflow))
20248 ));
20249 assert!(ctx.take_commands().expect("commands").is_empty());
20250 }
20251
20252 #[test]
20253 fn workflow_memo_update_emits_canonical_command_and_replays_once() {
20254 let entries = AvroValue::Map(BTreeMap::from([
20255 ("text".to_string(), AvroValue::String("same".to_string())),
20256 (
20257 "nested".to_string(),
20258 AvroValue::Map(BTreeMap::from([
20259 ("beta".to_string(), AvroValue::Long(2)),
20260 ("alpha".to_string(), AvroValue::Long(1)),
20261 ])),
20262 ),
20263 ("long".to_string(), AvroValue::Long(7)),
20264 ("double".to_string(), AvroValue::Double(7.0)),
20265 ("binary".to_string(), AvroValue::Bytes(b"same".to_vec())),
20266 ]));
20267 let ctx = workflow_context(Vec::new());
20268 ctx.upsert_memo(entries.clone()).expect("valid memo update");
20269 let commands = ctx.take_commands().expect("commands");
20270
20271 assert_eq!(commands.len(), 1);
20272 assert_eq!(commands[0]["type"], "upsert_memo");
20273 let server_entries = json!({
20274 "codec": "avro",
20275 "blob": "wwHioz3/VYAiNw4KDGJpbmFyeQgIc2FtZQxkb3VibGUGAAAAAAAAHEAIbG9uZwQODG5lc3RlZA4ECmFscGhhBAIIYmV0YQQEAAh0ZXh0CghzYW1lAA==",
20276 });
20277 assert_eq!(
20278 commands[0]["entries"]
20279 .as_object()
20280 .expect("entries envelope")
20281 .keys()
20282 .collect::<Vec<_>>(),
20283 vec!["blob", "codec"]
20284 );
20285 assert_eq!(commands[0]["entries"], server_entries);
20286 let wire_entries =
20287 decode_wire_avro_value(&commands[0]["entries"], DEFAULT_CODEC).expect("memo entries");
20288 assert_eq!(wire_entries, entries);
20289
20290 let history = vec![history_event(
20291 "MemoUpserted",
20292 json!({
20293 "sequence": 1,
20294 "entries": server_entries.clone(),
20295 "merged": server_entries,
20296 }),
20297 )];
20298 let replay = workflow_context(history.clone());
20299 replay
20300 .upsert_memo(entries.clone())
20301 .expect("matching replay identity");
20302 assert!(replay.take_commands().expect("replay commands").is_empty());
20303
20304 let changed_types = AvroValue::Map(BTreeMap::from([
20305 ("text".to_string(), AvroValue::Bytes(b"same".to_vec())),
20306 (
20307 "nested".to_string(),
20308 AvroValue::Map(BTreeMap::from([
20309 ("alpha".to_string(), AvroValue::Long(1)),
20310 ("beta".to_string(), AvroValue::Long(2)),
20311 ])),
20312 ),
20313 ("long".to_string(), AvroValue::Double(7.0)),
20314 ("double".to_string(), AvroValue::Long(7)),
20315 ("binary".to_string(), AvroValue::String("same".to_string())),
20316 ]));
20317 let error = workflow_context(history)
20318 .upsert_memo(changed_types)
20319 .expect_err("memo replay identity must preserve Avro value types");
20320 assert!(matches!(
20321 error,
20322 Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20323 ));
20324 }
20325
20326 #[test]
20327 fn workflow_memo_update_rejects_changed_replay_identity_and_invalid_keys() {
20328 let original = encode_value_envelope(&json!({"stage": "original"}), DEFAULT_CODEC)
20329 .expect("memo envelope");
20330 let replay = workflow_context(vec![history_event(
20331 "MemoUpserted",
20332 json!({
20333 "sequence": 1,
20334 "entries": original.clone(),
20335 "merged": original
20336 }),
20337 )]);
20338 let error = replay
20339 .upsert_memo(json!({"stage": "changed"}))
20340 .expect_err("changed memo update must fail replay");
20341 assert!(matches!(
20342 error,
20343 Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20344 ));
20345
20346 let invalid = workflow_context(Vec::new())
20347 .upsert_memo(
20348 json!({"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx": true}),
20349 )
20350 .expect_err("oversized key");
20351 assert!(matches!(invalid, Error::InvalidMemoUpdate(_)));
20352 }
20353
20354 #[test]
20355 fn workflow_memo_replay_distinguishes_signed_zero_identity() {
20356 let negative_zero = AvroValue::Map(BTreeMap::from([(
20357 "reading".to_string(),
20358 AvroValue::Double(-0.0),
20359 )]));
20360 let negative_zero_envelope =
20361 encode_typed_envelope(&negative_zero, DEFAULT_CODEC).expect("negative zero envelope");
20362 let history = vec![history_event(
20363 "MemoUpserted",
20364 json!({
20365 "sequence": 1,
20366 "entries": negative_zero_envelope.clone(),
20367 "merged": negative_zero_envelope,
20368 }),
20369 )];
20370
20371 workflow_context(history.clone())
20372 .upsert_memo(negative_zero)
20373 .expect("matching negative-zero history identity");
20374
20375 let error = workflow_context(history)
20376 .upsert_memo(AvroValue::Map(BTreeMap::from([(
20377 "reading".to_string(),
20378 AvroValue::Double(0.0),
20379 )])))
20380 .expect_err("positive zero must not consume negative-zero memo history");
20381 assert!(matches!(
20382 error,
20383 Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20384 ));
20385 }
20386
20387 #[test]
20388 fn workflow_memo_capability_requires_flag_and_command_advertisement() {
20389 let supported = json!({
20390 "workflow_memo_updates": {"supported": true, "minimum_protocol_version": "1.14"},
20391 "supported_workflow_task_commands": ["complete_workflow", "upsert_memo"]
20392 });
20393 assert!(runtime_supports_workflow_memo_updates(Some(&supported)));
20394 assert!(!runtime_supports_workflow_memo_updates(Some(&json!({
20395 "workflow_memo_updates": {"supported": false},
20396 "supported_workflow_task_commands": ["upsert_memo"]
20397 }))));
20398 assert!(commands_use_workflow_memo_updates(&[json!({
20399 "type": "upsert_memo",
20400 "entries": {"stage": "processing"}
20401 })]));
20402 }
20403
20404 #[test]
20405 fn workflow_task_replay_completes_without_rescheduling_recorded_commands() {
20406 let client = Client::new("http://127.0.0.1:8080").expect("client");
20407 let mut worker = Worker::new(client, "rust-workers");
20408 worker.register_workflow("rust.timer", |ctx, _input| async move {
20409 ctx.sleep(Duration::from_secs(5)).await?;
20410 ctx.activity("after-timer", json!([])).await
20411 });
20412
20413 let task = |history_events| WorkflowTask {
20414 task_id: "wft-rust-timer-1".to_string(),
20415 workflow_command_id: None,
20416 workflow_id: Some("wf-rust-timer".to_string()),
20417 run_id: Some("run-rust-timer".to_string()),
20418 workflow_type: "rust.timer".to_string(),
20419 cancel_requested: false,
20420 payload_codec: DEFAULT_CODEC.to_string(),
20421 arguments: Some(
20422 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
20423 ),
20424 history_events,
20425 total_history_events: None,
20426 history_size_bytes: None,
20427 continue_as_new_recommended: None,
20428 history_budget_pressure: None,
20429 next_history_page_token: None,
20430 workflow_task_attempt: 1,
20431 workflow_signal_id: None,
20432 signal_name: None,
20433 signal_arguments: None,
20434 workflow_update_id: None,
20435 update_name: None,
20436 lease_owner: Some("rust-worker".to_string()),
20437 };
20438
20439 let initial = worker
20440 .execute_workflow_task(task(Vec::new()))
20441 .expect("initial timer task");
20442 assert_eq!(
20443 initial,
20444 vec![json!({"type": "start_timer", "delay_seconds": 5})]
20445 );
20446
20447 let activity_result =
20448 encode_value_envelope(&json!("done"), DEFAULT_CODEC).expect("activity result");
20449 let replayed = worker
20450 .execute_workflow_task(task(vec![
20451 history_event(
20452 "TimerScheduled",
20453 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20454 ),
20455 history_event(
20456 "TimerFired",
20457 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20458 ),
20459 history_event(
20460 "ActivityCompleted",
20461 json!({
20462 "sequence": 2,
20463 "activity_type": "after-timer",
20464 "payload_codec": DEFAULT_CODEC,
20465 "result": activity_result,
20466 }),
20467 ),
20468 ]))
20469 .expect("replayed workflow task");
20470 assert_eq!(replayed.len(), 1);
20471 assert_eq!(replayed[0]["type"], "complete_workflow");
20472 assert_eq!(
20473 decode_wire_value(&replayed[0]["result"], DEFAULT_CODEC).expect("result"),
20474 json!("done")
20475 );
20476 }
20477
20478 #[test]
20479 fn workflow_continue_as_new_emits_arguments_type_and_queue_once() {
20480 let client = Client::new("http://127.0.0.1:8080").expect("client");
20481 let mut worker = Worker::new(client, "rust-workers");
20482 worker.register_workflow("rust.continue", |ctx, _input| async move {
20483 ctx.continue_as_new_with_options(
20484 ContinueAsNewOptions::new()
20485 .workflow_type("rust.next")
20486 .task_queue("next-workers"),
20487 json!([2, {"cursor": "next"}]),
20488 )
20489 });
20490
20491 let commands = worker
20492 .execute_workflow_task(workflow_task("rust.continue", Vec::new(), DEFAULT_CODEC))
20493 .expect("continue-as-new command");
20494
20495 assert_eq!(commands.len(), 1);
20496 assert_eq!(commands[0]["type"], "continue_as_new");
20497 assert_eq!(commands[0]["workflow_type"], "rust.next");
20498 assert_eq!(commands[0]["queue"], "next-workers");
20499 assert_eq!(
20500 decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC)
20501 .expect("continue-as-new arguments"),
20502 json!([2, {"cursor": "next"}])
20503 );
20504 }
20505
20506 #[test]
20507 fn continue_as_new_preserves_typed_arguments() {
20508 let client = Client::new("http://127.0.0.1:8080").expect("client");
20509 let mut worker = Worker::new(client, "rust-workers");
20510 worker.register_workflow_avro_value("rust.typed-continue", |ctx, _input| async move {
20511 ctx.continue_as_new(AvroValue::Array(vec![typed_fidelity_probe()]))?;
20512 unreachable!("continue-as-new returns a control-flow error")
20513 });
20514
20515 let commands = worker
20516 .execute_workflow_task(workflow_task(
20517 "rust.typed-continue",
20518 Vec::new(),
20519 DEFAULT_CODEC,
20520 ))
20521 .expect("typed continue-as-new command");
20522
20523 assert_eq!(commands[0]["type"], "continue_as_new");
20524 assert_eq!(
20525 decode_wire_avro_value(&commands[0]["arguments"], DEFAULT_CODEC)
20526 .expect("typed continue arguments"),
20527 AvroValue::Array(vec![typed_fidelity_probe()])
20528 );
20529 }
20530
20531 #[test]
20532 fn recorded_continue_as_new_is_consumed_without_duplicate_successor_command() {
20533 let client = Client::new("http://127.0.0.1:8080").expect("client");
20534 let mut worker = Worker::new(client, "rust-workers");
20535 worker.register_workflow("rust.continue", |ctx, _input| async move {
20536 ctx.continue_as_new(json!([2]))
20537 });
20538 let task = workflow_task(
20539 "rust.continue",
20540 vec![history_event(
20541 "WorkflowContinuedAsNew",
20542 json!({"sequence": 1, "continued_to_run_id": "run-next"}),
20543 )],
20544 DEFAULT_CODEC,
20545 );
20546
20547 for _worker_restart_or_redelivery in 0..2 {
20548 let commands = worker
20549 .execute_workflow_task(task.clone())
20550 .expect("recorded transition replays");
20551 assert!(
20552 commands.is_empty(),
20553 "replay must not emit another successor"
20554 );
20555 }
20556 }
20557
20558 #[test]
20559 fn continue_as_new_rejects_invalid_overrides_before_emitting_a_command() {
20560 let ctx = workflow_context(Vec::new());
20561 let error = ctx
20562 .continue_as_new_with_options(ContinueAsNewOptions::new().task_queue(" "), json!([1]))
20563 .expect_err("blank queue must be rejected");
20564
20565 let Error::InvalidContinueAsNewOptions(error) = error else {
20566 panic!("expected typed continue-as-new validation error");
20567 };
20568 assert_eq!(error.field, "task_queue");
20569 assert!(ctx.take_commands().expect("commands").is_empty());
20570 }
20571
20572 #[test]
20573 fn workflow_context_exposes_server_history_budget() {
20574 let client = Client::new("http://127.0.0.1:8080").expect("client");
20575 let mut worker = Worker::new(client, "rust-workers");
20576 worker.register_workflow("rust.history-budget", |ctx, _input| async move {
20577 let budget = ctx.history_budget()?;
20578 Ok(json!({
20579 "events": budget.event_count,
20580 "bytes": budget.size_bytes,
20581 "recommended": budget.continue_as_new_recommended,
20582 "pressure": budget.pressure,
20583 }))
20584 });
20585 let task: WorkflowTask = serde_json::from_value(json!({
20586 "task_id": "task-history-budget",
20587 "workflow_type": "rust.history-budget",
20588 "payload_codec": DEFAULT_CODEC,
20589 "history_events": [],
20590 "total_history_events": 480,
20591 "history_size_bytes": 1_048_576,
20592 "continue_as_new_recommended": true,
20593 "history_budget_pressure": "continue_as_new_recommended",
20594 }))
20595 .expect("published workflow task");
20596
20597 let commands = worker
20598 .execute_workflow_task(task)
20599 .expect("history-budget workflow");
20600 let result = decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("result");
20601 assert_eq!(result["events"], 480);
20602 assert_eq!(result["bytes"], 1_048_576);
20603 assert_eq!(result["recommended"], true);
20604 assert_eq!(result["pressure"], "continue_as_new_recommended");
20605 }
20606
20607 #[test]
20608 fn uncaught_workflow_handler_error_emits_terminal_failure_command() {
20609 let client = Client::new("http://127.0.0.1:8080").expect("client");
20610 let mut worker = Worker::new(client, "rust-workers");
20611 worker.register_workflow("rust.failing", |_ctx, _input| async move {
20612 Err(Error::Codec("rust_conformance_failure".to_string()))
20613 });
20614 let task = WorkflowTask {
20615 task_id: "wft-rust-failing-1".to_string(),
20616 workflow_command_id: None,
20617 workflow_id: Some("wf-rust-failing".to_string()),
20618 run_id: Some("run-rust-failing".to_string()),
20619 workflow_type: "rust.failing".to_string(),
20620 cancel_requested: false,
20621 payload_codec: DEFAULT_CODEC.to_string(),
20622 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
20623 history_events: Vec::new(),
20624 total_history_events: Some(0),
20625 history_size_bytes: None,
20626 continue_as_new_recommended: None,
20627 history_budget_pressure: None,
20628 next_history_page_token: None,
20629 workflow_task_attempt: 1,
20630 workflow_signal_id: None,
20631 signal_name: None,
20632 signal_arguments: None,
20633 workflow_update_id: None,
20634 update_name: None,
20635 lease_owner: Some("rust-worker".to_string()),
20636 };
20637
20638 let commands = worker
20639 .execute_workflow_task(task)
20640 .expect("handler failure becomes a workflow command");
20641
20642 assert_eq!(commands.len(), 1);
20643 assert_eq!(commands[0]["type"], "fail_workflow");
20644 assert_eq!(commands[0]["exception_type"], "RustWorkflowError");
20645 assert_eq!(commands[0]["exception_class"], "durable_workflow::Error");
20646 assert_eq!(commands[0]["non_retryable"], false);
20647 assert_eq!(
20648 commands[0]["message"],
20649 "codec error: rust_conformance_failure"
20650 );
20651 assert_eq!(
20652 commands[0]["exception"]["message"],
20653 "codec error: rust_conformance_failure"
20654 );
20655 }
20656
20657 #[test]
20658 fn ordinary_handler_error_preserves_commands_queued_in_the_same_decision() {
20659 let client = Client::new("http://127.0.0.1:8080").expect("client");
20660 let mut worker = Worker::new(client, "rust-workers");
20661 worker.register_workflow("rust.failing-after-side-effect", |ctx, _input| async move {
20662 let _: String = ctx.side_effect(|| "captured".to_string())?;
20663 Err(Error::WorkerLoop("application failure".to_string()))
20664 });
20665
20666 let commands = worker
20667 .execute_workflow_task(workflow_task(
20668 "rust.failing-after-side-effect",
20669 Vec::new(),
20670 DEFAULT_CODEC,
20671 ))
20672 .expect("ordinary failure remains a workflow decision");
20673
20674 assert_eq!(commands.len(), 2);
20675 assert_eq!(commands[0]["type"], "record_side_effect");
20676 assert_eq!(commands[1]["type"], "fail_workflow");
20677 }
20678
20679 #[test]
20680 fn handler_error_cannot_hide_an_unconsumed_committed_side_effect() {
20681 let client = Client::new("http://127.0.0.1:8080").expect("client");
20682 let mut worker = Worker::new(client, "rust-workers");
20683 worker.register_workflow("rust.removed-side-effect", |_ctx, _input| async move {
20684 Err(Error::WorkerLoop("application failure".to_string()))
20685 });
20686 let result =
20687 encode_value_envelope(&json!("committed"), DEFAULT_CODEC).expect("side-effect result");
20688
20689 let error = worker
20690 .execute_workflow_task(workflow_task(
20691 "rust.removed-side-effect",
20692 vec![history_event(
20693 "SideEffectRecorded",
20694 json!({"sequence": 1, "result": result}),
20695 )],
20696 DEFAULT_CODEC,
20697 ))
20698 .expect_err("removed committed history must not become fail_workflow");
20699
20700 let Error::NonDeterministicReplay(failure) = error else {
20701 panic!("expected typed replay failure");
20702 };
20703 assert_eq!(failure.reason, "recorded_commands_unconsumed");
20704 assert_eq!(failure.sequence, Some(1));
20705 assert_eq!(failure.expected.as_deref(), Some("side effect"));
20706 }
20707
20708 #[test]
20709 fn replay_error_discards_side_effect_queued_before_incompatible_marker_check() {
20710 let client = Client::new("http://127.0.0.1:8080").expect("client");
20711 let mut worker = Worker::new(client, "rust-workers");
20712 worker.register_workflow(
20713 "rust.side-effect-before-marker-error",
20714 |ctx, _input| async move {
20715 assert_eq!(ctx.get_version("restart-safe", 1, 1)?, 1);
20716 let _: String = ctx.side_effect(|| "must-not-commit".to_string())?;
20717 ctx.get_version("restart-safe", 2, 2)?;
20718 Ok(Value::Null)
20719 },
20720 );
20721
20722 let error = worker
20723 .execute_workflow_task(workflow_task(
20724 "rust.side-effect-before-marker-error",
20725 vec![history_event(
20726 "VersionMarkerRecorded",
20727 json!({
20728 "sequence": 1,
20729 "change_id": "restart-safe",
20730 "version": 1,
20731 "min_supported": 1,
20732 "max_supported": 1,
20733 }),
20734 )],
20735 DEFAULT_CODEC,
20736 ))
20737 .expect_err("replay error must return no queued workflow commands");
20738
20739 let Error::NonDeterministicReplay(failure) = error else {
20740 panic!("expected typed replay failure");
20741 };
20742 assert_eq!(failure.reason, "version_marker_incompatible_range");
20743 assert_eq!(failure.sequence, Some(1));
20744 }
20745
20746 #[test]
20747 fn workflow_task_replay_keeps_recorded_unfired_timer_pending_without_rescheduling() {
20748 let client = Client::new("http://127.0.0.1:8080").expect("client");
20749 let mut worker = Worker::new(client, "rust-workers");
20750 worker.register_workflow("rust.timer.pending", |ctx, _input| async move {
20751 ctx.sleep(Duration::from_secs(5)).await?;
20752 Ok(json!({"status": "timer fired"}))
20753 });
20754
20755 let task = WorkflowTask {
20756 task_id: "wft-rust-timer-pending".to_string(),
20757 workflow_command_id: None,
20758 workflow_id: Some("wf-rust-timer".to_string()),
20759 run_id: Some("run-rust-timer".to_string()),
20760 workflow_type: "rust.timer.pending".to_string(),
20761 cancel_requested: false,
20762 payload_codec: DEFAULT_CODEC.to_string(),
20763 arguments: Some(
20764 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
20765 ),
20766 history_events: vec![history_event(
20767 "TimerScheduled",
20768 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20769 )],
20770 total_history_events: Some(1),
20771 history_size_bytes: None,
20772 continue_as_new_recommended: None,
20773 history_budget_pressure: None,
20774 next_history_page_token: None,
20775 workflow_task_attempt: 1,
20776 workflow_signal_id: None,
20777 signal_name: None,
20778 signal_arguments: None,
20779 workflow_update_id: None,
20780 update_name: None,
20781 lease_owner: Some("rust-worker".to_string()),
20782 };
20783
20784 for _redelivery_or_restart in 0..2 {
20785 let commands = worker
20786 .execute_workflow_task(task.clone())
20787 .expect("recorded timer remains pending");
20788 assert!(
20789 commands.is_empty(),
20790 "recorded timer must not be rescheduled"
20791 );
20792 }
20793 }
20794
20795 #[test]
20796 fn workflow_task_rejects_recorded_command_removed_from_workflow_code() {
20797 let client = Client::new("http://127.0.0.1:8080").expect("client");
20798 let mut worker = Worker::new(client, "rust-workers");
20799 worker.register_workflow("rust.timer.removed", |_ctx, _input| async move {
20800 Ok(json!({"status": "completed"}))
20801 });
20802 let task = WorkflowTask {
20803 task_id: "wft-rust-timer-removed".to_string(),
20804 workflow_command_id: None,
20805 workflow_id: Some("wf-rust-timer".to_string()),
20806 run_id: Some("run-rust-timer".to_string()),
20807 workflow_type: "rust.timer.removed".to_string(),
20808 cancel_requested: false,
20809 payload_codec: DEFAULT_CODEC.to_string(),
20810 arguments: Some(
20811 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
20812 ),
20813 history_events: vec![
20814 history_event(
20815 "TimerScheduled",
20816 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20817 ),
20818 history_event(
20819 "TimerFired",
20820 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20821 ),
20822 ],
20823 total_history_events: Some(2),
20824 history_size_bytes: None,
20825 continue_as_new_recommended: None,
20826 history_budget_pressure: None,
20827 next_history_page_token: None,
20828 workflow_task_attempt: 1,
20829 workflow_signal_id: None,
20830 signal_name: None,
20831 signal_arguments: None,
20832 workflow_update_id: None,
20833 update_name: None,
20834 lease_owner: Some("rust-worker".to_string()),
20835 };
20836
20837 let Error::NonDeterministicReplay(failure) = worker
20838 .execute_workflow_task(task)
20839 .expect_err("removed timer must fail replay")
20840 else {
20841 panic!("expected typed replay failure");
20842 };
20843 assert_eq!(failure.reason, "recorded_commands_unconsumed");
20844 assert_eq!(failure.sequence, Some(1));
20845 }
20846
20847 #[test]
20848 fn workflow_context_emits_explicit_child_workflow_contract() {
20849 let ctx = WorkflowContext {
20850 state: Arc::new(Mutex::new(
20851 WorkflowState::new_with_identity(
20852 Vec::new(),
20853 Some("wf-parent".to_string()),
20854 Some("run-parent".to_string()),
20855 "parent-workers".to_string(),
20856 DEFAULT_CODEC.to_string(),
20857 None,
20858 )
20859 .expect("workflow state"),
20860 )),
20861 };
20862 let options = ChildWorkflowOptions::new("python-workers")
20863 .parent_close_policy(ParentClosePolicy::RequestCancel)
20864 .retry_policy(ChildWorkflowRetryPolicy {
20865 max_attempts: Some(3),
20866 backoff_seconds: vec![1, 5],
20867 non_retryable_error_types: vec!["ValidationError".to_string()],
20868 })
20869 .execution_timeout_seconds(600)
20870 .run_timeout_seconds(120);
20871 let mut call = Box::pin(ctx.start_child_workflow(
20872 "python.fulfil-order",
20873 options,
20874 json!([{"order_id": "order-42"}]),
20875 ));
20876 let mut task_context = TaskContext::from_waker(noop_waker_ref());
20877
20878 assert!(matches!(
20879 call.as_mut().poll(&mut task_context),
20880 Poll::Pending
20881 ));
20882 let commands = ctx.take_commands().expect("commands");
20883 assert_eq!(commands.len(), 1);
20884 let command = &commands[0];
20885 assert_eq!(command["type"], "start_child_workflow");
20886 assert_eq!(command["workflow_type"], "python.fulfil-order");
20887 assert_eq!(command["queue"], "python-workers");
20888 assert_eq!(command["parent_close_policy"], "request_cancel");
20889 assert_eq!(command["retry_policy"]["max_attempts"], 3);
20890 assert_eq!(command["execution_timeout_seconds"], 600);
20891 assert_eq!(command["run_timeout_seconds"], 120);
20892 assert_eq!(
20893 decode_wire_value(&command["arguments"], DEFAULT_CODEC).expect("child args"),
20894 json!([{"order_id": "order-42"}])
20895 );
20896 }
20897
20898 fn child_parent_worker() -> Worker {
20899 let client = Client::new("http://127.0.0.1:8080").expect("client");
20900 let mut worker = Worker::new(client, "rust-parent-workers");
20901 worker.register_workflow("rust.parent", |ctx, _input| async move {
20902 let child = ctx
20903 .start_child_workflow(
20904 "python.child",
20905 ChildWorkflowOptions::new("python-child-workers")
20906 .parent_close_policy(ParentClosePolicy::Terminate),
20907 json!([{"codec_probe": [1, true, "rust"]}]),
20908 )
20909 .await?;
20910 Ok(json!({
20911 "parent_workflow_id": child.parent.workflow_id,
20912 "parent_run_id": child.parent.run_id,
20913 "child_workflow_id": child.child.workflow_id,
20914 "child_run_id": child.child.run_id,
20915 "child_workflow_type": child.child_workflow_type,
20916 "result": child.result,
20917 }))
20918 });
20919 worker
20920 }
20921
20922 fn child_parent_task(event_type: &str, payload: Value) -> WorkflowTask {
20923 WorkflowTask {
20924 task_id: "wft-child-parent".to_string(),
20925 workflow_command_id: None,
20926 workflow_id: Some("wf-parent".to_string()),
20927 run_id: Some("run-parent".to_string()),
20928 workflow_type: "rust.parent".to_string(),
20929 cancel_requested: false,
20930 payload_codec: DEFAULT_CODEC.to_string(),
20931 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
20932 history_events: vec![
20933 HistoryEvent {
20934 event_type: "ChildWorkflowScheduled".to_string(),
20935 payload: json!({
20936 "sequence": 1,
20937 "child_call_id": "call-child",
20938 "child_workflow_instance_id": "wf-child",
20939 "child_workflow_run_id": "run-child",
20940 "child_workflow_type": "python.child",
20941 }),
20942 raw: HashMap::new(),
20943 },
20944 HistoryEvent {
20945 event_type: event_type.to_string(),
20946 payload,
20947 raw: HashMap::new(),
20948 },
20949 ],
20950 total_history_events: Some(2),
20951 history_size_bytes: None,
20952 continue_as_new_recommended: None,
20953 history_budget_pressure: None,
20954 next_history_page_token: None,
20955 workflow_task_attempt: 1,
20956 workflow_signal_id: None,
20957 signal_name: None,
20958 signal_arguments: None,
20959 workflow_update_id: None,
20960 update_name: None,
20961 lease_owner: Some("rust-worker".to_string()),
20962 }
20963 }
20964
20965 #[test]
20966 fn committed_child_result_replays_without_starting_a_duplicate() {
20967 let worker = child_parent_worker();
20968 let task = child_parent_task(
20969 "ChildRunCompleted",
20970 json!({
20971 "sequence": 1,
20972 "child_call_id": "call-child",
20973 "child_workflow_instance_id": "wf-child",
20974 "child_workflow_run_id": "run-child",
20975 "child_workflow_type": "python.child",
20976 "payload_codec": DEFAULT_CODEC,
20977 "result": fixture_envelope(json!({"from":"python","ok":true})),
20978 }),
20979 );
20980
20981 for _restart in 0..2 {
20982 let commands = worker
20983 .execute_workflow_task(task.clone())
20984 .expect("replayed parent task");
20985 assert_eq!(commands.len(), 1);
20986 assert_eq!(commands[0]["type"], "complete_workflow");
20987 assert!(!commands
20988 .iter()
20989 .any(|command| command["type"] == "start_child_workflow"));
20990 let output =
20991 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
20992 assert_eq!(output["parent_workflow_id"], "wf-parent");
20993 assert_eq!(output["parent_run_id"], "run-parent");
20994 assert_eq!(output["child_workflow_id"], "wf-child");
20995 assert_eq!(output["child_run_id"], "run-child");
20996 assert_eq!(output["result"], json!({"from": "python", "ok": true}));
20997 }
20998 }
20999
21000 #[test]
21001 fn typed_child_arguments_and_results_survive_replay() {
21002 let client = Client::new("http://127.0.0.1:8080").expect("client");
21003 let mut worker = Worker::new(client, "rust-parent-workers");
21004 worker.register_workflow_avro_value("rust.typed-parent", |ctx, _input| async move {
21005 let child = ctx
21006 .start_child_workflow_avro_value(
21007 "python.typed-child",
21008 ChildWorkflowOptions::new("python-workers"),
21009 AvroValue::Array(vec![typed_fidelity_probe()]),
21010 )
21011 .await?;
21012 Ok(child.result)
21013 });
21014
21015 let initial = worker
21016 .execute_workflow_task(workflow_task(
21017 "rust.typed-parent",
21018 Vec::new(),
21019 DEFAULT_CODEC,
21020 ))
21021 .expect("typed child start");
21022 assert_eq!(initial[0]["type"], "start_child_workflow");
21023 assert_eq!(
21024 decode_wire_avro_value(&initial[0]["arguments"], DEFAULT_CODEC)
21025 .expect("typed child arguments"),
21026 AvroValue::Array(vec![typed_fidelity_probe()])
21027 );
21028
21029 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
21030 .expect("typed child result");
21031 let task = workflow_task(
21032 "rust.typed-parent",
21033 vec![
21034 history_event(
21035 "ChildWorkflowScheduled",
21036 json!({
21037 "sequence": 1,
21038 "child_call_id": "call-typed",
21039 "child_workflow_instance_id": "wf-child",
21040 "child_workflow_run_id": "run-child",
21041 "child_workflow_type": "python.typed-child",
21042 }),
21043 ),
21044 history_event(
21045 "ChildRunCompleted",
21046 json!({
21047 "sequence": 1,
21048 "child_call_id": "call-typed",
21049 "child_workflow_instance_id": "wf-child",
21050 "child_workflow_run_id": "run-child",
21051 "child_workflow_type": "python.typed-child",
21052 "payload_codec": DEFAULT_CODEC,
21053 "result": result,
21054 }),
21055 ),
21056 ],
21057 DEFAULT_CODEC,
21058 );
21059
21060 let commands = worker
21061 .execute_workflow_task(task)
21062 .expect("typed child replay");
21063 assert_eq!(commands[0]["type"], "complete_workflow");
21064 assert_eq!(
21065 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
21066 .expect("typed parent result"),
21067 typed_fidelity_probe()
21068 );
21069 }
21070
21071 #[test]
21072 fn pending_child_replays_after_restart_without_starting_a_duplicate() {
21073 let worker = child_parent_worker();
21074 let mut task = child_parent_task("unused", Value::Null);
21075 task.history_events.truncate(1);
21076 task.total_history_events = Some(1);
21077
21078 for _redelivery_or_restart in 0..2 {
21079 let commands = worker
21080 .execute_workflow_task(task.clone())
21081 .expect("recorded child remains pending");
21082 assert!(
21083 commands.is_empty(),
21084 "recorded pending child must not be started again"
21085 );
21086 }
21087 }
21088
21089 #[test]
21090 fn child_cancellation_becomes_stable_parent_failure_command() {
21091 let worker = child_parent_worker();
21092 let task = child_parent_task(
21093 "ChildRunCancelled",
21094 json!({
21095 "sequence": 1,
21096 "child_workflow_instance_id": "wf-child",
21097 "child_workflow_run_id": "run-child",
21098 "child_workflow_type": "python.child",
21099 "failure_id": "failure-child",
21100 "failure_category": "cancelled",
21101 "message": "cancelled by parent-close policy",
21102 }),
21103 );
21104
21105 let commands = worker
21106 .execute_workflow_task(task)
21107 .expect("parent settlement");
21108 assert_eq!(commands.len(), 1);
21109 assert_eq!(commands[0]["type"], "fail_workflow");
21110 assert_eq!(commands[0]["exception_type"], "ChildWorkflowCancelled");
21111 assert_eq!(
21112 commands[0]["exception"]["properties"]["reason"],
21113 "cancelled"
21114 );
21115 assert_eq!(
21116 commands[0]["exception"]["properties"]["child_workflow_run_id"],
21117 "run-child"
21118 );
21119 }
21120
21121 #[test]
21122 fn workflow_can_handle_typed_child_failure() {
21123 let client = Client::new("http://127.0.0.1:8080").expect("client");
21124 let mut worker = Worker::new(client, "rust-parent-workers");
21125 worker.register_workflow("rust.handled-parent", |ctx, _input| async move {
21126 match ctx
21127 .start_child_workflow(
21128 "python.child",
21129 ChildWorkflowOptions::new("python-child-workers"),
21130 json!([]),
21131 )
21132 .await
21133 {
21134 Err(Error::ChildWorkflowFailed(failure)) => Ok(json!({
21135 "reason": failure.reason,
21136 "failure_id": failure.failure_id,
21137 "exception_class": failure.exception_class,
21138 "child_run_id": failure.child_workflow_run_id,
21139 })),
21140 Err(error) => Err(error),
21141 Ok(_) => Err(Error::WorkerLoop(
21142 "child unexpectedly succeeded".to_string(),
21143 )),
21144 }
21145 });
21146 let mut task = child_parent_task(
21147 "ChildRunFailed",
21148 json!({
21149 "sequence": 1,
21150 "child_workflow_instance_id": "wf-child",
21151 "child_workflow_run_id": "run-child",
21152 "child_workflow_type": "python.child",
21153 "failure_id": "failure-child",
21154 "failure_category": "child_workflow",
21155 "message": "payment rejected",
21156 "exception": {
21157 "type": "PaymentRejected",
21158 "class": "payments.PaymentRejected",
21159 "message": "payment rejected"
21160 }
21161 }),
21162 );
21163 task.workflow_type = "rust.handled-parent".to_string();
21164
21165 let commands = worker.execute_workflow_task(task).expect("handled failure");
21166 assert_eq!(commands[0]["type"], "complete_workflow");
21167 let output =
21168 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
21169 assert_eq!(output["reason"], "child_workflow");
21170 assert_eq!(output["failure_id"], "failure-child");
21171 assert_eq!(output["exception_class"], "payments.PaymentRejected");
21172 assert_eq!(output["child_run_id"], "run-child");
21173 }
21174
21175 #[test]
21176 fn rust_hello_world_uses_signal_arguments_from_resume_payload() {
21177 let client = Client::new("http://127.0.0.1:8080").expect("client");
21178 let mut worker = Worker::new(client, "rust-workers");
21179
21180 worker.register_workflow("rust.hello_workflow", |ctx, _input| async move {
21181 let signal = ctx.wait_signal("start").await?;
21182 let name = signal
21183 .first()
21184 .and_then(|value| value.as_str())
21185 .unwrap_or("world");
21186 let greeting = ctx.activity("rust.hello_activity", json!([name])).await?;
21187 Ok(json!({
21188 "greeting": greeting,
21189 "language": "rust"
21190 }))
21191 });
21192
21193 let signal_arguments =
21194 encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC).expect("signal arguments");
21195 let task = WorkflowTask {
21196 task_id: "wft-rust-signal-1".to_string(),
21197 workflow_command_id: None,
21198 workflow_id: Some("wf-rust-hello".to_string()),
21199 run_id: Some("run-rust-hello".to_string()),
21200 workflow_type: "rust.hello_workflow".to_string(),
21201 cancel_requested: false,
21202 payload_codec: DEFAULT_CODEC.to_string(),
21203 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21204 history_events: vec![HistoryEvent {
21205 event_type: "SignalReceived".to_string(),
21206 payload: json!({
21207 "signal_id": "sig-rust-1",
21208 "signal_name": "start"
21209 }),
21210 raw: HashMap::new(),
21211 }],
21212 total_history_events: Some(1),
21213 history_size_bytes: None,
21214 continue_as_new_recommended: None,
21215 history_budget_pressure: None,
21216 next_history_page_token: None,
21217 workflow_task_attempt: 1,
21218 workflow_signal_id: Some("sig-rust-1".to_string()),
21219 signal_name: Some("start".to_string()),
21220 signal_arguments: Some(signal_arguments),
21221 workflow_update_id: None,
21222 update_name: None,
21223 lease_owner: Some("rust-worker".to_string()),
21224 };
21225
21226 let commands = worker.execute_workflow_task(task).expect("workflow task");
21227
21228 assert_eq!(commands.len(), 1);
21229 assert_eq!(commands[0]["type"], "schedule_activity");
21230 assert_eq!(commands[0]["activity_type"], "rust.hello_activity");
21231 assert_eq!(
21232 decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC).expect("activity args"),
21233 json!(["Rust"])
21234 );
21235 }
21236
21237 #[test]
21238 fn workflow_task_appends_paginated_history_events() {
21239 let mut task = WorkflowTask {
21240 task_id: "wft-rust-pages-1".to_string(),
21241 workflow_command_id: None,
21242 workflow_id: Some("wf-rust-pages".to_string()),
21243 run_id: Some("run-rust-pages".to_string()),
21244 workflow_type: "rust.hello_workflow".to_string(),
21245 cancel_requested: false,
21246 payload_codec: DEFAULT_CODEC.to_string(),
21247 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21248 history_events: vec![HistoryEvent {
21249 event_type: "WorkflowStarted".to_string(),
21250 payload: json!({}),
21251 raw: HashMap::new(),
21252 }],
21253 total_history_events: Some(3),
21254 history_size_bytes: None,
21255 continue_as_new_recommended: None,
21256 history_budget_pressure: None,
21257 next_history_page_token: Some("MQ==".to_string()),
21258 workflow_task_attempt: 1,
21259 workflow_signal_id: None,
21260 signal_name: None,
21261 signal_arguments: None,
21262 workflow_update_id: None,
21263 update_name: None,
21264 lease_owner: Some("rust-worker".to_string()),
21265 };
21266
21267 task.append_history_page(WorkflowTaskHistoryPage {
21268 history_events: vec![
21269 HistoryEvent {
21270 event_type: "SignalReceived".to_string(),
21271 payload: json!({
21272 "signal_id": "sig-rust-1",
21273 "signal_name": "start",
21274 "arguments": encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC)
21275 .expect("signal arguments")
21276 }),
21277 raw: HashMap::new(),
21278 },
21279 HistoryEvent {
21280 event_type: "MarkerRecorded".to_string(),
21281 payload: json!({"sequence": 3}),
21282 raw: HashMap::new(),
21283 },
21284 ],
21285 total_history_events: Some(3),
21286 next_history_page_token: None,
21287 });
21288
21289 assert_eq!(task.history_events.len(), 3);
21290 assert_eq!(task.total_history_events, Some(3));
21291 assert_eq!(task.next_history_page_token, None);
21292
21293 let signal = task
21294 .history_events
21295 .iter()
21296 .find(|event| event.event_type == "SignalReceived")
21297 .expect("signal event");
21298 assert_eq!(
21299 decode_signal_event_arguments(signal, DEFAULT_CODEC).expect("signal arguments"),
21300 vec![AvroValue::String("Rust".to_string())]
21301 );
21302 }
21303
21304 #[tokio::test]
21305 async fn query_handler_reads_ordered_cross_codec_signals_without_commands() {
21306 let client = Client::new("http://127.0.0.1:8080").expect("client");
21307 let mut worker = Worker::new(client, "rust-workers");
21308 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21309 worker.register_query("counter", "current", |ctx, _args| async move {
21310 let mut count = 0_i64;
21311 for signal in ctx.signal_events() {
21312 let value = signal
21313 .arguments
21314 .first()
21315 .and_then(Value::as_i64)
21316 .unwrap_or_default();
21317 match signal.name.as_str() {
21318 "increment" => count += value,
21319 "set" => count = value,
21320 _ => {}
21321 }
21322 }
21323 Ok(json!(count))
21324 });
21325
21326 let task = QueryTask {
21327 query_task_id: "query-rust-counter".to_string(),
21328 query_task_attempt: 1,
21329 lease_owner: Some("rust-worker".to_string()),
21330 workflow_id: Some("counter-1".to_string()),
21331 run_id: Some("run-counter-1".to_string()),
21332 workflow_type: "counter".to_string(),
21333 query_name: "current".to_string(),
21334 payload_codec: DEFAULT_CODEC.to_string(),
21335 workflow_arguments: Some(
21336 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
21337 ),
21338 query_arguments: Some(
21339 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("query arguments"),
21340 ),
21341 history_events: vec![
21342 HistoryEvent {
21343 event_type: "SignalReceived".to_string(),
21344 payload: json!({
21345 "signal_id": "php-signal-1",
21346 "signal_name": "increment",
21347 "workflow_sequence": 1,
21348 "payload_codec": DEFAULT_CODEC,
21349 "arguments": encode_value_envelope(&json!([3]), DEFAULT_CODEC).expect("php avro signal")
21350 }),
21351 raw: HashMap::new(),
21352 },
21353 HistoryEvent {
21354 event_type: "SignalReceived".to_string(),
21355 payload: json!({
21356 "signal_id": "python-signal-2",
21357 "signal_name": "increment",
21358 "workflow_sequence": 2,
21359 "payload_codec": DEFAULT_CODEC,
21360 "arguments": encode_value_envelope(&json!([5]), DEFAULT_CODEC).expect("python avro signal")
21361 }),
21362 raw: HashMap::new(),
21363 },
21364 HistoryEvent {
21365 event_type: "SignalReceived".to_string(),
21366 payload: json!({
21367 "signal_id": "rust-signal-3",
21368 "signal_name": "set",
21369 "workflow_sequence": 3,
21370 "payload_codec": DEFAULT_CODEC,
21371 "arguments": encode_value_envelope(&json!([0]), DEFAULT_CODEC).expect("rust avro signal")
21372 }),
21373 raw: HashMap::new(),
21374 },
21375 ],
21376 history_export: None,
21377 run_status: Some("completed".to_string()),
21378 };
21379
21380 let result = worker.execute_query_task(task).await.expect("query result");
21381 assert_eq!(result.into_json().expect("query projection"), json!(0));
21382 }
21383
21384 #[tokio::test]
21385 async fn replayed_queries_read_running_completed_and_cold_restarted_instance_state() {
21386 let worker = replay_counter_worker();
21387 let running_history = json!([
21388 {
21389 "type": "ActivityCompleted",
21390 "payload": {
21391 "sequence": 1,
21392 "activity_type": "load-counter",
21393 "payload_codec": DEFAULT_CODEC,
21394 "result": fixture_envelope(json!("loaded"))
21395 }
21396 },
21397 {
21398 "type": "SignalWaitOpened",
21399 "payload": {
21400 "sequence": 3,
21401 "signal_name": "increment"
21402 }
21403 },
21404 {
21405 "type": "SignalReceived",
21406 "payload": {
21407 "signal_id": "signal-3",
21408 "signal_name": "increment",
21409 "workflow_sequence": 2,
21410 "payload_codec": DEFAULT_CODEC,
21411 "arguments": fixture_envelope(json!([3]))
21412 }
21413 },
21414 {
21415 "type": "SignalApplied",
21416 "payload": {
21417 "sequence": 3,
21418 "signal_id": "signal-3",
21419 "signal_name": "increment",
21420 "payload_codec": DEFAULT_CODEC,
21421 "value": fixture_envelope(json!([3]))
21422 }
21423 }
21424 ]);
21425
21426 let running = worker
21427 .execute_query_task(replay_counter_query(
21428 "current",
21429 running_history.clone(),
21430 "running",
21431 ))
21432 .await
21433 .expect("running replay query");
21434 assert_eq!(
21435 running.clone().into_json().expect("query projection"),
21436 json!({"loaded": "loaded", "count": 3, "finished": false})
21437 );
21438
21439 let detached = worker
21440 .execute_query_task(replay_counter_query(
21441 "detached-mutation",
21442 running_history.clone(),
21443 "running",
21444 ))
21445 .await
21446 .expect("query mutates only its detached state clone");
21447 assert_eq!(detached.into_json().expect("query projection"), json!(999));
21448 let failed = worker
21449 .execute_query_task(replay_counter_query(
21450 "failed-mutation",
21451 running_history.clone(),
21452 "running",
21453 ))
21454 .await
21455 .expect_err("failed query");
21456 assert_eq!(failed.reason, "query_rejected");
21457 let unchanged = worker
21458 .execute_query_task(replay_counter_query("current", running_history, "running"))
21459 .await
21460 .expect("later query reconstructs unchanged state");
21461 assert_eq!(unchanged, running);
21462
21463 let restarted_worker = replay_counter_worker();
21464 let empty_arguments = fixture_envelope(json!([]));
21465 let loaded_result = fixture_envelope(json!("loaded"));
21466 let signal_three = fixture_blob(json!([3]));
21467 let signal_five = fixture_blob(json!([5]));
21468 let restarted_task: QueryTask = serde_json::from_value(json!({
21469 "query_task_id": "query-after-restart",
21470 "workflow_id": "counter-1",
21471 "run_id": "run-counter-1",
21472 "workflow_type": "replay-counter",
21473 "query_name": "current",
21474 "payload_codec": DEFAULT_CODEC,
21475 "workflow_arguments": empty_arguments.clone(),
21476 "query_arguments": empty_arguments,
21477 "history_events": [],
21478 "history_export": {
21479 "payloads": {"codec": DEFAULT_CODEC},
21480 "history_events": [
21481 {
21482 "type": "ActivityCompleted",
21483 "payload": {
21484 "sequence": 1,
21485 "activity_type": "load-counter",
21486 "payload_codec": DEFAULT_CODEC,
21487 "result": null
21488 }
21489 },
21490 {
21491 "type": "SignalWaitOpened",
21492 "payload": {
21493 "sequence": 3,
21494 "signal_name": "increment"
21495 }
21496 },
21497 {
21498 "type": "SignalReceived",
21499 "payload": {
21500 "signal_id": "signal-3",
21501 "signal_name": "increment",
21502 "workflow_sequence": 2
21503 }
21504 },
21505 {
21506 "type": "SignalApplied",
21507 "payload": {
21508 "sequence": 3,
21509 "signal_id": "signal-3",
21510 "signal_name": "increment"
21511 }
21512 },
21513 {
21514 "type": "SignalWaitOpened",
21515 "payload": {
21516 "sequence": 5,
21517 "signal_name": "increment"
21518 }
21519 },
21520 {
21521 "type": "SignalReceived",
21522 "payload": {
21523 "signal_id": "signal-5",
21524 "signal_name": "increment",
21525 "workflow_sequence": 4
21526 }
21527 },
21528 {
21529 "type": "SignalApplied",
21530 "payload": {
21531 "sequence": 5,
21532 "signal_id": "signal-5",
21533 "signal_name": "increment"
21534 }
21535 }
21536 ],
21537 "activities": [{
21538 "sequence": 1,
21539 "activity_type": "load-counter",
21540 "payload_codec": DEFAULT_CODEC,
21541 "result": loaded_result
21542 }],
21543 "signals": [
21544 {
21545 "id": "signal-3",
21546 "name": "increment",
21547 "workflow_sequence": 2,
21548 "payload_codec": DEFAULT_CODEC,
21549 "arguments": signal_three
21550 },
21551 {
21552 "id": "signal-5",
21553 "name": "increment",
21554 "workflow_sequence": 4,
21555 "payload_codec": DEFAULT_CODEC,
21556 "arguments": signal_five
21557 }
21558 ]
21559 },
21560 "run_status": "completed"
21561 }))
21562 .expect("cold replay query task");
21563 let completed = restarted_worker
21564 .execute_query_task(restarted_task)
21565 .await
21566 .expect("completed cold replay query");
21567 assert_eq!(
21568 completed.into_json().expect("query projection"),
21569 json!({"loaded": "loaded", "count": 8, "finished": true})
21570 );
21571 }
21572
21573 #[tokio::test]
21574 async fn replayed_query_replay_failures_are_machine_readable() {
21575 let worker = replay_counter_worker();
21576 let task = replay_counter_query(
21577 "current",
21578 json!([{
21579 "type": "ActivityCompleted",
21580 "payload": {
21581 "sequence": 1,
21582 "payload_codec": DEFAULT_CODEC,
21583 "result": {"codec": DEFAULT_CODEC, "blob": "{"}
21584 }
21585 }]),
21586 "running",
21587 );
21588 let failure = worker
21589 .execute_query_task(task)
21590 .await
21591 .expect_err("invalid replay history payload");
21592 assert_eq!(failure.reason, "query_payload_decode_failed");
21593 assert_eq!(failure.failure_type, "QueryPayloadDecodeFailed");
21594 assert!(failure.message.contains("invalid_payload_framing"));
21595 }
21596
21597 #[tokio::test]
21598 async fn query_task_restores_compact_history_from_export() {
21599 let client = Client::new("http://127.0.0.1:8080").expect("client");
21600 let mut worker = Worker::new(client, "rust-workers");
21601 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21602 worker.register_query("counter", "current", |ctx, _args| async move {
21603 Ok(json!(ctx.signals("increment")[0][0]))
21604 });
21605 let empty_arguments = fixture_envelope(json!([]));
21606 let exported_signal = fixture_blob(json!([9]));
21607 let task: QueryTask = serde_json::from_value(json!({
21608 "query_task_id": "query-export",
21609 "workflow_type": "counter",
21610 "query_name": "current",
21611 "payload_codec": DEFAULT_CODEC,
21612 "workflow_arguments": empty_arguments.clone(),
21613 "query_arguments": empty_arguments,
21614 "history_events": [],
21615 "history_export": {
21616 "payloads": {"codec": DEFAULT_CODEC},
21617 "history_events": [{
21618 "type": "SignalReceived",
21619 "payload": {"signal_id": "signal-export", "signal_name": "increment"}
21620 }],
21621 "signals": [{
21622 "id": "signal-export",
21623 "name": "increment",
21624 "status": "applied",
21625 "workflow_sequence": 1,
21626 "payload_codec": DEFAULT_CODEC,
21627 "arguments": exported_signal
21628 }]
21629 }
21630 }))
21631 .expect("query task");
21632
21633 let result = worker.execute_query_task(task).await.expect("query result");
21634 assert_eq!(result.into_json().expect("query projection"), json!(9));
21635 }
21636
21637 #[tokio::test]
21638 async fn query_task_failures_have_stable_reasons() {
21639 let client = Client::new("http://127.0.0.1:8080").expect("client");
21640 let mut worker = Worker::new(client, "rust-workers");
21641 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21642 worker.register_query(
21643 "counter",
21644 "current",
21645 |_ctx, _args| async move { Ok(json!(0)) },
21646 );
21647
21648 let base_task = QueryTask {
21649 query_task_id: "query-errors".to_string(),
21650 query_task_attempt: 1,
21651 lease_owner: None,
21652 workflow_id: Some("counter-errors".to_string()),
21653 run_id: Some("run-errors".to_string()),
21654 workflow_type: "counter".to_string(),
21655 query_name: "missing".to_string(),
21656 payload_codec: DEFAULT_CODEC.to_string(),
21657 workflow_arguments: Some(fixture_envelope(json!([]))),
21658 query_arguments: Some(fixture_envelope(json!([]))),
21659 history_events: Vec::new(),
21660 history_export: None,
21661 run_status: Some("running".to_string()),
21662 };
21663
21664 let unknown = worker
21665 .execute_query_task(base_task.clone())
21666 .await
21667 .expect_err("unknown query");
21668 assert_eq!(unknown.reason, "rejected_unknown_query");
21669
21670 let mut malformed = base_task;
21671 malformed.query_name = "current".to_string();
21672 malformed.query_arguments = Some(json!({"codec": DEFAULT_CODEC, "blob": "{"}));
21673 let malformed = worker
21674 .execute_query_task(malformed)
21675 .await
21676 .expect_err("malformed payload");
21677 assert_eq!(malformed.reason, "query_payload_decode_failed");
21678
21679 let client = Client::new("http://127.0.0.1:8080").expect("client");
21680 let mut unavailable_worker = Worker::new(client, "rust-workers");
21681 unavailable_worker
21682 .register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21683 let empty_arguments = fixture_envelope(json!([]));
21684 let unavailable_task: QueryTask = serde_json::from_value(json!({
21685 "query_task_id": "query-unavailable",
21686 "workflow_type": "counter",
21687 "query_name": "current",
21688 "payload_codec": DEFAULT_CODEC,
21689 "workflow_arguments": empty_arguments.clone(),
21690 "query_arguments": empty_arguments
21691 }))
21692 .expect("query task");
21693 let unavailable = unavailable_worker
21694 .execute_query_task(unavailable_task)
21695 .await
21696 .expect_err("query handler unavailable");
21697 assert_eq!(unavailable.reason, "query_handler_unavailable");
21698 }
21699
21700 #[tokio::test]
21701 async fn client_query_decodes_result_and_typed_failure() {
21702 let server = MockWorkerServer::start();
21703 let client = Client::builder(server.base_url())
21704 .timeout(Duration::from_secs(2))
21705 .build()
21706 .expect("client");
21707
21708 let result = client
21709 .query_workflow("counter-1", "current", json!([]))
21710 .await
21711 .expect("query result");
21712 assert_eq!(result, json!({"count": 8}));
21713
21714 let error = client
21715 .query_workflow("counter-1", "missing", json!([]))
21716 .await
21717 .expect_err("unknown query");
21718 let Error::QueryFailed(failure) = error else {
21719 panic!("expected typed query failure");
21720 };
21721 assert_eq!(failure.status, 404);
21722 assert_eq!(failure.reason, "rejected_unknown_query");
21723 }
21724
21725 #[tokio::test]
21726 async fn public_client_surfaces_send_and_receive_lossless_avro_values() {
21727 let server = MockWorkerServer::start();
21728 let client = Client::builder(server.base_url())
21729 .timeout(Duration::from_secs(2))
21730 .build()
21731 .expect("client");
21732 let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
21733
21734 client
21735 .start_workflow(
21736 "typed.echo",
21737 "rust-workers",
21738 "typed-start",
21739 arguments.clone(),
21740 )
21741 .await
21742 .expect("typed workflow start");
21743 assert_eq!(
21744 decode_wire_avro_value(
21745 &server.request_body("/api/workflows")["input"],
21746 DEFAULT_CODEC,
21747 )
21748 .expect("typed start input"),
21749 arguments
21750 );
21751
21752 client
21753 .signal_workflow("typed-1", "changed", arguments.clone())
21754 .await
21755 .expect("typed signal");
21756 assert_eq!(
21757 decode_wire_avro_value(
21758 &server.request_body("/api/workflows/typed-1/signal/changed")["input"],
21759 DEFAULT_CODEC,
21760 )
21761 .expect("typed signal input"),
21762 arguments
21763 );
21764
21765 assert_eq!(
21766 client
21767 .query_workflow_avro_value("typed-1", "inspect", arguments.clone())
21768 .await
21769 .expect("typed query"),
21770 typed_fidelity_probe()
21771 );
21772 assert_eq!(
21773 decode_wire_avro_value(
21774 &server.request_body("/api/workflows/typed-1/query/inspect")["input"],
21775 DEFAULT_CODEC,
21776 )
21777 .expect("typed query input"),
21778 arguments
21779 );
21780
21781 assert_eq!(
21782 client
21783 .update_workflow_avro_value(
21784 "typed-1",
21785 "replace",
21786 arguments.clone(),
21787 Some("typed-request"),
21788 )
21789 .await
21790 .expect("typed update"),
21791 typed_fidelity_probe()
21792 );
21793 let update = server.request_body("/api/workflows/typed-1/update/replace");
21794 assert_eq!(update["request_id"], "typed-request");
21795 assert_eq!(
21796 decode_wire_avro_value(&update["input"], DEFAULT_CODEC).expect("typed update input"),
21797 arguments
21798 );
21799
21800 let handle = WorkflowHandle {
21801 client: client.clone(),
21802 workflow_id: "typed-1".to_string(),
21803 run_id: Some("run-typed-1".to_string()),
21804 workflow_type: "typed.echo".to_string(),
21805 };
21806 assert_eq!(
21807 handle
21808 .result_avro_value(WorkflowResultOptions::default())
21809 .await
21810 .expect("typed workflow result"),
21811 typed_fidelity_probe()
21812 );
21813
21814 client
21815 .complete_activity_task(
21816 "activity-typed",
21817 "attempt-typed",
21818 "rust-worker",
21819 typed_fidelity_probe(),
21820 DEFAULT_CODEC,
21821 )
21822 .await
21823 .expect("typed activity completion");
21824 assert_eq!(
21825 decode_wire_avro_value(
21826 &server.request_body("/api/worker/activity-tasks/activity-typed/complete")
21827 ["result"],
21828 DEFAULT_CODEC,
21829 )
21830 .expect("typed activity result"),
21831 typed_fidelity_probe()
21832 );
21833 client
21834 .fail_activity_task(
21835 "activity-typed",
21836 "attempt-typed",
21837 "rust-worker",
21838 "typed failure",
21839 true,
21840 )
21841 .await
21842 .expect("activity failure");
21843 }
21844
21845 #[tokio::test]
21846 async fn lifecycle_commands_support_instance_and_selected_run_targets() {
21847 let server = MockWorkerServer::start();
21848 let client = Client::builder(server.base_url())
21849 .timeout(Duration::from_secs(2))
21850 .build()
21851 .expect("client");
21852
21853 let options = WorkflowCommandOptions::new()
21854 .reason("cleanup requested")
21855 .request_id("cancel-17");
21856 let cancelled = client
21857 .cancel_workflow("wf-lifecycle", options)
21858 .await
21859 .expect("instance cancellation");
21860 assert_eq!(cancelled.command, WorkflowCommandKind::Cancel);
21861 assert_eq!(cancelled.run_id.as_deref(), Some("run-current"));
21862 assert_eq!(cancelled.outcome.as_deref(), Some("cancelled"));
21863 assert_eq!(
21864 server.request_body("/api/workflows/wf-lifecycle/cancel"),
21865 json!({"reason":"cleanup requested","request_id":"cancel-17"})
21866 );
21867
21868 let terminated = client
21869 .terminate_workflow(
21870 "wf-lifecycle",
21871 WorkflowCommandOptions::new().reason("forced stop"),
21872 )
21873 .await
21874 .expect("instance termination");
21875 assert_eq!(terminated.command, WorkflowCommandKind::Terminate);
21876 assert_eq!(terminated.outcome.as_deref(), Some("terminated"));
21877
21878 client
21879 .cancel_workflow_run(
21880 "wf-lifecycle",
21881 "run-current",
21882 WorkflowCommandOptions::default(),
21883 )
21884 .await
21885 .expect("selected run cancellation");
21886 client
21887 .terminate_workflow_run(
21888 "wf-lifecycle",
21889 "run-current",
21890 WorkflowCommandOptions::default(),
21891 )
21892 .await
21893 .expect("selected run termination");
21894
21895 for (command, error) in [
21896 (
21897 WorkflowCommandKind::Cancel,
21898 client
21899 .cancel_workflow_run(
21900 "wf-lifecycle",
21901 "run-stale",
21902 WorkflowCommandOptions::default(),
21903 )
21904 .await
21905 .expect_err("stale cancellation must be rejected"),
21906 ),
21907 (
21908 WorkflowCommandKind::Terminate,
21909 client
21910 .terminate_workflow_run(
21911 "wf-lifecycle",
21912 "run-stale",
21913 WorkflowCommandOptions::default(),
21914 )
21915 .await
21916 .expect_err("stale termination must be rejected"),
21917 ),
21918 ] {
21919 let Error::WorkflowCommandRejected(rejection) = error else {
21920 panic!("expected typed command rejection");
21921 };
21922 assert_eq!(rejection.command, command);
21923 assert_eq!(rejection.status, 409);
21924 assert_eq!(rejection.reason, "historical_run_command_rejected");
21925 assert_eq!(rejection.run_id.as_deref(), Some("run-stale"));
21926 assert_eq!(rejection.target_scope.as_deref(), Some("run"));
21927 }
21928 }
21929
21930 #[tokio::test]
21931 async fn workflow_start_options_send_server_enforced_deadlines() {
21932 let server = MockWorkerServer::start();
21933 let client = Client::builder(server.base_url())
21934 .timeout(Duration::from_secs(2))
21935 .build()
21936 .expect("client");
21937
21938 let handle = client
21939 .start_workflow_with_options(
21940 "rust.timeout",
21941 "rust-timeouts",
21942 "wf-start-options",
21943 WorkflowStartOptions::new()
21944 .execution_timeout_seconds(30)
21945 .run_timeout_seconds(1),
21946 json!([]),
21947 )
21948 .await
21949 .expect("workflow start");
21950
21951 assert_eq!(handle.run_id.as_deref(), Some("run-start-options"));
21952 let body = server.request_body("/api/workflows");
21953 assert_eq!(body["execution_timeout_seconds"], 30);
21954 assert_eq!(body["run_timeout_seconds"], 1);
21955
21956 let invalid = client
21957 .start_workflow_with_options(
21958 "rust.timeout",
21959 "rust-timeouts",
21960 "wf-invalid-options",
21961 WorkflowStartOptions::new()
21962 .execution_timeout_seconds(1)
21963 .run_timeout_seconds(2),
21964 json!([]),
21965 )
21966 .await
21967 .expect_err("invalid deadline ordering");
21968 assert!(invalid
21969 .to_string()
21970 .contains("run_timeout_seconds cannot exceed execution_timeout_seconds"));
21971 }
21972
21973 #[tokio::test]
21974 async fn workflow_result_returns_each_typed_terminal_outcome() {
21975 let server = MockWorkerServer::start();
21976 let client = Client::builder(server.base_url())
21977 .timeout(Duration::from_secs(2))
21978 .build()
21979 .expect("client");
21980 let options = WorkflowResultOptions {
21981 poll_interval: Duration::ZERO,
21982 timeout: Duration::from_secs(1),
21983 };
21984
21985 let failed = WorkflowHandle {
21986 client: client.clone(),
21987 workflow_id: "wf-failed".to_string(),
21988 run_id: Some("run-failed".to_string()),
21989 workflow_type: "failure".to_string(),
21990 }
21991 .result(options)
21992 .await
21993 .expect_err("failed outcome");
21994 let Error::WorkflowFailed(failure) = failed else {
21995 panic!("expected WorkflowFailed");
21996 };
21997 assert_eq!(failure.workflow_id, "wf-failed");
21998 assert_eq!(failure.run_id.as_deref(), Some("run-failed"));
21999 assert_eq!(failure.failure_id.as_deref(), Some("failure-17"));
22000 assert_eq!(failure.failure_category.as_deref(), Some("application"));
22001 assert_eq!(failure.exception_type.as_deref(), Some("PaymentError"));
22002 assert_eq!(
22003 failure.exception_class.as_deref(),
22004 Some("billing::PaymentError")
22005 );
22006 assert_eq!(failure.non_retryable, Some(true));
22007
22008 for (workflow_id, expected_kind, expected_reason) in [
22009 (
22010 "wf-cancelled",
22011 WorkflowTerminalKind::Cancelled,
22012 "cleanup requested",
22013 ),
22014 (
22015 "wf-terminated",
22016 WorkflowTerminalKind::Terminated,
22017 "forced stop",
22018 ),
22019 (
22020 "wf-timed-out",
22021 WorkflowTerminalKind::TimedOut,
22022 "run_timeout",
22023 ),
22024 ] {
22025 let error = WorkflowHandle {
22026 client: client.clone(),
22027 workflow_id: workflow_id.to_string(),
22028 run_id: None,
22029 workflow_type: "terminal".to_string(),
22030 }
22031 .result(options)
22032 .await
22033 .expect_err("typed terminal outcome");
22034 let outcome = match error {
22035 Error::WorkflowCancelled(outcome) => outcome,
22036 Error::WorkflowTerminated(outcome) => outcome,
22037 Error::WorkflowTimedOut(outcome) => outcome,
22038 other => panic!("unexpected terminal error: {other}"),
22039 };
22040 assert_eq!(outcome.kind, expected_kind);
22041 assert_eq!(outcome.workflow_id, workflow_id);
22042 assert_eq!(outcome.reason, expected_reason);
22043 }
22044
22045 let wait_timeout = WorkflowHandle {
22046 client,
22047 workflow_id: "wf-waiting".to_string(),
22048 run_id: Some("run-waiting".to_string()),
22049 workflow_type: "waiting".to_string(),
22050 }
22051 .result(WorkflowResultOptions {
22052 poll_interval: Duration::ZERO,
22053 timeout: Duration::ZERO,
22054 })
22055 .await
22056 .expect_err("client wait timeout");
22057 let Error::WorkflowTimedOut(timeout) = wait_timeout else {
22058 panic!("expected typed client timeout");
22059 };
22060 assert_eq!(timeout.reason, "result_wait_timeout");
22061 assert_eq!(timeout.failure_category.as_deref(), Some("client_timeout"));
22062 assert_eq!(timeout.run_id.as_deref(), Some("run-waiting"));
22063 }
22064
22065 #[tokio::test]
22066 async fn workflow_result_follows_chain_and_selected_result_preserves_history() {
22067 let server = MockWorkerServer::start();
22068 let client = Client::builder(server.base_url())
22069 .timeout(Duration::from_secs(2))
22070 .build()
22071 .expect("client");
22072
22073 let handle = WorkflowHandle {
22074 client,
22075 workflow_id: "wf-selected".to_string(),
22076 run_id: Some("run-selected".to_string()),
22077 workflow_type: "selected".to_string(),
22078 };
22079 let options = WorkflowResultOptions {
22080 poll_interval: Duration::ZERO,
22081 timeout: Duration::from_secs(1),
22082 };
22083
22084 let current = handle
22085 .result(options)
22086 .await
22087 .expect("instance result follows the current run");
22088 assert_eq!(current, json!("current run output"));
22089
22090 let error = handle
22091 .result_selected_run(options)
22092 .await
22093 .expect_err("the selected run is cancelled even though the current run completed");
22094
22095 let Error::WorkflowCancelled(outcome) = error else {
22096 panic!("expected selected run cancellation");
22097 };
22098 assert_eq!(outcome.run_id.as_deref(), Some("run-selected"));
22099 assert_eq!(outcome.reason, "selected run cancelled");
22100 assert_eq!(
22101 server.request_count("/api/workflows/wf-selected/runs/run-selected"),
22102 1
22103 );
22104 assert_eq!(server.request_count("/api/workflows/wf-selected"), 1);
22105 }
22106
22107 #[tokio::test]
22108 async fn poll_responses_decode_http_conflict_drain_as_a_stable_stop() {
22109 let server = MockWorkerServer::draining_polls();
22110 let client = Client::builder(server.base_url())
22111 .timeout(Duration::from_secs(2))
22112 .build()
22113 .expect("client");
22114
22115 let workflow = client
22116 .poll_workflow_task_response("draining-worker", "rust-workers", Duration::ZERO)
22117 .await
22118 .expect("workflow drain response");
22119 let activity = client
22120 .poll_activity_task_response("draining-worker", "rust-workers", Duration::ZERO)
22121 .await
22122 .expect("activity drain response");
22123 let query = client
22124 .poll_query_task_response("draining-worker", "rust-workers", Duration::ZERO)
22125 .await
22126 .expect("query drain response");
22127
22128 for outcome in [workflow.outcome(), activity.outcome(), query.outcome()] {
22129 assert_eq!(
22130 outcome,
22131 WorkerPollOutcome::Stop {
22132 poll_status: Some("draining".to_string()),
22133 reason: Some("worker_draining".to_string()),
22134 }
22135 );
22136 }
22137
22138 assert!(client
22139 .poll_workflow_task("draining-worker", "rust-workers", Duration::ZERO)
22140 .await
22141 .expect("compatibility poll")
22142 .is_none());
22143 }
22144
22145 #[tokio::test]
22146 async fn managed_worker_honors_drain_stop_for_every_task_family() {
22147 let server = MockWorkerServer::draining_polls();
22148 let client = Client::builder(server.base_url())
22149 .timeout(Duration::from_secs(2))
22150 .build()
22151 .expect("client");
22152
22153 let mut workflow_worker = Worker::new(client.clone(), "rust-workers")
22154 .worker_id("draining-workflow-worker")
22155 .poll_timeout(Duration::ZERO);
22156 workflow_worker.register_workflow("counter", |_ctx, _args| async { Ok(Value::Null) });
22157 workflow_worker
22158 .run()
22159 .await
22160 .expect("workflow drain is a clean stop");
22161
22162 let mut activity_worker = Worker::new(client.clone(), "rust-workers")
22163 .worker_id("draining-activity-worker")
22164 .poll_timeout(Duration::ZERO);
22165 activity_worker.register_activity("write", |_ctx, _args| async { Ok(Value::Null) });
22166 activity_worker
22167 .run()
22168 .await
22169 .expect("activity drain is a clean stop");
22170
22171 let mut query_worker = Worker::new(client, "rust-workers")
22172 .worker_id("draining-query-worker")
22173 .poll_timeout(Duration::ZERO);
22174 query_worker.register_query("counter", "current", |_ctx, _args| async {
22175 Ok(Value::Null)
22176 });
22177 query_worker
22178 .run()
22179 .await
22180 .expect("query drain is a clean stop");
22181 }
22182
22183 #[tokio::test]
22184 async fn activity_cancellation_and_late_completion_remain_machine_readable() {
22185 let server = MockWorkerServer::start();
22186 let client = Client::builder(server.base_url())
22187 .timeout(Duration::from_secs(2))
22188 .build()
22189 .expect("client");
22190
22191 let heartbeat = client
22192 .heartbeat_activity_task(
22193 "activity-cancel",
22194 "attempt-cancel",
22195 "rust-worker",
22196 typed_fidelity_probe(),
22197 )
22198 .await
22199 .expect("cancellation heartbeat");
22200 assert!(heartbeat.cancel_requested);
22201 assert!(heartbeat.should_stop());
22202 assert_eq!(heartbeat.reason.as_deref(), Some("run_cancelled"));
22203 assert_eq!(heartbeat.run_closed_reason.as_deref(), Some("cancelled"));
22204 let heartbeat_body =
22205 server.request_body("/api/worker/activity-tasks/activity-cancel/heartbeat");
22206 assert_eq!(heartbeat_body["details"]["codec"], DEFAULT_CODEC);
22207 assert_eq!(
22208 decode_wire_avro_value(&heartbeat_body["details"], DEFAULT_CODEC)
22209 .expect("typed heartbeat details"),
22210 typed_fidelity_probe()
22211 );
22212
22213 let error = client
22214 .complete_activity_task(
22215 "activity-cancel",
22216 "attempt-cancel",
22217 "rust-worker",
22218 json!({"late":true}),
22219 DEFAULT_CODEC,
22220 )
22221 .await
22222 .expect_err("late completion must be refused");
22223 assert!(activity_task_rejection_is_final(&error));
22224 let Error::ActivityTaskRejected(rejection) = error else {
22225 panic!("expected typed activity rejection");
22226 };
22227 assert_eq!(rejection.status, 409);
22228 assert_eq!(rejection.reason, "run_cancelled");
22229 assert!(rejection.cancel_requested);
22230 assert_eq!(rejection.can_continue, Some(false));
22231 }
22232
22233 #[tokio::test]
22234 async fn managed_worker_survives_late_completion_and_restart_during_cancellation() {
22235 let server = MockWorkerServer::cancelled_activity();
22236 let client = Client::builder(server.base_url())
22237 .timeout(Duration::from_secs(2))
22238 .build()
22239 .expect("client");
22240 let cancellation_observed = Arc::new(AtomicBool::new(false));
22241 let observed = Arc::clone(&cancellation_observed);
22242 let mut worker = Worker::new(client.clone(), "rust-workers")
22243 .worker_id("rust-cancel-worker")
22244 .poll_timeout(Duration::from_millis(10));
22245 worker.register_activity("cancel-aware", move |ctx, _args| {
22246 let observed = Arc::clone(&observed);
22247 async move {
22248 let heartbeat = ctx.heartbeat(json!({"stage":"running"})).await?;
22249 observed.store(heartbeat.should_stop(), Ordering::SeqCst);
22250 Ok(json!({"late":"completion"}))
22251 }
22252 });
22253
22254 assert_eq!(
22255 worker.run_once().await.expect("cancelled attempt handled"),
22256 1
22257 );
22258 assert!(cancellation_observed.load(Ordering::SeqCst));
22259 assert_eq!(
22260 server.request_count("/api/worker/activity-tasks/activity-cancel/complete"),
22261 1
22262 );
22263
22264 let mut restarted = Worker::new(client, "rust-workers")
22265 .worker_id("rust-cancel-worker-restarted")
22266 .poll_timeout(Duration::from_millis(10));
22267 restarted.register_activity("cancel-aware", |_ctx, _args| async move { Ok(Value::Null) });
22268 assert_eq!(
22269 restarted
22270 .run_once()
22271 .await
22272 .expect("replacement worker continues polling"),
22273 0
22274 );
22275 }
22276
22277 #[tokio::test]
22278 async fn managed_worker_absorbs_selected_run_terminal_timeout_completion_race() {
22279 let response = r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"outcome":"completed","recorded":false,"run_id":"run-selected-timeout","run_status":"failed","created_task_ids":[],"reason":"run_timed_out"}"#;
22280 let server = MockWorkerServer::workflow_completion("409 Conflict", response);
22281 let client = Client::builder(server.base_url())
22282 .timeout(Duration::from_secs(2))
22283 .build()
22284 .expect("client");
22285
22286 let direct_error = client
22287 .complete_workflow_task(
22288 "workflow-timeout-task",
22289 "timeout-worker",
22290 3,
22291 vec![json!({
22292 "type": "complete_workflow",
22293 "result": fixture_envelope(Value::Null)
22294 })],
22295 )
22296 .await
22297 .expect_err("the low-level client preserves the completion rejection");
22298 let Error::Http { status, body } = direct_error else {
22299 panic!("expected the original HTTP completion rejection");
22300 };
22301 assert_eq!(status, reqwest::StatusCode::CONFLICT);
22302 assert_eq!(
22303 serde_json::from_str::<Value>(&body).expect("response body")["reason"],
22304 "run_timed_out"
22305 );
22306
22307 let mut worker = Worker::new(client, "rust-workers")
22308 .worker_id("timeout-worker")
22309 .poll_timeout(Duration::from_millis(10));
22310 worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
22311 Ok(json!({"late": "result"}))
22312 });
22313
22314 assert_eq!(
22315 worker
22316 .run_once()
22317 .await
22318 .expect("authoritative selected-run timeout settles the tick"),
22319 1
22320 );
22321 assert_eq!(
22322 server.request_count("/api/worker/workflow-tasks/workflow-timeout-task/complete"),
22323 2,
22324 "both the direct client proof and managed worker must see the rejection"
22325 );
22326 }
22327
22328 #[tokio::test]
22329 async fn managed_worker_does_not_swallow_nearby_completion_errors() {
22330 for (name, status, response) in [
22331 ("bare conflict", "409 Conflict", r#"{"message":"conflict"}"#),
22332 (
22333 "command was recorded",
22334 "409 Conflict",
22335 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":true,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22336 ),
22337 (
22338 "lease conflict",
22339 "409 Conflict",
22340 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"lease_expired"}"#,
22341 ),
22342 (
22343 "nonterminal run",
22344 "409 Conflict",
22345 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"waiting","reason":"run_timed_out"}"#,
22346 ),
22347 (
22348 "different selected run",
22349 "409 Conflict",
22350 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-reused-workflow-current","run_status":"failed","reason":"run_timed_out"}"#,
22351 ),
22352 (
22353 "different task attempt",
22354 "409 Conflict",
22355 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":4,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22356 ),
22357 (
22358 "authentication failure",
22359 "401 Unauthorized",
22360 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22361 ),
22362 (
22363 "authorization failure",
22364 "403 Forbidden",
22365 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22366 ),
22367 (
22368 "protocol failure",
22369 "400 Bad Request",
22370 r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.2","requested_version":"1.3"}"#,
22371 ),
22372 (
22373 "malformed command",
22374 "422 Unprocessable Entity",
22375 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22376 ),
22377 (
22378 "transient server failure",
22379 "503 Service Unavailable",
22380 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22381 ),
22382 ] {
22383 let server = MockWorkerServer::workflow_completion(status, response);
22384 let client = Client::builder(server.base_url())
22385 .timeout(Duration::from_secs(2))
22386 .build()
22387 .expect("client");
22388 let mut worker = Worker::new(client, "rust-workers")
22389 .worker_id("timeout-worker")
22390 .poll_timeout(Duration::from_millis(10));
22391 worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
22392 Ok(json!({"late": "result"}))
22393 });
22394
22395 let error = worker
22396 .run_once()
22397 .await
22398 .expect_err(&format!("{name} must remain an error"));
22399 assert!(
22400 matches!(error, Error::Http { .. } | Error::Protocol(_)),
22401 "{name} returned an unexpected error variant: {error}"
22402 );
22403 }
22404 }
22405
22406 #[tokio::test]
22407 async fn worker_deregistration_uses_worker_plane_method_path_headers_and_result() {
22408 let server = MockWorkerServer::start();
22409 let client = Client::builder(server.base_url())
22410 .worker_token(Some("worker-secret".to_string()))
22411 .namespace("orders")
22412 .timeout(Duration::from_secs(2))
22413 .build()
22414 .expect("client");
22415 let path = "/api/worker/registrations/worker%2F%CE%B1%20space";
22416
22417 let result = client
22418 .deregister_worker_registration("worker/α space")
22419 .await
22420 .expect("deregister worker registration");
22421
22422 assert_eq!(server.method_for(path).as_deref(), Some("DELETE"));
22423 assert_eq!(
22424 server.worker_protocol_for(path).as_deref(),
22425 Some(WORKER_PROTOCOL_VERSION)
22426 );
22427 assert_eq!(server.control_protocol_for(path), None);
22428 assert_eq!(server.namespace_for(path).as_deref(), Some("orders"));
22429 assert_eq!(
22430 server.authorization_for(path).as_deref(),
22431 Some("Bearer worker-secret")
22432 );
22433 assert_eq!(
22434 result,
22435 WorkerDeregistrationEnvelope {
22436 worker_id: "deregistered-worker".to_string(),
22437 outcome: "deregistered".to_string(),
22438 recovered_workflow_task_count: 2,
22439 }
22440 );
22441 }
22442
22443 #[tokio::test]
22444 async fn low_level_registration_rejects_update_validators_before_transport() {
22445 let server = MockWorkerServer::start();
22446 let client = Client::builder(server.base_url())
22447 .timeout(Duration::from_secs(2))
22448 .build()
22449 .expect("client");
22450
22451 for update_validators in [json!(["approve"]), json!("approve")] {
22452 let error = client
22453 .register_worker_with_command_contracts(
22454 "validator-claiming-worker",
22455 "rust-workers",
22456 vec!["orders".to_string()],
22457 vec![],
22458 1,
22459 1,
22460 vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
22461 json!({
22462 "orders": {
22463 "queries": ["current"],
22464 "updates": ["approve"],
22465 "update_validators": update_validators,
22466 },
22467 }),
22468 )
22469 .await
22470 .expect_err("unsupported validator claims must fail before registration");
22471
22472 let Error::UnsupportedUpdateValidators { workflow_type } = error else {
22473 panic!("expected typed unsupported-validator failure");
22474 };
22475 assert_eq!(workflow_type, "orders");
22476 }
22477 assert_eq!(server.request_count("/api/worker/register"), 0);
22478 }
22479
22480 #[tokio::test]
22481 async fn low_level_registration_preserves_query_and_update_contracts() {
22482 let server = MockWorkerServer::start();
22483 let client = Client::builder(server.base_url())
22484 .timeout(Duration::from_secs(2))
22485 .build()
22486 .expect("client");
22487 let contracts = json!({
22488 "orders": {
22489 "queries": ["current"],
22490 "updates": ["approve"],
22491 "update_validators": [],
22492 },
22493 "payments": {
22494 "queries": ["status"],
22495 "updates": ["capture"],
22496 },
22497 });
22498
22499 client
22500 .register_worker_with_command_contracts(
22501 "command-worker",
22502 "rust-workers",
22503 vec!["orders".to_string(), "payments".to_string()],
22504 vec![],
22505 1,
22506 1,
22507 vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
22508 contracts.clone(),
22509 )
22510 .await
22511 .expect("query and update contracts must remain supported");
22512
22513 assert_eq!(
22514 server.request_body("/api/worker/register")["workflow_command_contracts"],
22515 contracts
22516 );
22517 }
22518
22519 #[tokio::test]
22520 async fn role_scoped_tokens_are_never_used_for_the_opposite_plane() {
22521 let server = MockWorkerServer::start();
22522 let control_only = Client::builder(server.base_url())
22523 .control_token(Some("control-secret".to_string()))
22524 .build()
22525 .expect("control client");
22526
22527 let error = control_only
22528 .register_worker("worker", "queue", vec![], vec![], 1, 1)
22529 .await
22530 .expect_err("control token must not authorize a worker request");
22531 assert!(matches!(
22532 error,
22533 Error::MissingRoleCredentials { role: "worker", .. }
22534 ));
22535 assert_eq!(server.request_count("/api/worker/register"), 0);
22536
22537 let worker_only = Client::builder(server.base_url())
22538 .worker_token(Some("worker-secret".to_string()))
22539 .build()
22540 .expect("worker client");
22541 let error = worker_only
22542 .health()
22543 .await
22544 .expect_err("worker token must not authorize a control request");
22545 assert!(matches!(
22546 error,
22547 Error::MissingRoleCredentials {
22548 role: "control",
22549 ..
22550 }
22551 ));
22552 assert_eq!(server.request_count("/api/health"), 0);
22553 }
22554
22555 #[tokio::test]
22556 async fn shared_token_supports_worker_and_control_planes() {
22557 let server = MockWorkerServer::start();
22558 let client = Client::builder(server.base_url())
22559 .token(Some("shared-secret".to_string()))
22560 .build()
22561 .expect("client");
22562
22563 client.health().await.expect("control request");
22564 client
22565 .register_worker("worker", "queue", vec![], vec![], 1, 1)
22566 .await
22567 .expect("worker request");
22568
22569 assert_eq!(
22570 server.authorization_for("/api/health").as_deref(),
22571 Some("Bearer shared-secret")
22572 );
22573 assert_eq!(
22574 server.control_protocol_for("/api/health").as_deref(),
22575 Some(CONTROL_PLANE_VERSION)
22576 );
22577 assert_eq!(
22578 server.authorization_for("/api/worker/register").as_deref(),
22579 Some("Bearer shared-secret")
22580 );
22581 assert_eq!(
22582 server
22583 .worker_protocol_for("/api/worker/register")
22584 .as_deref(),
22585 Some(WORKER_PROTOCOL_VERSION)
22586 );
22587 }
22588
22589 #[tokio::test]
22590 async fn baseline_worker_endpoints_send_the_baseline_protocol() {
22591 let server = MockWorkerServer::start();
22592 let client = Client::builder(server.base_url())
22593 .timeout(Duration::from_secs(2))
22594 .build()
22595 .expect("client");
22596
22597 client
22598 .register_worker("capture-worker", "capture", vec![], vec![], 1, 1)
22599 .await
22600 .expect("register");
22601 client
22602 .heartbeat_worker("capture-worker", 1, 1)
22603 .await
22604 .expect("heartbeat");
22605 client
22606 .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
22607 .await
22608 .expect("workflow poll");
22609 client
22610 .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
22611 .await
22612 .expect("activity poll");
22613
22614 for path in [
22615 "/api/worker/register",
22616 "/api/worker/heartbeat",
22617 "/api/worker/workflow-tasks/poll",
22618 "/api/worker/activity-tasks/poll",
22619 ] {
22620 assert_eq!(
22621 server.worker_protocol_for(path).as_deref(),
22622 Some(WORKER_PROTOCOL_VERSION),
22623 "unexpected protocol for {path}"
22624 );
22625 }
22626
22627 assert_eq!(
22628 server.request_body("/api/worker/workflow-tasks/poll")["timeout_seconds"],
22629 1
22630 );
22631 assert_eq!(
22632 server.request_body("/api/worker/activity-tasks/poll")["timeout_seconds"],
22633 1
22634 );
22635 assert!(
22636 server.request_body("/api/worker/workflow-tasks/poll")["poll_request_id"]
22637 .as_str()
22638 .is_some_and(|id| id.starts_with("rust-workflow-poll-"))
22639 );
22640 assert!(
22641 server.request_body("/api/worker/activity-tasks/poll")["poll_request_id"]
22642 .as_str()
22643 .is_some_and(|id| id.starts_with("rust-activity-poll-"))
22644 );
22645 }
22646
22647 #[tokio::test]
22648 async fn query_task_endpoints_send_the_query_feature_protocol() {
22649 let server = MockWorkerServer::start();
22650 let client = Client::builder(server.base_url())
22651 .timeout(Duration::from_secs(2))
22652 .build()
22653 .expect("client");
22654
22655 client
22656 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
22657 .await
22658 .expect("query poll");
22659 client
22660 .complete_query_task(
22661 "query-capture",
22662 "capture-worker",
22663 1,
22664 json!(8),
22665 DEFAULT_CODEC,
22666 )
22667 .await
22668 .expect("query complete");
22669 client
22670 .fail_query_task(
22671 "query-capture",
22672 "capture-worker",
22673 1,
22674 "failed",
22675 "query_rejected",
22676 "QueryFailed",
22677 )
22678 .await
22679 .expect("query fail");
22680
22681 for path in [
22682 "/api/worker/query-tasks/poll",
22683 "/api/worker/query-tasks/query-capture/complete",
22684 "/api/worker/query-tasks/query-capture/fail",
22685 ] {
22686 assert_eq!(
22687 server.worker_protocol_for(path).as_deref(),
22688 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
22689 "unexpected protocol for {path}"
22690 );
22691 }
22692
22693 assert_eq!(
22694 server.request_body("/api/worker/query-tasks/poll")["timeout_seconds"],
22695 1
22696 );
22697 assert!(
22698 server.request_body("/api/worker/query-tasks/poll")["poll_request_id"]
22699 .as_str()
22700 .is_some_and(|id| id.starts_with("rust-query-poll-"))
22701 );
22702 }
22703
22704 #[tokio::test]
22705 async fn disconnected_client_polls_retry_once_with_the_same_request_id() {
22706 let server = MockWorkerServer::transient_worker_failures();
22707 let client = Client::builder(server.base_url())
22708 .timeout(Duration::from_secs(2))
22709 .build()
22710 .expect("client");
22711
22712 client
22713 .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
22714 .await
22715 .expect("workflow poll retry");
22716 client
22717 .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
22718 .await
22719 .expect("activity poll retry");
22720 client
22721 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
22722 .await
22723 .expect("query poll retry");
22724
22725 for path in [
22726 "/api/worker/workflow-tasks/poll",
22727 "/api/worker/activity-tasks/poll",
22728 "/api/worker/query-tasks/poll",
22729 ] {
22730 let bodies = server.request_bodies(path);
22731 assert_eq!(bodies.len(), 2, "{path} must be retried once");
22732 assert_eq!(
22733 bodies[0]["poll_request_id"], bodies[1]["poll_request_id"],
22734 "{path} must preserve the request binding across retry"
22735 );
22736 }
22737 }
22738
22739 #[tokio::test]
22740 async fn worker_poll_retries_preserve_request_id_across_consecutive_failures() {
22741 let server = MockWorkerServer::consecutive_poll_failures(2);
22742 let client = Client::builder(server.base_url())
22743 .timeout(Duration::from_secs(2))
22744 .build()
22745 .expect("client");
22746 let mut worker = Worker::new(client, "capture")
22747 .worker_id("capture-worker")
22748 .poll_timeout(Duration::from_millis(10))
22749 .retry_policy(WorkerRetryPolicy {
22750 max_retries: 2,
22751 initial_backoff: Duration::from_millis(1),
22752 max_backoff: Duration::from_millis(1),
22753 });
22754 worker.register_workflow(
22755 "capture.workflow",
22756 |_ctx, _input| async move { Ok(Value::Null) },
22757 );
22758 worker.register_activity(
22759 "capture.activity",
22760 |_ctx, _input| async move { Ok(Value::Null) },
22761 );
22762 worker.register_query("capture.workflow", "current", |_ctx, _args| async move {
22763 Ok(Value::Null)
22764 });
22765
22766 assert_eq!(worker.run_once().await.expect("poll retries"), 0);
22767
22768 for path in [
22769 "/api/worker/workflow-tasks/poll",
22770 "/api/worker/activity-tasks/poll",
22771 "/api/worker/query-tasks/poll",
22772 ] {
22773 let bodies = server.request_bodies(path);
22774 assert_eq!(bodies.len(), 3, "{path} must use exactly two retries");
22775 assert!(
22776 bodies
22777 .iter()
22778 .all(|body| body["poll_request_id"] == bodies[0]["poll_request_id"]),
22779 "{path} must preserve one request binding across every retry"
22780 );
22781 }
22782 }
22783
22784 #[tokio::test]
22785 async fn query_protocol_rejection_from_older_server_is_typed() {
22786 let server = MockWorkerServer::reject_query_protocol();
22787 let client = Client::builder(server.base_url())
22788 .timeout(Duration::from_secs(2))
22789 .build()
22790 .expect("client");
22791
22792 let error = client
22793 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
22794 .await
22795 .expect_err("server below query protocol floor must reject");
22796 let Error::Protocol(failure) = error else {
22797 panic!("expected typed protocol failure");
22798 };
22799
22800 assert_eq!(failure.status, 400);
22801 assert_eq!(failure.reason, "unsupported_protocol_version");
22802 assert_eq!(failure.supported_version.as_deref(), Some("1.7"));
22803 assert_eq!(
22804 failure.requested_version.as_deref(),
22805 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
22806 );
22807 assert_eq!(
22808 server
22809 .worker_protocol_for("/api/worker/query-tasks/poll")
22810 .as_deref(),
22811 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
22812 );
22813 }
22814
22815 #[tokio::test]
22816 async fn run_once_without_query_handlers_keeps_pre_query_server_compatibility() {
22817 let server = MockWorkerServer::reject_query_protocol();
22818 let client = Client::builder(server.base_url())
22819 .timeout(Duration::from_secs(2))
22820 .build()
22821 .expect("client");
22822 let mut worker = Worker::new(client, "rust-workers")
22823 .worker_id("baseline-worker")
22824 .poll_timeout(Duration::from_millis(10));
22825
22826 worker.register_workflow("baseline.workflow", |_ctx, _input| async move {
22827 Ok(Value::Null)
22828 });
22829
22830 assert_eq!(worker.run_once().await.expect("baseline run once"), 0);
22831 assert_eq!(
22832 server
22833 .worker_protocol_for("/api/worker/workflow-tasks/poll")
22834 .as_deref(),
22835 Some(WORKER_PROTOCOL_VERSION)
22836 );
22837 assert_eq!(
22838 server.worker_protocol_for("/api/worker/query-tasks/poll"),
22839 None,
22840 "a worker without query handlers must not use the query-task endpoint"
22841 );
22842 }
22843
22844 #[tokio::test]
22845 async fn completion_time_query_rejection_is_typed_without_stopping_worker() {
22846 let server = MockWorkerServer::reject_query_completion();
22847 let client = Client::builder(server.base_url())
22848 .timeout(Duration::from_secs(2))
22849 .build()
22850 .expect("client");
22851
22852 let error = client
22853 .complete_query_task("query-late", "late-worker", 1, json!(8), DEFAULT_CODEC)
22854 .await
22855 .expect_err("expired completion must be rejected");
22856 let Error::QueryFailed(failure) = error else {
22857 panic!("expected typed query failure");
22858 };
22859 assert_eq!(failure.status, 409);
22860 assert_eq!(failure.reason, "query_task_timed_out");
22861
22862 let mut worker = Worker::new(client, "rust-workers")
22863 .worker_id("late-worker")
22864 .poll_timeout(Duration::from_millis(10));
22865 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
22866 worker.register_query(
22867 "counter",
22868 "current",
22869 |_ctx, _args| async move { Ok(json!(8)) },
22870 );
22871
22872 assert_eq!(worker.run_once().await.expect("late task is handled"), 1);
22873 assert_eq!(
22874 worker
22875 .run_once()
22876 .await
22877 .expect("worker continues after late completion"),
22878 0
22879 );
22880 assert_eq!(
22881 server.request_count("/api/worker/query-tasks/query-late/complete"),
22882 2
22883 );
22884 assert_eq!(
22885 server.request_count("/api/worker/query-tasks/query-late/fail"),
22886 0,
22887 "a server completion rejection must not be reported as an encoding failure"
22888 );
22889 }
22890
22891 #[tokio::test]
22892 async fn normal_shutdown_joins_pollers_and_deregisters_once() {
22893 let server = MockWorkerServer::start();
22894 let client = Client::builder(server.base_url())
22895 .timeout(Duration::from_secs(2))
22896 .build()
22897 .expect("client");
22898 let mut worker = Worker::new(client, "rust-workers")
22899 .worker_id("joined-worker")
22900 .poll_timeout(Duration::from_millis(10));
22901 worker.register_workflow(
22902 "joined.workflow",
22903 |_ctx, _input| async move { Ok(Value::Null) },
22904 );
22905 worker.register_activity(
22906 "joined.activity",
22907 |_ctx, _input| async move { Ok(Value::Null) },
22908 );
22909 worker.register_query("joined.workflow", "state", |_ctx, _input| async move {
22910 Ok(Value::Null)
22911 });
22912
22913 worker
22914 .run_until(tokio::time::sleep(Duration::from_millis(20)))
22915 .await
22916 .expect("normal shutdown");
22917
22918 let deregistration_path = "/api/worker/registrations/mock-worker";
22919 assert_eq!(server.request_count(deregistration_path), 1);
22920 for poll_path in [
22921 "/api/worker/workflow-tasks/poll",
22922 "/api/worker/activity-tasks/poll",
22923 "/api/worker/query-tasks/poll",
22924 ] {
22925 assert!(server.request_count(poll_path) > 0, "missing {poll_path}");
22926 }
22927 assert_eq!(
22928 server.captured_paths().last().map(String::as_str),
22929 Some(deregistration_path),
22930 "deregistration must start only after every poller has joined"
22931 );
22932 }
22933
22934 #[tokio::test]
22935 async fn registration_failure_does_not_deregister() {
22936 let server = MockWorkerServer::rejected_registration();
22937 let client = Client::builder(server.base_url())
22938 .timeout(Duration::from_secs(2))
22939 .build()
22940 .expect("client");
22941 let worker = Worker::new(client, "rust-workers").worker_id("never-registered");
22942
22943 let error = worker
22944 .run_until(async {})
22945 .await
22946 .expect_err("registration must fail");
22947 assert!(matches!(
22948 error,
22949 Error::Http {
22950 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
22951 ..
22952 }
22953 ));
22954 assert!(server
22955 .captured_paths()
22956 .iter()
22957 .all(|path| !path.starts_with("/api/worker/registrations/")));
22958 }
22959
22960 #[tokio::test]
22961 async fn protocol_116_server_rejects_occurrence_identity_worker_registration() {
22962 let server = MockWorkerServer::rejected_registration_protocol();
22963 let client = Client::builder(server.base_url())
22964 .timeout(Duration::from_secs(2))
22965 .build()
22966 .expect("client");
22967 let worker = Worker::new(client, "rust-workers").worker_id("protocol-117-worker");
22968
22969 let error = worker
22970 .run_until(async {})
22971 .await
22972 .expect_err("a protocol 1.16 server must reject this worker");
22973 let Error::Protocol(failure) = error else {
22974 panic!("expected typed protocol rejection");
22975 };
22976 assert_eq!(failure.reason, "unsupported_protocol_version");
22977 assert_eq!(failure.supported_version.as_deref(), Some("1.16"));
22978 assert_eq!(failure.requested_version.as_deref(), Some("1.17"));
22979 assert_eq!(
22980 server
22981 .worker_protocol_for("/api/worker/register")
22982 .as_deref(),
22983 Some(WORKER_PROTOCOL_VERSION)
22984 );
22985 }
22986
22987 #[tokio::test]
22988 async fn declined_registration_does_not_deregister() {
22989 let server = MockWorkerServer::declined_registration();
22990 let client = Client::builder(server.base_url())
22991 .timeout(Duration::from_secs(2))
22992 .build()
22993 .expect("client");
22994 let worker = Worker::new(client, "rust-workers").worker_id("declined-worker");
22995
22996 let error = worker
22997 .run_until(async {})
22998 .await
22999 .expect_err("declined registration must fail");
23000 assert!(matches!(error, Error::WorkerLoop(_)));
23001 assert!(error.to_string().contains("was not accepted"));
23002 assert!(server
23003 .captured_paths()
23004 .iter()
23005 .all(|path| !path.starts_with("/api/worker/registrations/")));
23006 }
23007
23008 #[tokio::test]
23009 async fn deregistration_http_failure_is_returned_after_normal_shutdown() {
23010 let server = MockWorkerServer::rejected_deregistration();
23011 let client = Client::builder(server.base_url())
23012 .timeout(Duration::from_secs(2))
23013 .build()
23014 .expect("client");
23015 let worker = Worker::new(client, "rust-workers").worker_id("forbidden-cleanup");
23016
23017 let error = worker
23018 .run_until(async {})
23019 .await
23020 .expect_err("deregistration must fail");
23021 assert!(matches!(
23022 error,
23023 Error::Http {
23024 status: reqwest::StatusCode::FORBIDDEN,
23025 ..
23026 }
23027 ));
23028 assert_eq!(
23029 server.request_count("/api/worker/registrations/mock-worker"),
23030 1
23031 );
23032 }
23033
23034 #[tokio::test]
23035 async fn deregistration_protocol_failure_is_returned_after_normal_shutdown() {
23036 let server = MockWorkerServer::rejected_deregistration_protocol();
23037 let client = Client::builder(server.base_url())
23038 .timeout(Duration::from_secs(2))
23039 .build()
23040 .expect("client");
23041 let worker = Worker::new(client, "rust-workers").worker_id("protocol-cleanup");
23042
23043 let error = worker
23044 .run_until(async {})
23045 .await
23046 .expect_err("protocol rejection must fail shutdown");
23047 let Error::Protocol(failure) = error else {
23048 panic!("expected typed protocol failure");
23049 };
23050 assert_eq!(failure.reason, "unsupported_protocol_version");
23051 assert_eq!(
23052 failure.requested_version.as_deref(),
23053 Some(WORKER_PROTOCOL_VERSION)
23054 );
23055 assert_eq!(
23056 server.request_count("/api/worker/registrations/mock-worker"),
23057 1
23058 );
23059 }
23060
23061 #[tokio::test]
23062 async fn primary_poller_error_retains_deregistration_failure_context() {
23063 let server = MockWorkerServer::unauthorized_polls_and_rejected_deregistration();
23064 let client = Client::builder(server.base_url())
23065 .timeout(Duration::from_secs(2))
23066 .build()
23067 .expect("client");
23068 let mut worker = Worker::new(client, "rust-workers")
23069 .worker_id("combined-failure")
23070 .poll_timeout(Duration::from_millis(10));
23071 worker.register_workflow("combined.workflow", |_ctx, _input| async move {
23072 Ok(Value::Null)
23073 });
23074
23075 let error = worker
23076 .run()
23077 .await
23078 .expect_err("worker and cleanup must fail");
23079 let summary = error.to_string();
23080 assert!(summary.contains("authentication_failed"));
23081 assert!(summary.contains("worker cannot deregister"));
23082 let Error::WorkerShutdown {
23083 primary,
23084 deregistration,
23085 } = error
23086 else {
23087 panic!("expected combined worker shutdown error");
23088 };
23089 assert!(matches!(
23090 *primary,
23091 Error::Http {
23092 status: reqwest::StatusCode::UNAUTHORIZED,
23093 ..
23094 }
23095 ));
23096 assert!(matches!(
23097 *deregistration,
23098 Error::Http {
23099 status: reqwest::StatusCode::FORBIDDEN,
23100 ..
23101 }
23102 ));
23103 assert_eq!(
23104 server.request_count("/api/worker/registrations/mock-worker"),
23105 1
23106 );
23107 }
23108
23109 #[tokio::test]
23110 async fn activity_only_worker_can_shutdown_without_workflow_poller() {
23111 let server = MockWorkerServer::start();
23112 let client = Client::builder(server.base_url())
23113 .timeout(Duration::from_secs(2))
23114 .build()
23115 .expect("client");
23116 let mut worker = Worker::new(client, "rust-workers")
23117 .worker_id("activity-only-worker")
23118 .poll_timeout(Duration::from_millis(10));
23119
23120 worker.register_activity(
23121 "activity.only",
23122 |_ctx, _args| async move { Ok(Value::Null) },
23123 );
23124
23125 worker.run_until(async {}).await.expect("run worker");
23126 }
23127
23128 #[tokio::test]
23129 async fn workflow_only_worker_can_shutdown_without_activity_poller() {
23130 let server = MockWorkerServer::start();
23131 let client = Client::builder(server.base_url())
23132 .timeout(Duration::from_secs(2))
23133 .build()
23134 .expect("client");
23135 let mut worker = Worker::new(client, "rust-workers")
23136 .worker_id("workflow-only-worker")
23137 .poll_timeout(Duration::from_millis(10));
23138
23139 worker.register_workflow(
23140 "workflow.only",
23141 |_ctx, _input| async move { Ok(Value::Null) },
23142 );
23143
23144 worker.run_until(async {}).await.expect("run worker");
23145 }
23146
23147 #[tokio::test]
23148 async fn worker_heartbeat_observer_receives_server_acknowledgements() {
23149 let server = MockWorkerServer::start();
23150 let client = Client::builder(server.base_url())
23151 .timeout(Duration::from_secs(2))
23152 .build()
23153 .expect("client");
23154 let observations = Arc::new(Mutex::new(Vec::new()));
23155 let observed = Arc::clone(&observations);
23156 let mut worker = Worker::new(client, "rust-workers")
23157 .worker_id("observed-heartbeat-worker")
23158 .poll_timeout(Duration::from_millis(10))
23159 .on_worker_heartbeat(move |observation| {
23160 observed
23161 .lock()
23162 .expect("heartbeat observations")
23163 .push(observation.clone());
23164 });
23165
23166 worker.register_workflow("workflow.observed", |_ctx, _input| async move {
23167 Ok(Value::Null)
23168 });
23169 let acknowledged = Arc::clone(&observations);
23170 worker
23171 .run_until(async move {
23172 tokio::time::timeout(Duration::from_secs(2), async move {
23173 loop {
23174 if !acknowledged
23175 .lock()
23176 .expect("heartbeat observations")
23177 .is_empty()
23178 {
23179 break;
23180 }
23181 tokio::time::sleep(Duration::from_millis(1)).await;
23182 }
23183 })
23184 .await
23185 .expect("heartbeat acknowledgement within timeout");
23186 })
23187 .await
23188 .expect("run worker");
23189
23190 let observations = observations.lock().expect("heartbeat observations");
23191 let first = observations.first().expect("heartbeat acknowledgement");
23192 assert_eq!(first.worker_id, "observed-heartbeat-worker");
23193 assert_eq!(first.task_queue, "rust-workers");
23194 assert!(first.acknowledged_at_unix_millis > 0);
23195 assert_eq!(first.acknowledgement, json!({}));
23196 }
23197
23198 #[tokio::test]
23199 async fn delayed_worker_heartbeat_keeps_cadence_and_pollers_live() {
23200 let server = MockWorkerServer::delayed_heartbeat_worker();
23201 let client = Client::builder(server.base_url())
23202 .timeout(Duration::from_secs(3))
23203 .build()
23204 .expect("client");
23205 let observations = Arc::new(Mutex::new(Vec::new()));
23206 let observed = Arc::clone(&observations);
23207 let mut worker = Worker::new(client, "rust-snapshot-workers")
23208 .worker_id("rust-snapshot-worker")
23209 .poll_timeout(Duration::from_millis(10))
23210 .on_worker_heartbeat(move |observation| {
23211 observed
23212 .lock()
23213 .expect("heartbeat observations")
23214 .push(observation.clone());
23215 });
23216
23217 worker.register_workflow("snapshot", |ctx, _input| async move {
23218 ctx.wait_signal("finish").await?;
23219 Ok(json!({"status": "finished"}))
23220 });
23221 worker.register_query("snapshot", "current", |ctx, _args| async move {
23222 Ok(json!(ctx
23223 .signals("increment")
23224 .iter()
23225 .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
23226 .sum::<i64>()))
23227 });
23228 worker.register_activity("cancel-aware", |_ctx, _args| async move {
23229 Ok(json!({"late": "completion"}))
23230 });
23231
23232 worker
23233 .run_until(tokio::time::sleep(Duration::from_millis(3_800)))
23234 .await
23235 .expect("delayed heartbeat must allow a clean worker shutdown");
23236
23237 let observations = observations.lock().expect("heartbeat observations");
23238 assert!(
23239 observations.len() >= 3,
23240 "the immediate heartbeat, delayed acknowledgement, and next cadence heartbeat must complete"
23241 );
23242 assert!(
23243 observations.windows(2).all(|pair| {
23244 pair[1].acknowledged_at_unix_millis
23245 .saturating_sub(pair[0].acknowledged_at_unix_millis)
23246 >= 850
23247 }),
23248 "successful acknowledgements must not catch up faster than the advertised one-second cadence: {observations:?}"
23249 );
23250 drop(observations);
23251
23252 let heartbeat_times = server.request_times("/api/worker/heartbeat");
23253 let delayed_request_at = *heartbeat_times
23254 .get(1)
23255 .expect("intentionally delayed heartbeat request");
23256 let delay_window_start = delayed_request_at + Duration::from_millis(100);
23257 let delay_window_end = delayed_request_at + Duration::from_millis(1_400);
23258 for path in [
23259 "/api/worker/workflow-tasks/poll",
23260 "/api/worker/activity-tasks/poll",
23261 "/api/worker/query-tasks/poll",
23262 ] {
23263 assert!(
23264 server
23265 .request_times(path)
23266 .iter()
23267 .any(|received_at| *received_at >= delay_window_start
23268 && *received_at <= delay_window_end),
23269 "{path} must keep polling while a heartbeat acknowledgement is delayed"
23270 );
23271 }
23272 assert!(
23273 server.request_count("/api/worker/workflow-tasks/snapshot-wait-3/fail") >= 1,
23274 "workflow work must be settled"
23275 );
23276 assert!(
23277 server.request_count("/api/worker/activity-tasks/activity-cancel/complete") >= 1,
23278 "activity work must be settled"
23279 );
23280 assert!(
23281 server.request_count("/api/worker/query-tasks/snapshot-current/complete") >= 1,
23282 "query work must be settled"
23283 );
23284 }
23285
23286 #[tokio::test]
23287 async fn retried_worker_heartbeat_restarts_the_advertised_cadence() {
23288 let server = MockWorkerServer::heartbeat_retry_worker();
23289 let client = Client::builder(server.base_url())
23290 .timeout(Duration::from_secs(2))
23291 .build()
23292 .expect("client");
23293 let observations = Arc::new(Mutex::new(Vec::new()));
23294 let observed = Arc::clone(&observations);
23295 let worker = Worker::new(client, "rust-workers")
23296 .worker_id("heartbeat-retry-worker")
23297 .retry_policy(WorkerRetryPolicy {
23298 max_retries: 1,
23299 initial_backoff: Duration::from_millis(300),
23300 max_backoff: Duration::from_millis(300),
23301 })
23302 .on_worker_heartbeat(move |observation| {
23303 observed
23304 .lock()
23305 .expect("heartbeat observations")
23306 .push(observation.clone());
23307 });
23308
23309 worker
23310 .run_until(tokio::time::sleep(Duration::from_millis(2_700)))
23311 .await
23312 .expect("retryable heartbeat failure must remain bounded and recover");
23313
23314 let observations = observations.lock().expect("heartbeat observations");
23315 assert!(observations.len() >= 3, "heartbeat retry must recover");
23316 assert!(
23317 observations.windows(2).all(|pair| {
23318 pair[1]
23319 .acknowledged_at_unix_millis
23320 .saturating_sub(pair[0].acknowledged_at_unix_millis)
23321 >= 850
23322 }),
23323 "a successful retry must start a fresh advertised cadence: {observations:?}"
23324 );
23325 assert_eq!(
23326 server.request_count("/api/worker/heartbeat"),
23327 observations.len() + 1,
23328 "one retryable failure must add exactly one bounded request"
23329 );
23330 }
23331
23332 #[tokio::test]
23333 async fn query_enabled_worker_ignores_unmatched_signals_then_completes_once() {
23334 let server = MockWorkerServer::waiting_query_worker();
23335 let client = Client::builder(server.base_url())
23336 .timeout(Duration::from_secs(2))
23337 .build()
23338 .expect("client");
23339 let observations = Arc::new(Mutex::new(Vec::new()));
23340 let observed = Arc::clone(&observations);
23341 let mut worker = Worker::new(client, "rust-snapshot-workers")
23342 .worker_id("rust-snapshot-worker")
23343 .poll_timeout(Duration::from_millis(10))
23344 .on_worker_heartbeat(move |observation| {
23345 observed
23346 .lock()
23347 .expect("heartbeat observations")
23348 .push(observation.clone());
23349 });
23350
23351 worker.register_workflow("snapshot", |ctx, _input| async move {
23352 ctx.wait_signal("finish").await?;
23353 Ok(json!({"status": "finished"}))
23354 });
23355 worker.register_query("snapshot", "current", |ctx, _args| async move {
23356 let current = ctx
23357 .signals("increment")
23358 .iter()
23359 .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
23360 .sum::<i64>();
23361 Ok(json!(current))
23362 });
23363 worker.register_update("snapshot", "replace", |_ctx, args| async move { Ok(args) });
23364
23365 worker
23366 .run_until(tokio::time::sleep(Duration::from_millis(3_200)))
23367 .await
23368 .expect("pending workflow and query poller must remain live until shutdown");
23369
23370 assert!(
23371 observations.lock().expect("heartbeat observations").len() >= 4,
23372 "the immediate heartbeat and at least three advertised one-second intervals must be acknowledged"
23373 );
23374 assert!(
23375 server.request_count("/api/worker/workflow-tasks/poll") >= 3,
23376 "workflow polling must continue after empty replay acknowledgements"
23377 );
23378 assert!(
23379 server.request_count("/api/worker/query-tasks/poll") >= 2,
23380 "query polling must continue after serving the current query"
23381 );
23382 assert_eq!(
23383 server.request_body("/api/worker/register")["capabilities"],
23384 json!([
23385 CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY,
23386 DURABLE_SELECTION_CAPABILITY,
23387 MEMO_UPSERTS_CAPABILITY,
23388 TYPED_SEARCH_ATTRIBUTES_CAPABILITY,
23389 QUERY_TASKS_CAPABILITY,
23390 WORKFLOW_UPDATES_CAPABILITY,
23391 MESSAGE_STREAMS_CAPABILITY
23392 ])
23393 );
23394 assert_eq!(
23395 server.request_body("/api/worker/register")["workflow_command_contracts"]["snapshot"],
23396 json!({
23397 "queries": ["current"],
23398 "query_contracts": [],
23399 "signals": [],
23400 "signal_contracts": [],
23401 "updates": ["replace"],
23402 "update_contracts": [],
23403 "update_validators": [],
23404 })
23405 );
23406
23407 let opened = server.request_body("/api/worker/workflow-tasks/snapshot-open/complete");
23408 assert_eq!(
23409 opened["commands"],
23410 json!([{
23411 "type": "open_signal_wait",
23412 "signal_name": "finish",
23413 }])
23414 );
23415
23416 for task_id in ["snapshot-wait-3", "snapshot-wait-5"] {
23417 let fail_path = format!("/api/worker/workflow-tasks/{task_id}/fail");
23418 let completion_path = format!("/api/worker/workflow-tasks/{task_id}/complete");
23419 let failure = server.request_body(&fail_path);
23420 assert_eq!(
23421 failure["failure"]["type"],
23422 WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE
23423 );
23424 assert_eq!(server.request_count(&completion_path), 0);
23425 }
23426
23427 let query_completion =
23428 server.request_body("/api/worker/query-tasks/snapshot-current/complete");
23429 assert_eq!(query_completion["result"], json!(8));
23430
23431 let terminal_path = "/api/worker/workflow-tasks/snapshot-finish/complete";
23432 assert_eq!(
23433 server.request_count(terminal_path),
23434 1,
23435 "the matching signal must settle the workflow exactly once"
23436 );
23437 let terminal = server.request_body(terminal_path);
23438 assert_eq!(terminal["commands"].as_array().map(Vec::len), Some(1));
23439 assert_eq!(terminal["commands"][0]["type"], "complete_workflow");
23440 assert_eq!(
23441 decode_wire_value(&terminal["commands"][0]["result"], DEFAULT_CODEC)
23442 .expect("terminal workflow result"),
23443 json!({"status": "finished"})
23444 );
23445 }
23446
23447 #[tokio::test]
23448 async fn worker_retries_poll_and_heartbeat_transport_failures_independently() {
23449 let server = MockWorkerServer::transient_worker_failures();
23450 let client = Client::builder(server.base_url())
23451 .timeout(Duration::from_secs(2))
23452 .build()
23453 .expect("client");
23454 let mut worker = Worker::new(client, "rust-workers")
23455 .worker_id("retry-worker")
23456 .poll_timeout(Duration::from_millis(10))
23457 .retry_policy(WorkerRetryPolicy {
23458 max_retries: 2,
23459 initial_backoff: Duration::from_millis(1),
23460 max_backoff: Duration::from_millis(1),
23461 });
23462 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
23463 worker.register_activity(
23464 "counter.activity",
23465 |_ctx, _input| async move { Ok(Value::Null) },
23466 );
23467 worker.register_query(
23468 "counter",
23469 "current",
23470 |_ctx, _args| async move { Ok(json!(8)) },
23471 );
23472
23473 worker
23474 .run_until(tokio::time::sleep(Duration::from_millis(75)))
23475 .await
23476 .expect("transient failures must not stop the worker");
23477
23478 for path in [
23479 "/api/worker/heartbeat",
23480 "/api/worker/workflow-tasks/poll",
23481 "/api/worker/activity-tasks/poll",
23482 "/api/worker/query-tasks/poll",
23483 ] {
23484 assert!(
23485 server.request_count(path) >= 2,
23486 "{path} must continue after its transient failure"
23487 );
23488 }
23489 }
23490
23491 #[tokio::test]
23492 async fn worker_bounds_transport_retries() {
23493 let server = MockWorkerServer::unavailable_polls();
23494 let client = Client::builder(server.base_url())
23495 .timeout(Duration::from_secs(2))
23496 .build()
23497 .expect("client");
23498 let mut worker = Worker::new(client, "rust-workers")
23499 .worker_id("bounded-retry-worker")
23500 .poll_timeout(Duration::from_millis(10))
23501 .retry_policy(WorkerRetryPolicy {
23502 max_retries: 2,
23503 initial_backoff: Duration::from_millis(1),
23504 max_backoff: Duration::from_millis(1),
23505 });
23506 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
23507
23508 let error = worker.run().await.expect_err("retry bound must terminate");
23509 assert!(matches!(error, Error::Transport(_)));
23510 assert_eq!(
23511 server.request_count("/api/worker/workflow-tasks/poll"),
23512 3,
23513 "one initial request plus exactly two retries"
23514 );
23515 }
23516
23517 #[tokio::test]
23518 async fn worker_retry_policy_can_disable_poll_retries() {
23519 let server = MockWorkerServer::unavailable_polls();
23520 let client = Client::builder(server.base_url())
23521 .timeout(Duration::from_secs(2))
23522 .build()
23523 .expect("client");
23524 let mut worker = Worker::new(client, "rust-workers")
23525 .worker_id("no-retry-worker")
23526 .poll_timeout(Duration::from_millis(10))
23527 .retry_policy(WorkerRetryPolicy {
23528 max_retries: 0,
23529 initial_backoff: Duration::from_millis(1),
23530 max_backoff: Duration::from_millis(1),
23531 });
23532 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
23533
23534 let error = worker
23535 .run_once()
23536 .await
23537 .expect_err("disabled retries must return the first transport failure");
23538 assert!(matches!(error, Error::Transport(_)));
23539 assert_eq!(
23540 server.request_count("/api/worker/workflow-tasks/poll"),
23541 1,
23542 "max_retries=0 must send only the initial request"
23543 );
23544 }
23545
23546 #[tokio::test]
23547 async fn worker_does_not_retry_authentication_failures() {
23548 let server = MockWorkerServer::unauthorized_polls();
23549 let client = Client::builder(server.base_url())
23550 .timeout(Duration::from_secs(2))
23551 .build()
23552 .expect("client");
23553 let mut worker = Worker::new(client, "rust-workers")
23554 .worker_id("unauthorized-worker")
23555 .poll_timeout(Duration::from_millis(10));
23556 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
23557
23558 let error = worker
23559 .run()
23560 .await
23561 .expect_err("authentication must terminate");
23562 let Error::Http { status, body } = error else {
23563 panic!("expected stable HTTP authentication error");
23564 };
23565 assert_eq!(status, reqwest::StatusCode::UNAUTHORIZED);
23566 assert!(body.contains("authentication_failed"));
23567 assert_eq!(
23568 server.request_count("/api/worker/workflow-tasks/poll"),
23569 1,
23570 "authentication failures must not be retried"
23571 );
23572 }
23573
23574 #[derive(Clone, Debug)]
23575 struct CapturedRequest {
23576 method: String,
23577 path: String,
23578 authorization: Option<String>,
23579 namespace: Option<String>,
23580 worker_protocol: Option<String>,
23581 control_protocol: Option<String>,
23582 body: String,
23583 received_at: Instant,
23584 }
23585
23586 struct MockWorkerServer {
23587 addr: SocketAddr,
23588 stop: Arc<AtomicBool>,
23589 requests: Arc<Mutex<Vec<CapturedRequest>>>,
23590 thread: Option<thread::JoinHandle<()>>,
23591 }
23592
23593 #[derive(Clone, Copy, Default)]
23594 struct MockWorkerBehavior {
23595 reject_query_protocol: bool,
23596 reject_query_completion: bool,
23597 waiting_query_worker: bool,
23598 decline_registration: bool,
23599 complete_named_signal: bool,
23600 poll_failures_per_path: usize,
23601 heartbeat_failures: usize,
23602 heartbeat_failure_request: Option<usize>,
23603 delayed_heartbeat_request: Option<usize>,
23604 heartbeat_response_delay: Duration,
23605 concurrent_requests: bool,
23606 unauthorized_polls: bool,
23607 reject_registration: bool,
23608 reject_registration_protocol: bool,
23609 reject_deregistration: bool,
23610 reject_deregistration_protocol: bool,
23611 cancelled_activity: bool,
23612 draining_polls: bool,
23613 invalid_task_payload_codec: Option<InvalidTaskPayloadCodec>,
23614 workflow_completion_status: Option<&'static str>,
23615 workflow_completion_body: Option<&'static str>,
23616 }
23617
23618 impl MockWorkerServer {
23619 fn start() -> Self {
23620 Self::start_with_behavior(MockWorkerBehavior::default())
23621 }
23622
23623 fn reject_query_protocol() -> Self {
23624 Self::start_with_behavior(MockWorkerBehavior {
23625 reject_query_protocol: true,
23626 ..MockWorkerBehavior::default()
23627 })
23628 }
23629
23630 fn reject_query_completion() -> Self {
23631 Self::start_with_behavior(MockWorkerBehavior {
23632 reject_query_completion: true,
23633 ..MockWorkerBehavior::default()
23634 })
23635 }
23636
23637 fn waiting_query_worker() -> Self {
23638 Self::start_with_behavior(MockWorkerBehavior {
23639 waiting_query_worker: true,
23640 complete_named_signal: true,
23641 ..MockWorkerBehavior::default()
23642 })
23643 }
23644
23645 fn transient_worker_failures() -> Self {
23646 Self::start_with_behavior(MockWorkerBehavior {
23647 poll_failures_per_path: 1,
23648 heartbeat_failures: 1,
23649 ..MockWorkerBehavior::default()
23650 })
23651 }
23652
23653 fn consecutive_poll_failures(count: usize) -> Self {
23654 Self::start_with_behavior(MockWorkerBehavior {
23655 poll_failures_per_path: count,
23656 ..MockWorkerBehavior::default()
23657 })
23658 }
23659
23660 fn delayed_heartbeat_worker() -> Self {
23661 Self::start_with_behavior(MockWorkerBehavior {
23662 waiting_query_worker: true,
23663 delayed_heartbeat_request: Some(2),
23664 heartbeat_response_delay: Duration::from_millis(1_500),
23665 concurrent_requests: true,
23666 cancelled_activity: true,
23667 ..MockWorkerBehavior::default()
23668 })
23669 }
23670
23671 fn heartbeat_retry_worker() -> Self {
23672 Self::start_with_behavior(MockWorkerBehavior {
23673 waiting_query_worker: true,
23674 heartbeat_failure_request: Some(2),
23675 concurrent_requests: true,
23676 ..MockWorkerBehavior::default()
23677 })
23678 }
23679
23680 fn unavailable_polls() -> Self {
23681 Self::start_with_behavior(MockWorkerBehavior {
23682 poll_failures_per_path: usize::MAX,
23683 ..MockWorkerBehavior::default()
23684 })
23685 }
23686
23687 fn unauthorized_polls() -> Self {
23688 Self::start_with_behavior(MockWorkerBehavior {
23689 unauthorized_polls: true,
23690 ..MockWorkerBehavior::default()
23691 })
23692 }
23693
23694 fn rejected_registration() -> Self {
23695 Self::start_with_behavior(MockWorkerBehavior {
23696 reject_registration: true,
23697 ..MockWorkerBehavior::default()
23698 })
23699 }
23700
23701 fn rejected_registration_protocol() -> Self {
23702 Self::start_with_behavior(MockWorkerBehavior {
23703 reject_registration_protocol: true,
23704 ..MockWorkerBehavior::default()
23705 })
23706 }
23707
23708 fn declined_registration() -> Self {
23709 Self::start_with_behavior(MockWorkerBehavior {
23710 decline_registration: true,
23711 ..MockWorkerBehavior::default()
23712 })
23713 }
23714
23715 fn rejected_deregistration() -> Self {
23716 Self::start_with_behavior(MockWorkerBehavior {
23717 reject_deregistration: true,
23718 ..MockWorkerBehavior::default()
23719 })
23720 }
23721
23722 fn rejected_deregistration_protocol() -> Self {
23723 Self::start_with_behavior(MockWorkerBehavior {
23724 reject_deregistration_protocol: true,
23725 ..MockWorkerBehavior::default()
23726 })
23727 }
23728
23729 fn unauthorized_polls_and_rejected_deregistration() -> Self {
23730 Self::start_with_behavior(MockWorkerBehavior {
23731 unauthorized_polls: true,
23732 reject_deregistration: true,
23733 ..MockWorkerBehavior::default()
23734 })
23735 }
23736
23737 fn cancelled_activity() -> Self {
23738 Self::start_with_behavior(MockWorkerBehavior {
23739 cancelled_activity: true,
23740 ..MockWorkerBehavior::default()
23741 })
23742 }
23743
23744 fn draining_polls() -> Self {
23745 Self::start_with_behavior(MockWorkerBehavior {
23746 draining_polls: true,
23747 ..MockWorkerBehavior::default()
23748 })
23749 }
23750
23751 fn invalid_task_payload_codec(codec: InvalidTaskPayloadCodec) -> Self {
23752 Self::start_with_behavior(MockWorkerBehavior {
23753 invalid_task_payload_codec: Some(codec),
23754 ..MockWorkerBehavior::default()
23755 })
23756 }
23757
23758 fn workflow_completion(status: &'static str, body: &'static str) -> Self {
23759 Self::start_with_behavior(MockWorkerBehavior {
23760 workflow_completion_status: Some(status),
23761 workflow_completion_body: Some(body),
23762 ..MockWorkerBehavior::default()
23763 })
23764 }
23765
23766 fn start_with_behavior(behavior: MockWorkerBehavior) -> Self {
23767 let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock server");
23768 listener
23769 .set_nonblocking(true)
23770 .expect("configure mock listener");
23771 let addr = listener.local_addr().expect("mock server address");
23772 let stop = Arc::new(AtomicBool::new(false));
23773 let server_stop = Arc::clone(&stop);
23774 let requests = Arc::new(Mutex::new(Vec::new()));
23775 let server_requests = Arc::clone(&requests);
23776 let thread = thread::spawn(move || {
23777 let mut request_threads = Vec::new();
23778 while !server_stop.load(Ordering::SeqCst) {
23779 match listener.accept() {
23780 Ok((mut stream, _)) => {
23781 if behavior.concurrent_requests {
23782 let requests = Arc::clone(&server_requests);
23783 request_threads.push(thread::spawn(move || {
23784 handle_mock_worker_request(&mut stream, &requests, behavior)
23785 }));
23786 } else {
23787 handle_mock_worker_request(&mut stream, &server_requests, behavior);
23788 }
23789 }
23790 Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
23791 let mut index = 0;
23792 while index < request_threads.len() {
23793 if request_threads[index].is_finished() {
23794 request_threads
23795 .swap_remove(index)
23796 .join()
23797 .expect("join mock request");
23798 } else {
23799 index += 1;
23800 }
23801 }
23802 thread::sleep(Duration::from_millis(5));
23803 }
23804 Err(_) => break,
23805 }
23806 }
23807 for request_thread in request_threads {
23808 request_thread.join().expect("join mock request");
23809 }
23810 });
23811
23812 Self {
23813 addr,
23814 stop,
23815 requests,
23816 thread: Some(thread),
23817 }
23818 }
23819
23820 fn base_url(&self) -> String {
23821 format!("http://{}", self.addr)
23822 }
23823
23824 fn worker_protocol_for(&self, path: &str) -> Option<String> {
23825 self.requests
23826 .lock()
23827 .expect("captured requests")
23828 .iter()
23829 .find(|request| request.path == path)
23830 .and_then(|request| request.worker_protocol.clone())
23831 }
23832
23833 fn control_protocol_for(&self, path: &str) -> Option<String> {
23834 self.requests
23835 .lock()
23836 .expect("captured requests")
23837 .iter()
23838 .find(|request| request.path == path)
23839 .and_then(|request| request.control_protocol.clone())
23840 }
23841
23842 fn method_for(&self, path: &str) -> Option<String> {
23843 self.requests
23844 .lock()
23845 .expect("captured requests")
23846 .iter()
23847 .find(|request| request.path == path)
23848 .map(|request| request.method.clone())
23849 }
23850
23851 fn authorization_for(&self, path: &str) -> Option<String> {
23852 self.requests
23853 .lock()
23854 .expect("captured requests")
23855 .iter()
23856 .find(|request| request.path == path)
23857 .and_then(|request| request.authorization.clone())
23858 }
23859
23860 fn namespace_for(&self, path: &str) -> Option<String> {
23861 self.requests
23862 .lock()
23863 .expect("captured requests")
23864 .iter()
23865 .find(|request| request.path == path)
23866 .and_then(|request| request.namespace.clone())
23867 }
23868
23869 fn request_count(&self, path: &str) -> usize {
23870 self.requests
23871 .lock()
23872 .expect("captured requests")
23873 .iter()
23874 .filter(|request| request.path == path)
23875 .count()
23876 }
23877
23878 fn captured_paths(&self) -> Vec<String> {
23879 self.requests
23880 .lock()
23881 .expect("captured requests")
23882 .iter()
23883 .map(|request| request.path.clone())
23884 .collect()
23885 }
23886
23887 fn request_times(&self, path: &str) -> Vec<Instant> {
23888 self.requests
23889 .lock()
23890 .expect("captured requests")
23891 .iter()
23892 .filter(|request| request.path == path)
23893 .map(|request| request.received_at)
23894 .collect()
23895 }
23896
23897 fn request_body(&self, path: &str) -> Value {
23898 let requests = self.requests.lock().expect("captured requests");
23899 let body = &requests
23900 .iter()
23901 .find(|request| request.path == path)
23902 .unwrap_or_else(|| panic!("missing request for {path}"))
23903 .body;
23904 serde_json::from_str(body).unwrap_or_else(|error| {
23905 panic!("invalid JSON request body for {path}: {error}: {body:?}")
23906 })
23907 }
23908
23909 fn request_bodies(&self, path: &str) -> Vec<Value> {
23910 self.requests
23911 .lock()
23912 .expect("captured requests")
23913 .iter()
23914 .filter(|request| request.path == path)
23915 .map(|request| {
23916 serde_json::from_str(&request.body).unwrap_or_else(|error| {
23917 panic!(
23918 "invalid JSON request body for {path}: {error}: {:?}",
23919 request.body
23920 )
23921 })
23922 })
23923 .collect()
23924 }
23925 }
23926
23927 impl Drop for MockWorkerServer {
23928 fn drop(&mut self) {
23929 self.stop.store(true, Ordering::SeqCst);
23930 let _ = TcpStream::connect(self.addr);
23931
23932 if let Some(thread) = self.thread.take() {
23933 thread.join().expect("join mock server");
23934 }
23935 }
23936 }
23937
23938 fn handle_mock_worker_request(
23939 stream: &mut TcpStream,
23940 requests: &Arc<Mutex<Vec<CapturedRequest>>>,
23941 behavior: MockWorkerBehavior,
23942 ) {
23943 let _ = stream.set_read_timeout(Some(Duration::from_millis(200)));
23944 let mut buffer = [0_u8; 8192];
23945 let mut request = Vec::new();
23946
23947 loop {
23948 match stream.read(&mut buffer) {
23949 Ok(0) => break,
23950 Ok(read) => {
23951 request.extend_from_slice(&buffer[..read]);
23952 if mock_request_is_complete(&request) {
23953 break;
23954 }
23955 }
23956 Err(error)
23957 if matches!(
23958 error.kind(),
23959 std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
23960 ) =>
23961 {
23962 break;
23963 }
23964 Err(_) => return,
23965 }
23966 }
23967
23968 let request = String::from_utf8_lossy(&request);
23969 let body = request
23970 .split_once("\r\n\r\n")
23971 .map(|(_, body)| body)
23972 .unwrap_or_default();
23973 let path = request
23974 .lines()
23975 .next()
23976 .and_then(|line| line.split_whitespace().nth(1))
23977 .unwrap_or_default();
23978 let method = request
23979 .lines()
23980 .next()
23981 .and_then(|line| line.split_whitespace().next())
23982 .unwrap_or_default();
23983 let authorization = request.lines().find_map(|line| {
23984 let (name, value) = line.split_once(':')?;
23985 name.eq_ignore_ascii_case("Authorization")
23986 .then(|| value.trim().to_string())
23987 });
23988 let namespace = request.lines().find_map(|line| {
23989 let (name, value) = line.split_once(':')?;
23990 name.eq_ignore_ascii_case("X-Namespace")
23991 .then(|| value.trim().to_string())
23992 });
23993 let worker_protocol = request.lines().find_map(|line| {
23994 let (name, value) = line.split_once(':')?;
23995 name.eq_ignore_ascii_case("X-Durable-Workflow-Protocol-Version")
23996 .then(|| value.trim().to_string())
23997 });
23998 let control_protocol = request.lines().find_map(|line| {
23999 let (name, value) = line.split_once(':')?;
24000 name.eq_ignore_ascii_case("X-Durable-Workflow-Control-Plane-Version")
24001 .then(|| value.trim().to_string())
24002 });
24003 let request_number = {
24004 let mut requests = requests.lock().expect("captured requests");
24005 requests.push(CapturedRequest {
24006 method: method.to_string(),
24007 path: path.to_string(),
24008 authorization,
24009 namespace,
24010 worker_protocol: worker_protocol.clone(),
24011 control_protocol,
24012 body: body.to_string(),
24013 received_at: Instant::now(),
24014 });
24015 requests
24016 .iter()
24017 .filter(|request| request.path == path)
24018 .count()
24019 };
24020
24021 if path == "/api/worker/register" {
24022 if behavior.reject_registration_protocol {
24023 write_mock_response(
24024 stream,
24025 "400 Bad Request",
24026 r#"{"reason":"unsupported_protocol_version","message":"condition-wait occurrence identity requires worker protocol 1.17","supported_version":"1.16","requested_version":"1.17"}"#,
24027 );
24028 return;
24029 }
24030 if behavior.reject_registration {
24031 write_mock_response(
24032 stream,
24033 "503 Service Unavailable",
24034 r#"{"reason":"registration_unavailable","message":"registration failed"}"#,
24035 );
24036 return;
24037 }
24038 }
24039
24040 if path.starts_with("/api/worker/registrations/") {
24041 if behavior.reject_deregistration_protocol {
24042 write_mock_response(
24043 stream,
24044 "400 Bad Request",
24045 r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.17","requested_version":"1.19"}"#,
24046 );
24047 } else if behavior.reject_deregistration {
24048 write_mock_response(
24049 stream,
24050 "403 Forbidden",
24051 r#"{"reason":"authorization_failed","message":"worker cannot deregister"}"#,
24052 );
24053 } else {
24054 write_mock_response(
24055 stream,
24056 "200 OK",
24057 r#"{"worker_id":"deregistered-worker","outcome":"deregistered","recovered_workflow_task_count":2}"#,
24058 );
24059 }
24060 return;
24061 }
24062
24063 let is_poll = matches!(
24064 path,
24065 "/api/worker/workflow-tasks/poll"
24066 | "/api/worker/activity-tasks/poll"
24067 | "/api/worker/query-tasks/poll"
24068 );
24069 if is_poll && request_number <= behavior.poll_failures_per_path {
24070 return;
24071 }
24072 if path == "/api/worker/heartbeat" && request_number <= behavior.heartbeat_failures {
24073 return;
24074 }
24075 if path == "/api/worker/heartbeat"
24076 && behavior.heartbeat_failure_request == Some(request_number)
24077 {
24078 return;
24079 }
24080 if path == "/api/worker/heartbeat"
24081 && behavior.delayed_heartbeat_request == Some(request_number)
24082 {
24083 thread::sleep(behavior.heartbeat_response_delay);
24084 }
24085 if behavior.unauthorized_polls && is_poll {
24086 write_mock_response(
24087 stream,
24088 "401 Unauthorized",
24089 r#"{"reason":"authentication_failed","message":"invalid worker token"}"#,
24090 );
24091 return;
24092 }
24093 if behavior.draining_polls && is_poll {
24094 write_mock_response(
24095 stream,
24096 "409 Conflict",
24097 r#"{"task":null,"poll_status":"draining","reason":"worker_draining","worker_status":"draining","drain_intent":"draining"}"#,
24098 );
24099 return;
24100 }
24101
24102 if let Some(codec_case) = behavior.invalid_task_payload_codec {
24103 if is_poll && request_number == 1 {
24104 let mut task = match path {
24105 "/api/worker/workflow-tasks/poll" => json!({
24106 "task_id": "codec-workflow",
24107 "workflow_type": "codec.workflow",
24108 "payload_codec": DEFAULT_CODEC,
24109 "workflow_task_attempt": 1,
24110 "lease_owner": "codec-worker"
24111 }),
24112 "/api/worker/activity-tasks/poll" => json!({
24113 "task_id": "codec-activity",
24114 "activity_attempt_id": "codec-activity-attempt",
24115 "activity_type": "codec.activity",
24116 "payload_codec": DEFAULT_CODEC,
24117 "attempt_number": 1,
24118 "lease_owner": "codec-worker"
24119 }),
24120 "/api/worker/query-tasks/poll" => json!({
24121 "query_task_id": "codec-query",
24122 "query_task_attempt": 1,
24123 "workflow_type": "codec.workflow",
24124 "query_name": "known",
24125 "payload_codec": DEFAULT_CODEC,
24126 "lease_owner": "codec-worker"
24127 }),
24128 _ => unreachable!("is_poll limits task codec probe paths"),
24129 };
24130 codec_case.apply(&mut task);
24131 write_mock_response(stream, "200 OK", &json!({"task": task}).to_string());
24132 return;
24133 }
24134
24135 if matches!(
24136 path,
24137 "/api/worker/workflow-tasks/codec-workflow/fail"
24138 | "/api/worker/activity-tasks/codec-activity/fail"
24139 | "/api/worker/query-tasks/codec-query/fail"
24140 ) {
24141 write_mock_response(stream, "200 OK", r#"{"outcome":"failed"}"#);
24142 return;
24143 }
24144 }
24145
24146 if behavior.reject_query_protocol && path.starts_with("/api/worker/query-tasks/") {
24147 let requested_version = worker_protocol.as_deref().unwrap_or("missing");
24148 let body = format!(
24149 r#"{{"reason":"unsupported_protocol_version","message":"Query tasks require worker protocol 1.8 or newer.","supported_version":"1.7","requested_version":"{requested_version}"}}"#
24150 );
24151 write_mock_response(stream, "400 Bad Request", &body);
24152 return;
24153 }
24154
24155 if behavior.reject_query_completion && path == "/api/worker/query-tasks/query-late/complete"
24156 {
24157 write_mock_response(
24158 stream,
24159 "409 Conflict",
24160 r#"{"reason":"query_task_timed_out","message":"query task timed out before completion"}"#,
24161 );
24162 return;
24163 }
24164
24165 if behavior.workflow_completion_status.is_some()
24166 && path == "/api/worker/workflow-tasks/poll"
24167 && request_number == 1
24168 {
24169 write_mock_response(
24170 stream,
24171 "200 OK",
24172 r#"{"task":{"task_id":"workflow-timeout-task","workflow_id":"reused-workflow-id","run_id":"run-selected-timeout","workflow_type":"timeout.workflow","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"workflow_task_attempt":3,"lease_owner":"timeout-worker"}}"#,
24173 );
24174 return;
24175 }
24176
24177 if path == "/api/worker/workflow-tasks/workflow-timeout-task/complete" {
24178 if let (Some(status), Some(body)) = (
24179 behavior.workflow_completion_status,
24180 behavior.workflow_completion_body,
24181 ) {
24182 write_mock_response(stream, status, body);
24183 return;
24184 }
24185 }
24186
24187 if behavior.waiting_query_worker {
24188 if behavior.complete_named_signal
24189 && path == "/api/worker/workflow-tasks/poll"
24190 && request_number == 1
24191 {
24192 let body = json!({
24193 "task": {
24194 "task_id": "snapshot-open",
24195 "workflow_id": "snapshot-1",
24196 "run_id": "snapshot-run-1",
24197 "workflow_type": "snapshot",
24198 "payload_codec": DEFAULT_CODEC,
24199 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
24200 .expect("Avro workflow arguments"),
24201 "history_events": [],
24202 "workflow_task_attempt": 1,
24203 "lease_owner": "rust-snapshot-worker"
24204 }
24205 })
24206 .to_string();
24207 write_mock_response(stream, "200 OK", &body);
24208 return;
24209 }
24210
24211 let signal_request = request_number - usize::from(behavior.complete_named_signal);
24212 let signal_request_limit = 2 + usize::from(behavior.complete_named_signal);
24213 if path == "/api/worker/workflow-tasks/poll"
24214 && signal_request >= 1
24215 && signal_request <= signal_request_limit
24216 {
24217 let finish = behavior.complete_named_signal && signal_request == 3;
24218 let amounts = if signal_request == 1 {
24219 vec![3]
24220 } else {
24221 vec![3, 5]
24222 };
24223 let task_id = if signal_request == 1 {
24224 "snapshot-wait-3"
24225 } else if finish {
24226 "snapshot-finish"
24227 } else {
24228 "snapshot-wait-5"
24229 };
24230 let mut history_events = std::iter::once(json!({
24231 "event_type": "SignalWaitOpened",
24232 "payload": {"sequence": 1, "signal_name": "finish"}
24233 }))
24234 .chain(amounts.iter().enumerate().map(|(index, amount)| {
24235 json!({
24236 "event_type": "SignalReceived",
24237 "payload": {
24238 "signal_id": format!("increment-{amount}"),
24239 "signal_name": "increment",
24240 "workflow_sequence": index + 2,
24241 "payload_codec": DEFAULT_CODEC,
24242 "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
24243 .expect("Avro signal envelope")
24244 }
24245 })
24246 }))
24247 .collect::<Vec<_>>();
24248 let (resume_id, resume_name, resume_arguments) = if finish {
24249 history_events.push(json!({
24250 "event_type": "SignalReceived",
24251 "payload": {
24252 "signal_id": "finish",
24253 "signal_name": "finish",
24254 "workflow_sequence": 4,
24255 "payload_codec": DEFAULT_CODEC,
24256 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
24257 .expect("Avro finish signal envelope")
24258 }
24259 }));
24260 (
24261 "finish".to_string(),
24262 "finish".to_string(),
24263 encode_value_envelope(&json!([]), DEFAULT_CODEC)
24264 .expect("Avro finish resume signal"),
24265 )
24266 } else {
24267 let amount = amounts.last().expect("amount");
24268 (
24269 format!("increment-{amount}"),
24270 "increment".to_string(),
24271 encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
24272 .expect("Avro increment resume signal"),
24273 )
24274 };
24275 let body = json!({
24276 "task": {
24277 "task_id": task_id,
24278 "workflow_id": "snapshot-1",
24279 "run_id": "snapshot-run-1",
24280 "workflow_type": "snapshot",
24281 "payload_codec": DEFAULT_CODEC,
24282 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
24283 .expect("Avro workflow arguments"),
24284 "history_events": history_events,
24285 "workflow_task_attempt": 1,
24286 "workflow_signal_id": resume_id,
24287 "signal_name": resume_name,
24288 "signal_arguments": resume_arguments,
24289 "lease_owner": "rust-snapshot-worker"
24290 }
24291 })
24292 .to_string();
24293 write_mock_response(stream, "200 OK", &body);
24294 return;
24295 }
24296
24297 if path == "/api/worker/query-tasks/poll" && request_number == 1 {
24298 let history_events = [3, 5]
24299 .into_iter()
24300 .enumerate()
24301 .map(|(index, amount)| {
24302 json!({
24303 "event_type": "SignalReceived",
24304 "payload": {
24305 "signal_id": format!("increment-{amount}"),
24306 "signal_name": "increment",
24307 "workflow_sequence": index + 2,
24308 "payload_codec": DEFAULT_CODEC,
24309 "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
24310 .expect("Avro query signal envelope")
24311 }
24312 })
24313 })
24314 .collect::<Vec<_>>();
24315 let body = json!({
24316 "task": {
24317 "query_task_id": "snapshot-current",
24318 "query_task_attempt": 1,
24319 "lease_owner": "rust-snapshot-worker",
24320 "workflow_id": "snapshot-1",
24321 "run_id": "snapshot-run-1",
24322 "workflow_type": "snapshot",
24323 "query_name": "current",
24324 "payload_codec": DEFAULT_CODEC,
24325 "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
24326 .expect("Avro workflow arguments"),
24327 "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
24328 .expect("Avro query arguments"),
24329 "history_events": history_events,
24330 "run_status": "waiting"
24331 }
24332 })
24333 .to_string();
24334 write_mock_response(stream, "200 OK", &body);
24335 return;
24336 }
24337
24338 if path == "/api/worker/workflow-tasks/snapshot-wait-3/fail"
24339 || path == "/api/worker/workflow-tasks/snapshot-wait-5/fail"
24340 {
24341 write_mock_response(
24342 stream,
24343 "200 OK",
24344 r#"{"outcome":"waiting_for_history","recorded":true}"#,
24345 );
24346 return;
24347 }
24348
24349 if path == "/api/worker/workflow-tasks/snapshot-open/complete" {
24350 write_mock_response(stream, "200 OK", r#"{"outcome":"waiting","recorded":true}"#);
24351 return;
24352 }
24353
24354 if path == "/api/worker/workflow-tasks/snapshot-finish/complete" {
24355 write_mock_response(
24356 stream,
24357 "200 OK",
24358 r#"{"outcome":"completed","run_status":"completed","recorded":true}"#,
24359 );
24360 return;
24361 }
24362
24363 if path == "/api/worker/query-tasks/snapshot-current/complete" {
24364 write_mock_response(stream, "200 OK", r#"{"outcome":"completed"}"#);
24365 return;
24366 }
24367 }
24368
24369 if matches!(
24370 path,
24371 "/api/workflows/typed-1/query/inspect" | "/api/workflows/typed-1/update/replace"
24372 ) {
24373 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
24374 .expect("typed mock result");
24375 let body = json!({
24376 "result": typed_fidelity_probe().into_json().expect("result projection"),
24377 "result_envelope": result,
24378 })
24379 .to_string();
24380 write_mock_response(stream, "200 OK", &body);
24381 return;
24382 }
24383
24384 if path == "/api/workflows/typed-1" {
24385 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
24386 .expect("typed mock result");
24387 let body = json!({
24388 "workflow_id": "typed-1",
24389 "run_id": "run-typed-1",
24390 "workflow_type": "typed.echo",
24391 "status": "completed",
24392 "output": typed_fidelity_probe().into_json().expect("output projection"),
24393 "output_envelope": result,
24394 })
24395 .to_string();
24396 write_mock_response(stream, "200 OK", &body);
24397 return;
24398 }
24399
24400 let (status, body) = match path {
24401 "/api/health" => ("200 OK", r#"{"status":"ok"}"#),
24402 "/api/workflows" => (
24403 "201 Created",
24404 r#"{"workflow_id":"wf-start-options","run_id":"run-start-options","workflow_type":"rust.timeout"}"#,
24405 ),
24406 "/api/worker/register" if behavior.decline_registration => (
24407 "200 OK",
24408 r#"{"worker_id":"declined-worker","registered":false}"#,
24409 ),
24410 "/api/worker/register" if behavior.waiting_query_worker => (
24411 "200 OK",
24412 r#"{"worker_id":"rust-snapshot-worker","registered":true,"heartbeat_interval_seconds":1}"#,
24413 ),
24414 "/api/worker/register" => (
24415 "200 OK",
24416 r#"{"worker_id":"mock-worker","registered":true,"heartbeat_interval_seconds":3600}"#,
24417 ),
24418 "/api/worker/heartbeat" => ("200 OK", "{}"),
24419 "/api/worker/activity-tasks/poll"
24420 if behavior.cancelled_activity && request_number == 1 =>
24421 {
24422 (
24423 "200 OK",
24424 r#"{"task":{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","activity_type":"cancel-aware","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"rust-cancel-worker"}}"#,
24425 )
24426 }
24427 "/api/worker/activity-tasks/poll" | "/api/worker/workflow-tasks/poll" => {
24428 ("200 OK", r#"{"task":null}"#)
24429 }
24430 "/api/worker/query-tasks/poll"
24431 if behavior.reject_query_completion && request_number == 1 =>
24432 {
24433 (
24434 "200 OK",
24435 r#"{"task":{"query_task_id":"query-late","query_task_attempt":1,"lease_owner":"late-worker","workflow_id":"counter-late","run_id":"run-late","workflow_type":"counter","query_name":"current","payload_codec":"avro","workflow_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"query_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"run_status":"running"}}"#,
24436 )
24437 }
24438 "/api/worker/query-tasks/poll" => ("200 OK", r#"{"task":null}"#),
24439 "/api/worker/query-tasks/query-capture/complete"
24440 | "/api/worker/query-tasks/query-capture/fail" => ("200 OK", "{}"),
24441 "/api/worker/activity-tasks/activity-cancel/heartbeat" => (
24442 "200 OK",
24443 r#"{"activity_attempt_id":"attempt-cancel","cancel_requested":true,"can_continue":false,"reason":"run_cancelled","run_closed_reason":"cancelled","heartbeat_recorded":false}"#,
24444 ),
24445 "/api/worker/activity-tasks/activity-cancel/complete" => (
24446 "409 Conflict",
24447 r#"{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","reason":"run_cancelled","cancel_requested":true,"can_continue":false,"run_closed_reason":"cancelled"}"#,
24448 ),
24449 "/api/worker/activity-tasks/activity-typed/complete"
24450 | "/api/worker/activity-tasks/activity-typed/fail"
24451 | "/api/workflows/typed-1/signal/changed" => ("200 OK", "{}"),
24452 "/api/workflows/counter-1/query/current" => (
24453 "200 OK",
24454 r#"{"workflow_id":"counter-1","query_name":"current","result":{"count":8},"result_envelope":{"codec":"avro","blob":"wwHioz3/VYAiNw4CCmNvdW50BBAA"}}"#,
24455 ),
24456 "/api/workflows/counter-1/query/missing" => (
24457 "404 Not Found",
24458 r#"{"workflow_id":"counter-1","query_name":"missing","reason":"rejected_unknown_query","message":"unknown query"}"#,
24459 ),
24460 "/api/workflows/wf-lifecycle/cancel" => (
24461 "200 OK",
24462 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","reason":"cleanup requested","command_status":"accepted"}"#,
24463 ),
24464 "/api/workflows/wf-lifecycle/terminate" => (
24465 "200 OK",
24466 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","reason":"forced stop","command_status":"accepted"}"#,
24467 ),
24468 "/api/workflows/wf-lifecycle/runs/run-current/cancel" => (
24469 "200 OK",
24470 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","command_status":"accepted"}"#,
24471 ),
24472 "/api/workflows/wf-lifecycle/runs/run-current/terminate" => (
24473 "200 OK",
24474 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","command_status":"accepted"}"#,
24475 ),
24476 "/api/workflows/wf-lifecycle/runs/run-stale/cancel"
24477 | "/api/workflows/wf-lifecycle/runs/run-stale/terminate" => (
24478 "409 Conflict",
24479 r#"{"workflow_id":"wf-lifecycle","run_id":"run-stale","reason":"historical_run_command_rejected","target_scope":"run","message":"Commands cannot target historical runs."}"#,
24480 ),
24481 "/api/workflows/wf-failed" | "/api/workflows/wf-failed/runs/run-failed" => (
24482 "200 OK",
24483 r#"{"workflow_id":"wf-failed","run_id":"run-failed","status":"failed","closed_reason":"failed","error":"payment failed","failure":{"message":"payment failed","failure_category":"application","exception_type":"PaymentError","exception_class":"billing::PaymentError","non_retryable":true,"exception":{"type":"PaymentError","class":"billing::PaymentError","message":"payment failed"},"failures":[{"id":"failure-17","failure_category":"application"}]}}"#,
24484 ),
24485 "/api/workflows/wf-cancelled" => (
24486 "200 OK",
24487 r#"{"workflow_id":"wf-cancelled","run_id":"run-cancelled","status":"cancelled","closed_reason":"cancelled","reason":"cleanup requested"}"#,
24488 ),
24489 "/api/workflows/wf-terminated" => (
24490 "200 OK",
24491 r#"{"workflow_id":"wf-terminated","run_id":"run-terminated","status":"terminated","closed_reason":"terminated","reason":"forced stop"}"#,
24492 ),
24493 "/api/workflows/wf-timed-out" => (
24494 "200 OK",
24495 r#"{"workflow_id":"wf-timed-out","run_id":"run-timed-out","status":"failed","closed_reason":"timed_out","reason":"run_timeout"}"#,
24496 ),
24497 "/api/workflows/wf-waiting" | "/api/workflows/wf-waiting/runs/run-waiting" => (
24498 "200 OK",
24499 r#"{"workflow_id":"wf-waiting","run_id":"run-waiting","status":"waiting"}"#,
24500 ),
24501 "/api/workflows/wf-selected" => (
24502 "200 OK",
24503 r#"{"workflow_id":"wf-selected","run_id":"run-current","status":"completed","output":"current run output"}"#,
24504 ),
24505 "/api/workflows/wf-selected/runs/run-selected" => (
24506 "200 OK",
24507 r#"{"workflow_id":"wf-selected","run_id":"run-selected","status":"cancelled","closed_reason":"cancelled","reason":"selected run cancelled"}"#,
24508 ),
24509 _ => ("404 Not Found", r#"{"message":"not found"}"#),
24510 };
24511 write_mock_response(stream, status, body);
24512 }
24513
24514 fn mock_request_is_complete(request: &[u8]) -> bool {
24515 let Some(header_end) = request
24516 .windows(4)
24517 .position(|window| window == b"\r\n\r\n")
24518 .map(|position| position + 4)
24519 else {
24520 return false;
24521 };
24522 let headers = String::from_utf8_lossy(&request[..header_end]);
24523 let content_length = headers.lines().find_map(|line| {
24524 let (name, value) = line.split_once(':')?;
24525 name.eq_ignore_ascii_case("content-length")
24526 .then(|| value.trim().parse::<usize>().ok())
24527 .flatten()
24528 });
24529
24530 request.len() >= header_end + content_length.unwrap_or(0)
24531 }
24532
24533 fn write_mock_response(stream: &mut TcpStream, status: &str, body: &str) {
24534 let response = format!(
24535 "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
24536 body.len()
24537 );
24538
24539 let _ = stream.write_all(response.as_bytes());
24540 let _ = stream.flush();
24541 }
24542}