1#![doc = include_str!("../README.md")]
2
3use std::{
4 any::{type_name, Any, TypeId},
5 collections::{BTreeMap, HashMap},
6 future::Future,
7 io::{self, Read},
8 pin::Pin,
9 sync::{
10 atomic::{AtomicBool, Ordering},
11 Arc, Mutex, OnceLock,
12 },
13 task::{Context as TaskContext, Poll},
14 time::{Duration, Instant, SystemTime, UNIX_EPOCH},
15};
16
17use apache_avro::{from_avro_datum, to_avro_datum, types::Value as AvroDatum, Schema};
18use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
19use chrono::DateTime;
20use futures_util::{future::OptionFuture, task::noop_waker_ref};
21use serde::{
22 de::DeserializeOwned,
23 ser::{SerializeMap, SerializeSeq},
24 Deserialize, Deserializer, Serialize, Serializer,
25};
26pub use serde_json::{json, Value};
27use sha2::{Digest, Sha256};
28use thiserror::Error;
29pub use uuid::Uuid;
30
31pub const WORKER_PROTOCOL_VERSION: &str = "1.19";
32pub const PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION: &str = "1.18";
34pub const CONTROL_PLANE_VERSION: &str = "2";
35pub const DEFAULT_CODEC: &str = "avro";
36pub const SDK_VERSION: &str = concat!("durable-workflow-rust/", env!("CARGO_PKG_VERSION"));
37pub const CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY: &str =
39 "condition_wait_occurrence_identity";
40pub const MEMO_UPSERTS_CAPABILITY: &str = "memo_upserts";
42pub const QUERY_TASKS_CAPABILITY: &str = "query_tasks";
44pub const TYPED_SEARCH_ATTRIBUTES_CAPABILITY: &str = "typed_search_attributes";
46pub const WORKFLOW_UPDATES_CAPABILITY: &str = "workflow_updates";
48pub const MESSAGE_STREAMS_CAPABILITY: &str = "message_streams";
50pub const DURABLE_SELECTION_CAPABILITY: &str = "durable_selection";
52pub const MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.15";
53pub const MESSAGE_STREAM_SIGNAL: &str = "__durable_workflow_message_stream";
54pub const MESSAGE_STREAM_SCHEMA: &str = "durable-workflow.v2.message-stream.message";
55pub const MESSAGE_STREAM_CURSOR_SCHEMA: &str = "durable-workflow.v2.message-stream.cursor";
56pub const MESSAGE_STREAM_MAX_BATCH: usize = 100;
57pub const QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
59pub const SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
61pub const MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.14";
63pub const TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.16";
65pub const CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.9";
67pub const CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.17";
69pub const DURABLE_SELECTION_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.19";
71
72pub fn worker_protocol_supports_message_streams(version: &str) -> bool {
73 let Some((major, minor)) = version.split_once('.') else {
74 return false;
75 };
76 major == "1" && minor.parse::<u64>().is_ok_and(|minor| minor >= 15)
77}
78
79fn validate_user_signal_name(signal_name: &str) -> Result<()> {
80 if signal_name == MESSAGE_STREAM_SIGNAL {
81 return Err(Error::Codec(format!(
82 "signal name {MESSAGE_STREAM_SIGNAL:?} is reserved by the workflow runtime"
83 )));
84 }
85 Ok(())
86}
87
88const MAX_LONG_POLL_TIMEOUT_SECONDS: u64 = 60;
89const WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE: &str =
90 "Workflow task waiting for scheduled history.";
91const WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE: &str = "WorkflowTaskWaitingForHistory";
92const MISSING_TASK_PAYLOAD_CODEC: &str = "\0missing-task-payload-codec";
93const NULL_TASK_PAYLOAD_CODEC: &str = "\0null-task-payload-codec";
94const NON_STRING_TASK_PAYLOAD_CODEC: &str = "\0non-string-task-payload-codec";
95const MAX_MEMO_ENTRIES: usize = 100;
96const MAX_MEMO_VALUE_SIZE_BYTES: usize = 10_240;
97const MAX_MEMO_TOTAL_SIZE_BYTES: usize = 65_536;
98
99const QUERY_TASK_FINAL_REJECTION_REASONS: &[&str] = &[
100 "lease_expired",
101 "query_task_not_found",
102 "query_task_not_leased",
103 "query_task_timed_out",
104];
105
106pub fn portable_worker_affinity_capability_manifest() -> Value {
108 json!({
109 "local_activities": {
110 "supported": false,
111 "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
112 "reason": "rust_worker_does_not_execute_record_local_activity",
113 },
114 "worker_sessions": {
115 "supported": false,
116 "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
117 "reason": "rust_worker_has_no_typed_session_lifecycle",
118 },
119 "sticky_execution": {
120 "supported": false,
121 "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
122 "reason": "rust_worker_uses_complete_durable_history_replay",
123 },
124 })
125}
126
127pub const AVRO_VALUE_SCHEMA_JSON: &str =
129 include_str!("../schema/durable_workflow.protocol.Value.v1.avsc");
130pub const AVRO_VALUE_SCHEMA_FINGERPRINT_HEX: &str = "e2a33dff55802237";
131pub const AVRO_VALUE_SCHEMA_FINGERPRINT: [u8; 8] = [0xe2, 0xa3, 0x3d, 0xff, 0x55, 0x80, 0x22, 0x37];
132const AVRO_SINGLE_OBJECT_MAGIC: [u8; 2] = [0xc3, 0x01];
133
134static AVRO_VALUE_SCHEMA: OnceLock<std::result::Result<Schema, String>> = OnceLock::new();
135static AVRO_VALUE_ORDERED_MAP_ENCODING_SCHEMA: OnceLock<std::result::Result<Schema, String>> =
136 OnceLock::new();
137
138#[derive(Clone, Copy)]
139enum RequestProtocol {
140 ControlPlane,
141 Worker(&'static str),
142}
143
144pub type Result<T> = std::result::Result<T, Error>;
145
146#[derive(Debug, Error)]
147pub enum Error {
148 #[error("transport error: {0}")]
149 Transport(#[from] reqwest::Error),
150 #[error(
151 "invalid Durable Workflow base URL: omit the SDK-owned /api suffix and pass the Server or Cloud runtime base URL; the SDK appends /api automatically"
152 )]
153 InvalidBaseUrl,
154 #[error("json error: {0}")]
155 Json(#[from] serde_json::Error),
156 #[error("http {status}: {body}")]
157 Http {
158 status: reqwest::StatusCode,
159 body: String,
160 },
161 #[error("codec error: {0}")]
162 Codec(String),
163 #[error(transparent)]
164 QueryFailed(QueryFailure),
165 #[error(transparent)]
166 Protocol(ProtocolFailure),
167 #[error(transparent)]
168 NonDeterministicReplay(ReplayFailure),
169 #[error(transparent)]
170 ChildWorkflowFailed(ChildWorkflowFailure),
171 #[error(transparent)]
172 ActivityFailed(ActivityFailure),
173 #[error(transparent)]
174 ParallelFailed(ParallelFailure),
175 #[error(transparent)]
176 SagaCompensationFailed(SagaCompensationFailure),
177 #[error(transparent)]
178 InvalidParallelGroup(ParallelGroupError),
179 #[error(transparent)]
180 DurableOperationCancelled(DurableOperationCancelled),
181 #[error(transparent)]
182 WorkflowCancellationRequested(WorkflowCancellationRequested),
183 #[error(transparent)]
184 WorkflowCommandRejected(WorkflowCommandRejection),
185 #[error(transparent)]
186 WorkflowFailed(WorkflowTerminalOutcome),
187 #[error(transparent)]
188 WorkflowCancelled(WorkflowTerminalOutcome),
189 #[error(transparent)]
190 WorkflowTerminated(WorkflowTerminalOutcome),
191 #[error(transparent)]
192 WorkflowTimedOut(WorkflowTerminalOutcome),
193 #[error(transparent)]
194 ActivityTaskRejected(ActivityTaskRejection),
195 #[error("workflow handler {0:?} is not registered")]
196 WorkflowNotRegistered(String),
197 #[error("activity handler {0:?} is not registered")]
198 ActivityNotRegistered(String),
199 #[error(
200 "{handler_kind} handler {handler_name:?} {value_kind} type {rust_type} is incompatible with the fixed Avro Value codec: {message}"
201 )]
202 HandlerType {
203 handler_kind: HandlerKind,
204 handler_name: String,
205 value_kind: HandlerValueKind,
206 rust_type: &'static str,
207 message: String,
208 },
209 #[error("workflow future yielded without emitting a durable command")]
210 WorkflowYieldedWithoutCommand,
211 #[error(
212 "workflow_stream_command_identity_missing: workflow stream authoring requires a non-empty server-provided workflow_command_id"
213 )]
214 MissingWorkflowCommandIdentity,
215 #[error("workflow state lock is poisoned")]
216 WorkflowStatePoisoned,
217 #[error("timer duration is too large for the worker protocol")]
218 TimerDurationOverflow,
219 #[error(transparent)]
220 InvalidConditionWaitOptions(#[from] ConditionWaitOptionsError),
221 #[error(transparent)]
222 InvalidSearchAttributeUpdate(#[from] SearchAttributeUpdateError),
223 #[error("operation timed out")]
224 Timeout,
225 #[error(
226 "missing {role}-plane credentials: configure ClientBuilder::{role}_token or ClientBuilder::token; a {opposite_role}-plane token cannot authorize this request"
227 )]
228 MissingRoleCredentials {
229 role: &'static str,
230 opposite_role: &'static str,
231 },
232 #[error("worker loop error: {0}")]
233 WorkerLoop(String),
234 #[error(
235 "workflow command contract for {workflow_type:?} declares update validators, but this Rust SDK cannot execute synchronous pre-accept update validation"
236 )]
237 UnsupportedUpdateValidators { workflow_type: String },
238 #[error("{primary}; worker deregistration also failed: {deregistration}")]
239 WorkerShutdown {
240 primary: Box<Error>,
241 deregistration: Box<Error>,
242 },
243 #[error("invalid child workflow options: {0}")]
244 InvalidChildWorkflowOptions(String),
245 #[error("invalid workflow memo update: {0}")]
246 InvalidMemoUpdate(String),
247 #[error(
248 "workflow_memo_updates_unavailable: the connected runtime did not advertise workflow memo update support"
249 )]
250 WorkflowMemoUpdatesUnavailable,
251 #[error(transparent)]
252 InvalidActivityOptions(ActivityOptionsError),
253 #[error(transparent)]
254 InvalidContinueAsNewOptions(#[from] ContinueAsNewOptionsError),
255 #[doc(hidden)]
256 #[error("workflow requested continue as new")]
257 ContinueAsNew(ContinueAsNewRequest),
258}
259
260#[derive(Clone, Debug, Error, PartialEq, Eq)]
262pub enum ConditionWaitOptionsError {
263 #[error("condition_key must be non-empty")]
264 EmptyKey,
265 #[error("condition_definition_fingerprint must be non-empty")]
266 EmptyPredicateIdentity,
267 #[error("condition timeout is too large for the worker protocol")]
268 TimeoutOverflow,
269}
270
271#[derive(Clone, Debug, PartialEq, Eq)]
278pub struct ConditionWaitOptions {
279 condition_key: String,
280 predicate_identity: String,
281 timeout: Option<Duration>,
282}
283
284impl ConditionWaitOptions {
285 pub fn new(condition_key: impl Into<String>, predicate_identity: impl Into<String>) -> Self {
286 Self {
287 condition_key: condition_key.into(),
288 predicate_identity: predicate_identity.into(),
289 timeout: None,
290 }
291 }
292
293 pub fn timeout(mut self, timeout: Duration) -> Self {
294 self.timeout = Some(timeout);
295 self
296 }
297
298 fn validate(
299 &self,
300 ) -> std::result::Result<ValidatedConditionWaitOptions, ConditionWaitOptionsError> {
301 let condition_key = self.condition_key.trim();
302 if condition_key.is_empty() {
303 return Err(ConditionWaitOptionsError::EmptyKey);
304 }
305 let predicate_identity = self.predicate_identity.trim();
306 if predicate_identity.is_empty() {
307 return Err(ConditionWaitOptionsError::EmptyPredicateIdentity);
308 }
309 let timeout_seconds = self
310 .timeout
311 .map(|timeout| {
312 timeout
313 .as_secs()
314 .checked_add(u64::from(timeout.subsec_nanos() > 0))
315 .ok_or(ConditionWaitOptionsError::TimeoutOverflow)
316 })
317 .transpose()?;
318
319 Ok(ValidatedConditionWaitOptions {
320 condition_key: condition_key.to_string(),
321 predicate_identity: predicate_identity.to_string(),
322 timeout_seconds,
323 })
324 }
325}
326
327#[derive(Clone, Debug, PartialEq, Eq)]
328struct ValidatedConditionWaitOptions {
329 condition_key: String,
330 predicate_identity: String,
331 timeout_seconds: Option<u64>,
332}
333
334const CONDITION_WAIT_OCCURRENCE_PREFIX: &str = "rust:condition-wait:";
335
336#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
338#[serde(rename_all = "snake_case")]
339pub enum ConditionWaitResult {
340 Satisfied,
341 TimedOut,
342}
343
344impl ConditionWaitResult {
345 pub fn is_satisfied(self) -> bool {
346 self == Self::Satisfied
347 }
348
349 pub fn is_timed_out(self) -> bool {
350 self == Self::TimedOut
351 }
352}
353
354#[doc(hidden)]
356pub fn __condition_definition_fingerprint(source: &str) -> String {
357 let mut digest = Sha256::new();
358 digest.update(b"durable-workflow-rust.wait-condition.v1\0");
359 digest.update(source.as_bytes());
360 format!("sha256:{:x}", digest.finalize())
361}
362
363#[macro_export]
369macro_rules! wait_condition {
370 ($ctx:expr, $key:expr, timeout: $timeout:expr, $predicate:expr $(,)?) => {{
371 $ctx.wait_condition(
372 $crate::ConditionWaitOptions::new(
373 $key,
374 $crate::__condition_definition_fingerprint(concat!(
375 module_path!(),
376 "\0",
377 stringify!($predicate)
378 )),
379 )
380 .timeout($timeout),
381 $predicate,
382 )
383 }};
384 ($ctx:expr, $key:expr, $predicate:expr $(,)?) => {{
385 $ctx.wait_condition(
386 $crate::ConditionWaitOptions::new(
387 $key,
388 $crate::__condition_definition_fingerprint(concat!(
389 module_path!(),
390 "\0",
391 stringify!($predicate)
392 )),
393 ),
394 $predicate,
395 )
396 }};
397}
398
399const MAX_SEARCH_ATTRIBUTES_PER_UPDATE: usize = 100;
400const MAX_SEARCH_ATTRIBUTE_KEY_LENGTH: usize = 64;
401const MAX_SEARCH_ATTRIBUTE_STRING_LENGTH: usize = 2_048;
402const MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH: usize = 255;
403const MAX_SEARCH_ATTRIBUTE_UPDATE_BYTES: usize = 65_536;
404
405#[derive(Clone, Debug, Error, PartialEq, Eq)]
407pub enum SearchAttributeUpdateError {
408 #[error("search-attribute update requires at least one attribute")]
409 Empty,
410 #[error("search attribute key {0:?} must be 1-64 URL-safe ASCII characters")]
411 InvalidKey(String),
412 #[error("search-attribute update exceeds the limit of 100 attributes")]
413 TooManyAttributes,
414 #[error("search attribute {key:?} {kind} value exceeds {limit} bytes")]
415 ValueTooLong {
416 key: String,
417 kind: &'static str,
418 limit: usize,
419 },
420 #[error(
421 "search attribute {0:?} must not contain an empty string value; use delete() to remove it"
422 )]
423 EmptyString(String),
424 #[error("search attribute {0:?} has a non-finite float value")]
425 NonFiniteFloat(String),
426 #[error("search attribute {0:?} must use an RFC 3339 datetime with an explicit timezone")]
427 InvalidDateTime(String),
428 #[error("search-attribute update exceeds the 65536-byte protocol limit")]
429 PayloadTooLarge,
430}
431
432#[derive(Clone, Debug, PartialEq)]
434pub enum SearchAttributeValue {
435 String(String),
436 Keyword(String),
437 KeywordList(Vec<String>),
438 Int(i64),
439 Float(f64),
440 Bool(bool),
441 DateTime(String),
442 Delete,
443}
444
445impl SearchAttributeValue {
446 fn type_name(&self) -> Option<&'static str> {
447 match self {
448 Self::String(_) => Some("string"),
449 Self::Keyword(_) => Some("keyword"),
450 Self::KeywordList(_) => Some("keyword_list"),
451 Self::Int(_) => Some("int"),
452 Self::Float(_) => Some("float"),
453 Self::Bool(_) => Some("bool"),
454 Self::DateTime(_) => Some("datetime"),
455 Self::Delete => None,
456 }
457 }
458
459 fn normalized(self, key: &str) -> std::result::Result<Self, SearchAttributeUpdateError> {
460 let normalize_string = |value: String, kind: &'static str, limit: usize| {
461 let value = value.trim().to_string();
462 if value.is_empty() {
463 return Err(SearchAttributeUpdateError::EmptyString(key.to_string()));
464 }
465 if value.len() > limit {
466 return Err(SearchAttributeUpdateError::ValueTooLong {
467 key: key.to_string(),
468 kind,
469 limit,
470 });
471 }
472 Ok(value)
473 };
474
475 match self {
476 Self::String(value) => Ok(Self::String(normalize_string(
477 value,
478 "string",
479 MAX_SEARCH_ATTRIBUTE_STRING_LENGTH,
480 )?)),
481 Self::Keyword(value) => Ok(Self::Keyword(normalize_string(
482 value,
483 "keyword",
484 MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH,
485 )?)),
486 Self::KeywordList(values) => {
487 let values = values
488 .into_iter()
489 .map(|value| {
490 let value = value.trim().to_string();
491 if value.len() > MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH {
492 return Err(SearchAttributeUpdateError::ValueTooLong {
493 key: key.to_string(),
494 kind: "keyword-list entry",
495 limit: MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH,
496 });
497 }
498 Ok(value)
499 })
500 .collect::<std::result::Result<Vec<_>, _>>()?;
501 Ok(Self::KeywordList(values))
502 }
503 Self::Float(value) if !value.is_finite() => {
504 Err(SearchAttributeUpdateError::NonFiniteFloat(key.to_string()))
505 }
506 Self::DateTime(value) => {
507 let value =
508 normalize_string(value, "datetime", MAX_SEARCH_ATTRIBUTE_STRING_LENGTH)?;
509 if DateTime::parse_from_rfc3339(&value).is_err() {
510 return Err(SearchAttributeUpdateError::InvalidDateTime(key.to_string()));
511 }
512 Ok(Self::DateTime(value))
513 }
514 value => Ok(value),
515 }
516 }
517
518 fn into_json(self) -> Value {
519 match self {
520 Self::String(value) | Self::Keyword(value) | Self::DateTime(value) => {
521 Value::String(value)
522 }
523 Self::KeywordList(values) => {
524 Value::Array(values.into_iter().map(Value::String).collect())
525 }
526 Self::Int(value) => json!(value),
527 Self::Float(value) => json!(value),
528 Self::Bool(value) => json!(value),
529 Self::Delete => Value::Null,
530 }
531 }
532}
533
534#[derive(Clone, Debug, Default, PartialEq)]
536pub struct SearchAttributeUpdate {
537 attributes: BTreeMap<String, SearchAttributeValue>,
538}
539
540impl SearchAttributeUpdate {
541 pub fn new() -> Self {
542 Self::default()
543 }
544
545 pub fn set(
546 mut self,
547 key: impl Into<String>,
548 value: SearchAttributeValue,
549 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
550 let key = key.into();
551 validate_search_attribute_key(&key)?;
552 if !self.attributes.contains_key(&key)
553 && self.attributes.len() >= MAX_SEARCH_ATTRIBUTES_PER_UPDATE
554 {
555 return Err(SearchAttributeUpdateError::TooManyAttributes);
556 }
557 self.attributes.insert(key.clone(), value.normalized(&key)?);
558 self.validate_size()?;
559 Ok(self)
560 }
561
562 pub fn string(
563 self,
564 key: impl Into<String>,
565 value: impl Into<String>,
566 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
567 self.set(key, SearchAttributeValue::String(value.into()))
568 }
569
570 pub fn keyword(
571 self,
572 key: impl Into<String>,
573 value: impl Into<String>,
574 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
575 self.set(key, SearchAttributeValue::Keyword(value.into()))
576 }
577
578 pub fn keyword_list<I, V>(
579 self,
580 key: impl Into<String>,
581 values: I,
582 ) -> std::result::Result<Self, SearchAttributeUpdateError>
583 where
584 I: IntoIterator<Item = V>,
585 V: Into<String>,
586 {
587 self.set(
588 key,
589 SearchAttributeValue::KeywordList(values.into_iter().map(Into::into).collect()),
590 )
591 }
592
593 pub fn int(
594 self,
595 key: impl Into<String>,
596 value: i64,
597 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
598 self.set(key, SearchAttributeValue::Int(value))
599 }
600
601 pub fn float(
602 self,
603 key: impl Into<String>,
604 value: f64,
605 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
606 self.set(key, SearchAttributeValue::Float(value))
607 }
608
609 pub fn bool(
610 self,
611 key: impl Into<String>,
612 value: bool,
613 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
614 self.set(key, SearchAttributeValue::Bool(value))
615 }
616
617 pub fn datetime(
618 self,
619 key: impl Into<String>,
620 value: impl Into<String>,
621 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
622 self.set(key, SearchAttributeValue::DateTime(value.into()))
623 }
624
625 pub fn delete(
626 self,
627 key: impl Into<String>,
628 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
629 self.set(key, SearchAttributeValue::Delete)
630 }
631
632 fn validate_size(&self) -> std::result::Result<(), SearchAttributeUpdateError> {
633 let (attributes, _) = self.clone().into_wire_parts();
634 if serde_json::to_vec(&attributes)
635 .map(|payload| payload.len() > MAX_SEARCH_ATTRIBUTE_UPDATE_BYTES)
636 .unwrap_or(true)
637 {
638 return Err(SearchAttributeUpdateError::PayloadTooLarge);
639 }
640 Ok(())
641 }
642
643 fn into_wire_parts(self) -> (Value, BTreeMap<String, String>) {
644 let mut attributes = serde_json::Map::new();
645 let mut attribute_types = BTreeMap::new();
646 for (key, value) in self.attributes {
647 if let Some(type_name) = value.type_name() {
648 attribute_types.insert(key.clone(), type_name.to_string());
649 }
650 attributes.insert(key, value.into_json());
651 }
652 (Value::Object(attributes), attribute_types)
653 }
654
655 fn validate(&self) -> std::result::Result<(), SearchAttributeUpdateError> {
656 if self.attributes.is_empty() {
657 return Err(SearchAttributeUpdateError::Empty);
658 }
659 self.validate_size()
660 }
661}
662
663fn validate_search_attribute_key(key: &str) -> std::result::Result<(), SearchAttributeUpdateError> {
664 let valid = !key.is_empty()
665 && key.len() <= MAX_SEARCH_ATTRIBUTE_KEY_LENGTH
666 && key
667 .bytes()
668 .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-' | b':'));
669 if valid {
670 Ok(())
671 } else {
672 Err(SearchAttributeUpdateError::InvalidKey(key.to_string()))
673 }
674}
675
676#[derive(Clone, Copy, Debug, PartialEq, Eq)]
678pub enum HandlerKind {
679 Workflow,
680 Activity,
681}
682
683impl std::fmt::Display for HandlerKind {
684 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
685 formatter.write_str(match self {
686 Self::Workflow => "workflow",
687 Self::Activity => "activity",
688 })
689 }
690}
691
692#[derive(Clone, Copy, Debug, PartialEq, Eq)]
694pub enum HandlerValueKind {
695 Input,
696 Result,
697}
698
699impl std::fmt::Display for HandlerValueKind {
700 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
701 formatter.write_str(match self {
702 Self::Input => "input",
703 Self::Result => "result",
704 })
705 }
706}
707
708#[derive(Clone, Copy, Debug, PartialEq, Eq)]
710pub enum WorkflowCommandKind {
711 Cancel,
712 Terminate,
713}
714
715impl WorkflowCommandKind {
716 fn as_str(self) -> &'static str {
717 match self {
718 Self::Cancel => "cancel",
719 Self::Terminate => "terminate",
720 }
721 }
722}
723
724#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
726pub struct WorkflowCommandOptions {
727 #[serde(skip_serializing_if = "Option::is_none")]
728 pub reason: Option<String>,
729 #[serde(skip_serializing_if = "Option::is_none")]
730 pub request_id: Option<String>,
731}
732
733#[derive(Clone, Debug, PartialEq, Eq)]
740pub struct WorkflowStartOptions {
741 pub execution_timeout_seconds: u64,
742 pub run_timeout_seconds: u64,
743}
744
745impl Default for WorkflowStartOptions {
746 fn default() -> Self {
747 Self {
748 execution_timeout_seconds: 3600,
749 run_timeout_seconds: 600,
750 }
751 }
752}
753
754impl WorkflowStartOptions {
755 pub fn new() -> Self {
756 Self::default()
757 }
758
759 pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
760 self.execution_timeout_seconds = seconds;
761 self
762 }
763
764 pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
765 self.run_timeout_seconds = seconds;
766 self
767 }
768
769 fn validate(&self) -> Result<()> {
770 if self.execution_timeout_seconds == 0 {
771 return Err(Error::Codec(
772 "execution_timeout_seconds must be at least 1".to_string(),
773 ));
774 }
775 if self.run_timeout_seconds == 0 {
776 return Err(Error::Codec(
777 "run_timeout_seconds must be at least 1".to_string(),
778 ));
779 }
780 if self.run_timeout_seconds > self.execution_timeout_seconds {
781 return Err(Error::Codec(
782 "run_timeout_seconds cannot exceed execution_timeout_seconds".to_string(),
783 ));
784 }
785
786 Ok(())
787 }
788}
789
790#[derive(Clone, Debug, Default, PartialEq, Eq)]
795pub struct ContinueAsNewOptions {
796 pub workflow_type: Option<String>,
797 pub task_queue: Option<String>,
798}
799
800impl ContinueAsNewOptions {
801 pub fn new() -> Self {
802 Self::default()
803 }
804
805 pub fn workflow_type(mut self, workflow_type: impl Into<String>) -> Self {
806 self.workflow_type = Some(workflow_type.into());
807 self
808 }
809
810 pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
811 self.task_queue = Some(task_queue.into());
812 self
813 }
814
815 fn validate(&self) -> std::result::Result<(), ContinueAsNewOptionsError> {
816 for (field, value) in [
817 ("workflow_type", self.workflow_type.as_deref()),
818 ("task_queue", self.task_queue.as_deref()),
819 ] {
820 if value.is_some_and(|value| value.trim().is_empty()) {
821 return Err(ContinueAsNewOptionsError {
822 field,
823 message: format!("{field} must not be empty"),
824 });
825 }
826 }
827 Ok(())
828 }
829}
830
831#[derive(Clone, Debug, Error, PartialEq, Eq)]
833#[error("invalid continue-as-new option {field}: {message}")]
834pub struct ContinueAsNewOptionsError {
835 pub field: &'static str,
836 pub message: String,
837}
838
839#[derive(Clone, Debug, Default, PartialEq, Eq)]
841pub struct WorkflowHistoryBudget {
842 pub event_count: u64,
843 pub size_bytes: Option<u64>,
844 pub continue_as_new_recommended: bool,
845 pub pressure: Option<String>,
846}
847
848#[doc(hidden)]
849#[derive(Clone, Debug)]
850pub struct ContinueAsNewRequest {
851 arguments: AvroValue,
852 options: ContinueAsNewOptions,
853}
854
855impl WorkflowCommandOptions {
856 pub fn new() -> Self {
857 Self::default()
858 }
859
860 pub fn reason(mut self, reason: impl Into<String>) -> Self {
861 self.reason = Some(reason.into());
862 self
863 }
864
865 pub fn request_id(mut self, request_id: impl Into<String>) -> Self {
866 self.request_id = Some(request_id.into());
867 self
868 }
869}
870
871#[derive(Clone, Debug, PartialEq)]
873pub struct WorkflowCommandResult {
874 pub command: WorkflowCommandKind,
875 pub workflow_id: String,
876 pub run_id: Option<String>,
877 pub outcome: Option<String>,
878 pub reason: Option<String>,
879 pub command_status: Option<String>,
880 pub raw: Value,
881}
882
883#[derive(Clone, Debug, Error)]
885#[error("workflow {command:?} rejected ({reason}, HTTP {status}): {message}")]
886pub struct WorkflowCommandRejection {
887 pub command: WorkflowCommandKind,
888 pub status: u16,
889 pub reason: String,
890 pub message: String,
891 pub workflow_id: String,
892 pub run_id: Option<String>,
893 pub target_scope: Option<String>,
894 pub body: Value,
895}
896
897#[derive(Clone, Copy, Debug, PartialEq, Eq)]
899pub enum WorkflowTerminalKind {
900 Failed,
901 Cancelled,
902 Terminated,
903 TimedOut,
904}
905
906#[derive(Clone, Debug, Error)]
912#[error("workflow {workflow_id} run {run_id:?} ended as {kind:?} ({reason})")]
913pub struct WorkflowTerminalOutcome {
914 pub kind: WorkflowTerminalKind,
915 pub workflow_id: String,
916 pub run_id: Option<String>,
917 pub reason: String,
918 pub failure_category: Option<String>,
919 pub failure_id: Option<String>,
920 pub exception_type: Option<String>,
921 pub exception_class: Option<String>,
922 pub non_retryable: Option<bool>,
923 pub message: Option<String>,
924 pub exception: Option<Value>,
925 pub raw: Value,
926}
927
928#[derive(Clone, Debug, Error)]
930#[error("activity task {operation} rejected ({reason}, HTTP {status})")]
931pub struct ActivityTaskRejection {
932 pub operation: String,
933 pub status: u16,
934 pub reason: String,
935 pub task_id: String,
936 pub activity_attempt_id: String,
937 pub cancel_requested: bool,
938 pub can_continue: Option<bool>,
939 pub run_closed_reason: Option<String>,
940 pub body: Value,
941}
942
943#[derive(Clone, Copy, Debug, PartialEq, Eq)]
945pub enum ActivityOptionsErrorKind {
946 EmptyTaskQueue,
947 EmptyRetryPolicy,
948 InvalidMaxAttempts,
949 BackoffWithoutRetryBudget,
950 TooManyBackoffIntervals,
951 InvalidBackoffCoefficient,
952 BackoffGenerationTooLarge,
953 BackoffOverflow,
954 EmptyNonRetryableErrorType,
955 TimeoutNotPositive,
956 TimeoutOverflow,
957 TimeoutOrder,
958}
959
960#[derive(Clone, Debug, Error, PartialEq, Eq)]
962#[error("invalid activity options ({kind:?}, {field:?}): {message}")]
963pub struct ActivityOptionsError {
964 pub kind: ActivityOptionsErrorKind,
965 pub field: Option<&'static str>,
966 pub message: String,
967}
968
969impl ActivityOptionsError {
970 fn new(
971 kind: ActivityOptionsErrorKind,
972 field: Option<&'static str>,
973 message: impl Into<String>,
974 ) -> Self {
975 Self {
976 kind,
977 field,
978 message: message.into(),
979 }
980 }
981}
982
983#[derive(Clone, Copy, Debug, PartialEq, Eq)]
985pub enum ActivityFailureKind {
986 Failed,
987 Cancelled,
988 TimedOut,
989}
990
991#[derive(Clone, Debug, Error)]
996#[error("activity failed ({reason}): {message}")]
997pub struct ActivityFailure {
998 pub kind: ActivityFailureKind,
999 pub reason: String,
1000 pub message: String,
1001 pub activity_execution_id: Option<String>,
1002 pub activity_attempt_id: Option<String>,
1003 pub activity_type: Option<String>,
1004 pub activity_class: Option<String>,
1005 pub attempt_number: Option<u64>,
1006 pub failure_id: Option<String>,
1007 pub failure_category: Option<String>,
1008 pub timeout_kind: Option<String>,
1009 pub non_retryable: bool,
1010 pub exception_type: Option<String>,
1011 pub exception_class: Option<String>,
1012 pub code: Option<Value>,
1013 pub exception: Option<Value>,
1014}
1015
1016#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1018pub enum ChildWorkflowFailureKind {
1019 Failed,
1020 Cancelled,
1021 Terminated,
1022}
1023
1024#[derive(Clone, Debug, Error)]
1030#[error("child workflow failed ({reason}): {message}")]
1031pub struct ChildWorkflowFailure {
1032 pub kind: ChildWorkflowFailureKind,
1033 pub reason: String,
1034 pub message: String,
1035 pub parent_workflow_id: Option<String>,
1036 pub parent_workflow_run_id: Option<String>,
1037 pub child_workflow_id: Option<String>,
1038 pub child_workflow_run_id: Option<String>,
1039 pub child_workflow_type: Option<String>,
1040 pub failure_id: Option<String>,
1041 pub failure_category: Option<String>,
1042 pub exception_type: Option<String>,
1043 pub exception_class: Option<String>,
1044 pub non_retryable: bool,
1045 pub code: Option<Value>,
1046 pub exception: Option<Value>,
1047}
1048
1049#[derive(Clone, Debug, PartialEq, Eq)]
1051pub struct WorkflowIdentity {
1052 pub workflow_id: Option<String>,
1053 pub run_id: Option<String>,
1054}
1055
1056#[derive(Clone, Debug, PartialEq)]
1058pub struct ChildWorkflowResult {
1059 pub parent: WorkflowIdentity,
1060 pub child: WorkflowIdentity,
1061 pub child_workflow_type: Option<String>,
1062 pub result: Value,
1063}
1064
1065#[derive(Clone, Debug, PartialEq)]
1067pub struct ChildWorkflowAvroResult {
1068 pub parent: WorkflowIdentity,
1069 pub child: WorkflowIdentity,
1070 pub child_workflow_type: Option<String>,
1071 pub result: AvroValue,
1072}
1073
1074#[derive(Clone, Debug, Deserialize, Hash, PartialEq, Eq, Serialize)]
1076#[serde(untagged)]
1077pub enum SelectionKey {
1078 Index(usize),
1079 Name(String),
1080}
1081
1082impl From<usize> for SelectionKey {
1083 fn from(value: usize) -> Self {
1084 Self::Index(value)
1085 }
1086}
1087
1088impl From<String> for SelectionKey {
1089 fn from(value: String) -> Self {
1090 Self::Name(value)
1091 }
1092}
1093
1094impl From<&str> for SelectionKey {
1095 fn from(value: &str) -> Self {
1096 Self::Name(value.to_string())
1097 }
1098}
1099
1100#[derive(Clone, Debug, Error, PartialEq, Eq)]
1102#[error("selected {operation_kind} operation {operation_identity} was explicitly cancelled")]
1103pub struct DurableOperationCancelled {
1104 pub selection_group_id: String,
1105 pub member_key: SelectionKey,
1106 pub member_index: usize,
1107 pub operation_kind: String,
1108 pub operation_identity: String,
1109}
1110
1111#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
1117pub struct ParallelGroupMetadata {
1118 pub parallel_group_id: String,
1119 pub parallel_group_kind: String,
1120 pub parallel_group_base_sequence: u64,
1121 pub parallel_group_size: usize,
1122 pub parallel_group_index: usize,
1123 #[serde(default, skip_serializing_if = "Option::is_none")]
1124 pub parallel_group_mode: Option<String>,
1125 #[serde(default, skip_serializing_if = "Option::is_none")]
1126 pub selection_member_key: Option<SelectionKey>,
1127 #[serde(default, skip_serializing_if = "Option::is_none")]
1128 pub selection_member_index: Option<usize>,
1129 #[serde(default, skip_serializing_if = "Option::is_none")]
1130 pub selection_member_base_sequence: Option<u64>,
1131 #[serde(default, skip_serializing_if = "Option::is_none")]
1132 pub selection_member_size: Option<usize>,
1133 #[serde(default, skip_serializing_if = "Option::is_none")]
1134 pub selection_member_kind: Option<String>,
1135}
1136
1137#[derive(Clone, Debug, PartialEq)]
1139pub enum ParallelResult {
1140 Activity(Value),
1141 ChildWorkflow(ChildWorkflowResult),
1142 Timer,
1143 Signal(Vec<Value>),
1144 Condition(ConditionWaitResult),
1145 Group(Vec<ParallelResult>),
1146}
1147
1148#[derive(Clone, Debug, PartialEq)]
1150pub enum ParallelAvroResult {
1151 Activity(AvroValue),
1152 ChildWorkflow(ChildWorkflowAvroResult),
1153 Timer,
1154 Signal(Vec<AvroValue>),
1155 Condition(ConditionWaitResult),
1156 Group(Vec<ParallelAvroResult>),
1157}
1158
1159impl ParallelAvroResult {
1160 fn into_json_result(self) -> Result<ParallelResult> {
1161 match self {
1162 Self::Activity(value) => Ok(ParallelResult::Activity(value.into_json()?)),
1163 Self::ChildWorkflow(result) => Ok(ParallelResult::ChildWorkflow(ChildWorkflowResult {
1164 parent: result.parent,
1165 child: result.child,
1166 child_workflow_type: result.child_workflow_type,
1167 result: result.result.into_json()?,
1168 })),
1169 Self::Timer => Ok(ParallelResult::Timer),
1170 Self::Signal(values) => Ok(ParallelResult::Signal(
1171 values
1172 .into_iter()
1173 .map(AvroValue::into_json)
1174 .collect::<Result<Vec<_>>>()?,
1175 )),
1176 Self::Condition(result) => Ok(ParallelResult::Condition(result)),
1177 Self::Group(results) => Ok(ParallelResult::Group(
1178 results
1179 .into_iter()
1180 .map(Self::into_json_result)
1181 .collect::<Result<Vec<_>>>()?,
1182 )),
1183 }
1184 }
1185}
1186
1187#[derive(Clone, Debug, PartialEq)]
1189pub struct ParallelCompletion {
1190 pub member_path: Vec<usize>,
1191 pub result: ParallelResult,
1192}
1193
1194#[derive(Debug, Error)]
1202#[error("parallel group {group_id} member {member_path:?} failed: {cause}")]
1203pub struct ParallelFailure {
1204 pub group_id: String,
1205 pub member_path: Vec<usize>,
1206 pub group_path: Vec<ParallelGroupMetadata>,
1207 pub completed: Vec<ParallelCompletion>,
1208 #[source]
1209 pub cause: Box<Error>,
1210}
1211
1212#[derive(Clone, Debug, Error, PartialEq, Eq)]
1214#[error("invalid deterministic parallel group ({reason}): {message}")]
1215pub struct ParallelGroupError {
1216 pub reason: &'static str,
1217 pub member_path: Vec<usize>,
1218 pub message: String,
1219}
1220
1221#[derive(Clone, Debug, Error, PartialEq, Eq)]
1223#[error("workflow cancellation was requested")]
1224pub struct WorkflowCancellationRequested;
1225
1226#[derive(Debug, Error)]
1228#[error(
1229 "saga forward execution failed; compensation activity {compensation_activity_type} (registration {compensation_registration_order}) also failed: {compensation_failure}"
1230)]
1231pub struct SagaCompensationFailure {
1232 pub initiating_failure: Box<Error>,
1233 pub compensation_failure: Box<Error>,
1234 pub compensation_activity_type: String,
1235 pub compensation_registration_order: usize,
1236}
1237
1238#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
1240pub enum ParentClosePolicy {
1241 #[default]
1242 Abandon,
1243 RequestCancel,
1244 Terminate,
1245}
1246
1247impl ParentClosePolicy {
1248 fn as_str(self) -> &'static str {
1249 match self {
1250 Self::Abandon => "abandon",
1251 Self::RequestCancel => "request_cancel",
1252 Self::Terminate => "terminate",
1253 }
1254 }
1255}
1256
1257#[derive(Clone, Debug, Default, PartialEq, Eq)]
1259pub struct ChildWorkflowRetryPolicy {
1260 pub max_attempts: Option<u32>,
1261 pub backoff_seconds: Vec<u64>,
1262 pub non_retryable_error_types: Vec<String>,
1263}
1264
1265#[derive(Clone, Debug, PartialEq, Eq)]
1269pub struct ChildWorkflowOptions {
1270 pub task_queue: String,
1271 pub parent_close_policy: ParentClosePolicy,
1272 pub retry_policy: Option<ChildWorkflowRetryPolicy>,
1273 pub execution_timeout_seconds: Option<u64>,
1274 pub run_timeout_seconds: Option<u64>,
1275}
1276
1277impl ChildWorkflowOptions {
1278 pub fn new(task_queue: impl Into<String>) -> Self {
1279 Self {
1280 task_queue: task_queue.into(),
1281 parent_close_policy: ParentClosePolicy::Abandon,
1282 retry_policy: None,
1283 execution_timeout_seconds: None,
1284 run_timeout_seconds: None,
1285 }
1286 }
1287
1288 pub fn parent_close_policy(mut self, policy: ParentClosePolicy) -> Self {
1289 self.parent_close_policy = policy;
1290 self
1291 }
1292
1293 pub fn retry_policy(mut self, policy: ChildWorkflowRetryPolicy) -> Self {
1294 self.retry_policy = Some(policy);
1295 self
1296 }
1297
1298 pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
1299 self.execution_timeout_seconds = Some(seconds);
1300 self
1301 }
1302
1303 pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
1304 self.run_timeout_seconds = Some(seconds);
1305 self
1306 }
1307}
1308
1309#[derive(Clone, Debug, PartialEq, Eq)]
1311pub enum ActivityBackoff {
1312 Explicit(Vec<Duration>),
1315 Exponential {
1317 initial_interval: Duration,
1318 coefficient: u32,
1319 maximum_interval: Option<Duration>,
1320 },
1321}
1322
1323#[derive(Clone, Debug, Default, PartialEq, Eq)]
1325pub struct ActivityRetryPolicy {
1326 pub max_attempts: Option<u32>,
1327 pub backoff: Option<ActivityBackoff>,
1328 pub non_retryable_error_types: Vec<String>,
1329}
1330
1331impl ActivityRetryPolicy {
1332 pub fn new(max_attempts: u32) -> Self {
1334 Self {
1335 max_attempts: Some(max_attempts),
1336 ..Self::default()
1337 }
1338 }
1339
1340 pub fn backoff_intervals(mut self, intervals: impl IntoIterator<Item = Duration>) -> Self {
1341 self.backoff = Some(ActivityBackoff::Explicit(intervals.into_iter().collect()));
1342 self
1343 }
1344
1345 pub fn exponential_backoff(
1346 mut self,
1347 initial_interval: Duration,
1348 coefficient: u32,
1349 maximum_interval: Option<Duration>,
1350 ) -> Self {
1351 self.backoff = Some(ActivityBackoff::Exponential {
1352 initial_interval,
1353 coefficient,
1354 maximum_interval,
1355 });
1356 self
1357 }
1358
1359 pub fn non_retryable_error_type(mut self, error_type: impl Into<String>) -> Self {
1360 self.non_retryable_error_types.push(error_type.into());
1361 self
1362 }
1363
1364 pub fn non_retryable_error_types(
1365 mut self,
1366 error_types: impl IntoIterator<Item = impl Into<String>>,
1367 ) -> Self {
1368 self.non_retryable_error_types
1369 .extend(error_types.into_iter().map(Into::into));
1370 self
1371 }
1372}
1373
1374#[derive(Clone, Debug, Default, PartialEq, Eq)]
1379pub struct ActivityOptions {
1380 pub task_queue: Option<String>,
1381 pub retry_policy: Option<ActivityRetryPolicy>,
1382 pub start_to_close_timeout: Option<Duration>,
1383 pub schedule_to_start_timeout: Option<Duration>,
1384 pub schedule_to_close_timeout: Option<Duration>,
1385 pub heartbeat_timeout: Option<Duration>,
1386}
1387
1388impl ActivityOptions {
1389 pub fn new() -> Self {
1390 Self::default()
1391 }
1392
1393 pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
1394 self.task_queue = Some(task_queue.into());
1395 self
1396 }
1397
1398 pub fn retry_policy(mut self, policy: ActivityRetryPolicy) -> Self {
1399 self.retry_policy = Some(policy);
1400 self
1401 }
1402
1403 pub fn start_to_close_timeout(mut self, timeout: Duration) -> Self {
1404 self.start_to_close_timeout = Some(timeout);
1405 self
1406 }
1407
1408 pub fn schedule_to_start_timeout(mut self, timeout: Duration) -> Self {
1409 self.schedule_to_start_timeout = Some(timeout);
1410 self
1411 }
1412
1413 pub fn schedule_to_close_timeout(mut self, timeout: Duration) -> Self {
1414 self.schedule_to_close_timeout = Some(timeout);
1415 self
1416 }
1417
1418 pub fn heartbeat_timeout(mut self, timeout: Duration) -> Self {
1419 self.heartbeat_timeout = Some(timeout);
1420 self
1421 }
1422
1423 fn validate(&self) -> std::result::Result<ValidatedActivityOptions, ActivityOptionsError> {
1424 if self
1425 .task_queue
1426 .as_deref()
1427 .is_some_and(|queue| queue.trim().is_empty())
1428 {
1429 return Err(ActivityOptionsError::new(
1430 ActivityOptionsErrorKind::EmptyTaskQueue,
1431 Some("task_queue"),
1432 "task_queue must not be empty",
1433 ));
1434 }
1435
1436 for (field, value) in [
1437 ("start_to_close_timeout", self.start_to_close_timeout),
1438 ("schedule_to_start_timeout", self.schedule_to_start_timeout),
1439 ("schedule_to_close_timeout", self.schedule_to_close_timeout),
1440 ("heartbeat_timeout", self.heartbeat_timeout),
1441 ] {
1442 if value.is_some_and(|value| value.is_zero()) {
1443 return Err(ActivityOptionsError::new(
1444 ActivityOptionsErrorKind::TimeoutNotPositive,
1445 Some(field),
1446 format!("{field} must be positive"),
1447 ));
1448 }
1449 }
1450
1451 validate_timeout_order(
1452 "heartbeat_timeout",
1453 self.heartbeat_timeout,
1454 "start_to_close_timeout",
1455 self.start_to_close_timeout,
1456 )?;
1457 validate_timeout_order(
1458 "start_to_close_timeout",
1459 self.start_to_close_timeout,
1460 "schedule_to_close_timeout",
1461 self.schedule_to_close_timeout,
1462 )?;
1463 validate_timeout_order(
1464 "schedule_to_start_timeout",
1465 self.schedule_to_start_timeout,
1466 "schedule_to_close_timeout",
1467 self.schedule_to_close_timeout,
1468 )?;
1469
1470 Ok(ValidatedActivityOptions {
1471 task_queue: self.task_queue.clone(),
1472 retry_policy: self
1473 .retry_policy
1474 .as_ref()
1475 .map(validate_activity_retry_policy)
1476 .transpose()?,
1477 start_to_close_timeout: timeout_seconds(
1478 "start_to_close_timeout",
1479 self.start_to_close_timeout,
1480 )?,
1481 schedule_to_start_timeout: timeout_seconds(
1482 "schedule_to_start_timeout",
1483 self.schedule_to_start_timeout,
1484 )?,
1485 schedule_to_close_timeout: timeout_seconds(
1486 "schedule_to_close_timeout",
1487 self.schedule_to_close_timeout,
1488 )?,
1489 heartbeat_timeout: timeout_seconds("heartbeat_timeout", self.heartbeat_timeout)?,
1490 })
1491 }
1492}
1493
1494pub enum ParallelOperation {
1500 Activity {
1501 activity_type: String,
1502 options: ActivityOptions,
1503 arguments: Result<AvroValue>,
1504 },
1505 ChildWorkflow {
1506 workflow_type: String,
1507 options: ChildWorkflowOptions,
1508 arguments: Result<AvroValue>,
1509 },
1510 Timer(Duration),
1511 Signal(String),
1512 Condition {
1513 options: ConditionWaitOptions,
1514 predicate: Box<dyn Fn() -> Result<bool> + Send + 'static>,
1515 },
1516 Group(Vec<ParallelOperation>),
1517}
1518
1519impl ParallelOperation {
1520 pub fn activity<T: Serialize>(activity_type: impl Into<String>, args: T) -> Self {
1521 Self::activity_with_options(activity_type, ActivityOptions::new(), args)
1522 }
1523
1524 pub fn activity_with_options<T: Serialize>(
1525 activity_type: impl Into<String>,
1526 options: ActivityOptions,
1527 args: T,
1528 ) -> Self {
1529 Self::Activity {
1530 activity_type: activity_type.into(),
1531 options,
1532 arguments: AvroValue::from_serialize(&args),
1533 }
1534 }
1535
1536 pub fn child_workflow<T: Serialize>(
1537 workflow_type: impl Into<String>,
1538 options: ChildWorkflowOptions,
1539 args: T,
1540 ) -> Self {
1541 Self::ChildWorkflow {
1542 workflow_type: workflow_type.into(),
1543 options,
1544 arguments: AvroValue::from_serialize(&args),
1545 }
1546 }
1547
1548 pub fn timer(duration: Duration) -> Self {
1549 Self::Timer(duration)
1550 }
1551
1552 pub fn signal(signal_name: impl Into<String>) -> Self {
1553 Self::Signal(signal_name.into())
1554 }
1555
1556 pub fn condition<F>(options: ConditionWaitOptions, predicate: F) -> Self
1557 where
1558 F: Fn() -> Result<bool> + Send + 'static,
1559 {
1560 Self::Condition {
1561 options,
1562 predicate: Box::new(predicate),
1563 }
1564 }
1565
1566 pub fn group(operations: Vec<ParallelOperation>) -> Self {
1567 Self::Group(operations)
1568 }
1569}
1570
1571#[derive(Clone, Debug)]
1572struct ValidatedActivityOptions {
1573 task_queue: Option<String>,
1574 retry_policy: Option<Value>,
1575 start_to_close_timeout: Option<u64>,
1576 schedule_to_start_timeout: Option<u64>,
1577 schedule_to_close_timeout: Option<u64>,
1578 heartbeat_timeout: Option<u64>,
1579}
1580
1581fn validate_timeout_order(
1582 smaller_name: &'static str,
1583 smaller: Option<Duration>,
1584 larger_name: &'static str,
1585 larger: Option<Duration>,
1586) -> std::result::Result<(), ActivityOptionsError> {
1587 if matches!((smaller, larger), (Some(smaller), Some(larger)) if smaller > larger) {
1588 return Err(ActivityOptionsError::new(
1589 ActivityOptionsErrorKind::TimeoutOrder,
1590 Some(smaller_name),
1591 format!("{smaller_name} must be <= {larger_name}"),
1592 ));
1593 }
1594 Ok(())
1595}
1596
1597fn timeout_seconds(
1598 field: &'static str,
1599 value: Option<Duration>,
1600) -> std::result::Result<Option<u64>, ActivityOptionsError> {
1601 value
1602 .map(|value| {
1603 activity_protocol_seconds(value).ok_or_else(|| {
1604 ActivityOptionsError::new(
1605 ActivityOptionsErrorKind::TimeoutOverflow,
1606 Some(field),
1607 format!("{field} is too large for the worker protocol"),
1608 )
1609 })
1610 })
1611 .transpose()
1612}
1613
1614fn duration_seconds_ceil(value: Duration) -> Option<u64> {
1615 value
1616 .as_secs()
1617 .checked_add(u64::from(value.subsec_nanos() > 0))
1618}
1619
1620fn activity_protocol_seconds(value: Duration) -> Option<u64> {
1621 duration_seconds_ceil(value).filter(|seconds| *seconds <= i64::MAX as u64)
1622}
1623
1624fn validate_activity_retry_policy(
1625 policy: &ActivityRetryPolicy,
1626) -> std::result::Result<Value, ActivityOptionsError> {
1627 if policy.max_attempts.is_none()
1628 && policy.backoff.is_none()
1629 && policy.non_retryable_error_types.is_empty()
1630 {
1631 return Err(ActivityOptionsError::new(
1632 ActivityOptionsErrorKind::EmptyRetryPolicy,
1633 Some("retry_policy"),
1634 "retry_policy must configure at least one field",
1635 ));
1636 }
1637 if policy.max_attempts == Some(0) {
1638 return Err(ActivityOptionsError::new(
1639 ActivityOptionsErrorKind::InvalidMaxAttempts,
1640 Some("retry_policy.max_attempts"),
1641 "max_attempts must be >= 1",
1642 ));
1643 }
1644 if policy
1645 .non_retryable_error_types
1646 .iter()
1647 .any(|error_type| error_type.trim().is_empty())
1648 {
1649 return Err(ActivityOptionsError::new(
1650 ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
1651 Some("retry_policy.non_retryable_error_types"),
1652 "non_retryable_error_types must not contain empty values",
1653 ));
1654 }
1655
1656 let backoff_seconds = match &policy.backoff {
1657 None => None,
1658 Some(backoff) => {
1659 let max_attempts = policy.max_attempts.ok_or_else(|| {
1660 ActivityOptionsError::new(
1661 ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
1662 Some("retry_policy.backoff"),
1663 "backoff requires max_attempts",
1664 )
1665 })?;
1666 let retry_count = max_attempts.saturating_sub(1) as usize;
1667 let intervals = match backoff {
1668 ActivityBackoff::Explicit(intervals) => {
1669 if intervals.len() > retry_count {
1670 return Err(ActivityOptionsError::new(
1671 ActivityOptionsErrorKind::TooManyBackoffIntervals,
1672 Some("retry_policy.backoff"),
1673 "backoff interval count must not exceed max_attempts - 1",
1674 ));
1675 }
1676 intervals.clone()
1677 }
1678 ActivityBackoff::Exponential {
1679 initial_interval,
1680 coefficient,
1681 maximum_interval,
1682 } => {
1683 if *coefficient < 1 {
1684 return Err(ActivityOptionsError::new(
1685 ActivityOptionsErrorKind::InvalidBackoffCoefficient,
1686 Some("retry_policy.backoff.coefficient"),
1687 "backoff coefficient must be >= 1",
1688 ));
1689 }
1690 if retry_count > 10_000 {
1691 return Err(ActivityOptionsError::new(
1692 ActivityOptionsErrorKind::BackoffGenerationTooLarge,
1693 Some("retry_policy.max_attempts"),
1694 "generated backoff supports at most 10000 retry intervals",
1695 ));
1696 }
1697 let mut current = *initial_interval;
1698 let mut intervals = Vec::with_capacity(retry_count);
1699 for _ in 0..retry_count {
1700 let interval = maximum_interval
1701 .map(|maximum| current.min(maximum))
1702 .unwrap_or(current);
1703 intervals.push(interval);
1704 if maximum_interval.is_some_and(|maximum| interval == maximum) {
1705 break;
1706 }
1707 current = current.checked_mul(*coefficient).ok_or_else(|| {
1708 ActivityOptionsError::new(
1709 ActivityOptionsErrorKind::BackoffOverflow,
1710 Some("retry_policy.backoff"),
1711 "generated backoff interval overflowed",
1712 )
1713 })?;
1714 }
1715 intervals
1716 }
1717 };
1718 Some(
1719 intervals
1720 .into_iter()
1721 .map(|interval| {
1722 activity_protocol_seconds(interval).ok_or_else(|| {
1723 ActivityOptionsError::new(
1724 ActivityOptionsErrorKind::BackoffOverflow,
1725 Some("retry_policy.backoff"),
1726 "backoff interval is too large for the worker protocol",
1727 )
1728 })
1729 })
1730 .collect::<std::result::Result<Vec<_>, _>>()?,
1731 )
1732 }
1733 };
1734
1735 let mut encoded = serde_json::Map::new();
1736 if let Some(max_attempts) = policy.max_attempts {
1737 encoded.insert("max_attempts".to_string(), json!(max_attempts));
1738 }
1739 if let Some(backoff_seconds) = backoff_seconds {
1740 encoded.insert("backoff_seconds".to_string(), json!(backoff_seconds));
1741 }
1742 if !policy.non_retryable_error_types.is_empty() {
1743 let mut canonical_error_types = Vec::new();
1744 for error_type in policy
1745 .non_retryable_error_types
1746 .iter()
1747 .map(|error_type| error_type.trim())
1748 {
1749 if !canonical_error_types.contains(&error_type) {
1750 canonical_error_types.push(error_type);
1751 }
1752 }
1753 encoded.insert(
1754 "non_retryable_error_types".to_string(),
1755 json!(canonical_error_types),
1756 );
1757 }
1758 Ok(Value::Object(encoded))
1759}
1760
1761#[derive(Clone, Debug, Error)]
1764#[error("non-deterministic workflow replay ({reason}) at sequence {sequence:?}: {message}")]
1765pub struct ReplayFailure {
1766 pub reason: String,
1767 pub sequence: Option<u64>,
1768 pub expected: Option<String>,
1769 pub actual: Option<String>,
1770 pub message: String,
1771}
1772
1773impl ReplayFailure {
1774 fn new(
1775 reason: impl Into<String>,
1776 sequence: Option<u64>,
1777 expected: Option<String>,
1778 actual: Option<String>,
1779 message: impl Into<String>,
1780 ) -> Self {
1781 Self {
1782 reason: reason.into(),
1783 sequence,
1784 expected,
1785 actual,
1786 message: message.into(),
1787 }
1788 }
1789}
1790
1791#[derive(Clone, Debug, Error)]
1793#[error("query failed ({reason}, HTTP {status}): {message}")]
1794pub struct QueryFailure {
1795 pub status: u16,
1796 pub reason: String,
1797 pub message: String,
1798 pub body: Value,
1799}
1800
1801#[derive(Clone, Debug, Error)]
1803#[error("protocol rejected ({reason}, HTTP {status}): {message}")]
1804pub struct ProtocolFailure {
1805 pub status: u16,
1806 pub reason: String,
1807 pub message: String,
1808 pub supported_version: Option<String>,
1809 pub requested_version: Option<String>,
1810 pub body: Value,
1811}
1812
1813#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
1814pub struct PayloadEnvelope {
1815 pub codec: String,
1816 pub blob: String,
1817}
1818
1819impl PayloadEnvelope {
1820 pub fn avro<T: Serialize>(value: &T) -> Result<Self> {
1821 encode_payload(value, DEFAULT_CODEC)
1822 }
1823
1824 pub fn avro_value(value: &AvroValue) -> Result<Self> {
1827 encode_avro_value(value)
1828 }
1829}
1830
1831#[derive(Clone, Debug)]
1833pub enum AvroValue {
1834 Null,
1835 Boolean(bool),
1836 Long(i64),
1837 Double(f64),
1838 Bytes(Vec<u8>),
1839 String(String),
1840 Array(Vec<AvroValue>),
1841 Map(BTreeMap<String, AvroValue>),
1842}
1843
1844impl PartialEq for AvroValue {
1845 fn eq(&self, other: &Self) -> bool {
1846 match (self, other) {
1847 (Self::Null, Self::Null) => true,
1848 (Self::Boolean(left), Self::Boolean(right)) => left == right,
1849 (Self::Long(left), Self::Long(right)) => left == right,
1850 (Self::Double(left), Self::Double(right)) => left.to_bits() == right.to_bits(),
1851 (Self::Bytes(left), Self::Bytes(right)) => left == right,
1852 (Self::String(left), Self::String(right)) => left == right,
1853 (Self::Array(left), Self::Array(right)) => left == right,
1854 (Self::Map(left), Self::Map(right)) => left == right,
1855 _ => false,
1856 }
1857 }
1858}
1859
1860impl AvroValue {
1861 fn from_serialize<T: Serialize>(value: &T) -> Result<Self> {
1862 Self::from_serde_value(
1863 serde_value::to_value(value).map_err(|error| {
1864 Error::Codec(format!("could not adapt value for Avro: {error}"))
1865 })?,
1866 )
1867 }
1868
1869 fn from_serde_value(value: serde_value::Value) -> Result<Self> {
1870 use serde_value::Value as SerdeValue;
1871
1872 match value {
1873 SerdeValue::Unit => Ok(Self::Null),
1874 SerdeValue::Bool(value) => Ok(Self::Boolean(value)),
1875 SerdeValue::I8(value) => Ok(Self::Long(i64::from(value))),
1876 SerdeValue::I16(value) => Ok(Self::Long(i64::from(value))),
1877 SerdeValue::I32(value) => Ok(Self::Long(i64::from(value))),
1878 SerdeValue::I64(value) => Ok(Self::Long(value)),
1879 SerdeValue::U8(value) => Ok(Self::Long(i64::from(value))),
1880 SerdeValue::U16(value) => Ok(Self::Long(i64::from(value))),
1881 SerdeValue::U32(value) => Ok(Self::Long(i64::from(value))),
1882 SerdeValue::U64(value) => i64::try_from(value).map(Self::Long).map_err(|_| {
1883 Error::Codec(
1884 "integer_overflow: Avro Value long must be within signed 64-bit range"
1885 .to_string(),
1886 )
1887 }),
1888 SerdeValue::F32(value) => Self::finite_double(f64::from(value)),
1889 SerdeValue::F64(value) => Self::finite_double(value),
1890 SerdeValue::Char(value) => Ok(Self::String(value.to_string())),
1891 SerdeValue::String(value) => Ok(Self::String(value)),
1892 SerdeValue::Bytes(value) => Ok(Self::Bytes(value)),
1893 SerdeValue::Option(None) => Ok(Self::Null),
1894 SerdeValue::Option(Some(value)) | SerdeValue::Newtype(value) => {
1895 Self::from_serde_value(*value)
1896 }
1897 SerdeValue::Seq(values) => values
1898 .into_iter()
1899 .map(Self::from_serde_value)
1900 .collect::<Result<Vec<_>>>()
1901 .map(Self::Array),
1902 SerdeValue::Map(values) => values
1903 .into_iter()
1904 .map(|(key, value)| {
1905 let SerdeValue::String(key) = key else {
1906 return Err(Error::Codec(
1907 "invalid_map_key: Avro Value map keys must be strings".to_string(),
1908 ));
1909 };
1910
1911 Ok((key, Self::from_serde_value(value)?))
1912 })
1913 .collect::<Result<BTreeMap<_, _>>>()
1914 .map(Self::Map),
1915 }
1916 }
1917
1918 fn finite_double(value: f64) -> Result<Self> {
1919 if !value.is_finite() {
1920 return Err(Error::Codec(
1921 "non_finite_float: Avro Value doubles must be finite".to_string(),
1922 ));
1923 }
1924
1925 Ok(Self::Double(value))
1926 }
1927
1928 fn into_json(self) -> Result<Value> {
1929 match self {
1930 Self::Null => Ok(Value::Null),
1931 Self::Boolean(value) => Ok(Value::Bool(value)),
1932 Self::Long(value) => Ok(Value::Number(value.into())),
1933 Self::Double(value) => serde_json::Number::from_f64(value)
1934 .map(Value::Number)
1935 .ok_or_else(|| {
1936 Error::Codec(
1937 "non_finite_float: decoded Avro Value double is not finite".to_string(),
1938 )
1939 }),
1940 Self::Bytes(value) => Ok(json!({
1941 "$type": "bytes",
1942 "base64": BASE64.encode(value),
1943 })),
1944 Self::String(value) => Ok(Value::String(value)),
1945 Self::Array(values) => values
1946 .into_iter()
1947 .map(Self::into_json)
1948 .collect::<Result<Vec<_>>>()
1949 .map(Value::Array),
1950 Self::Map(values) => values
1951 .into_iter()
1952 .map(|(key, value)| Ok((key, value.into_json()?)))
1953 .collect::<Result<serde_json::Map<_, _>>>()
1954 .map(Value::Object),
1955 }
1956 }
1957
1958 fn into_serde_value(self) -> serde_value::Value {
1959 use serde_value::Value as SerdeValue;
1960
1961 match self {
1962 Self::Null => SerdeValue::Unit,
1963 Self::Boolean(value) => SerdeValue::Bool(value),
1964 Self::Long(value) => SerdeValue::I64(value),
1965 Self::Double(value) => SerdeValue::F64(value),
1966 Self::Bytes(value) => SerdeValue::Bytes(value),
1967 Self::String(value) => SerdeValue::String(value),
1968 Self::Array(values) => {
1969 SerdeValue::Seq(values.into_iter().map(Self::into_serde_value).collect())
1970 }
1971 Self::Map(values) => SerdeValue::Map(
1972 values
1973 .into_iter()
1974 .map(|(key, value)| (SerdeValue::String(key), value.into_serde_value()))
1975 .collect(),
1976 ),
1977 }
1978 }
1979
1980 pub fn deserialize<T: DeserializeOwned>(self) -> Result<T> {
1981 self.into_serde_value().deserialize_into().map_err(|error| {
1982 Error::Codec(format!(
1983 "avro_value_type_mismatch: could not adapt decoded value: {error}"
1984 ))
1985 })
1986 }
1987}
1988
1989impl Serialize for AvroValue {
1990 fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
1991 where
1992 S: Serializer,
1993 {
1994 match self {
1995 Self::Null => serializer.serialize_unit(),
1996 Self::Boolean(value) => serializer.serialize_bool(*value),
1997 Self::Long(value) => serializer.serialize_i64(*value),
1998 Self::Double(value) => serializer.serialize_f64(*value),
1999 Self::Bytes(value) => serializer.serialize_bytes(value),
2000 Self::String(value) => serializer.serialize_str(value),
2001 Self::Array(values) => {
2002 let mut sequence = serializer.serialize_seq(Some(values.len()))?;
2003 for value in values {
2004 sequence.serialize_element(value)?;
2005 }
2006 sequence.end()
2007 }
2008 Self::Map(values) => {
2009 let mut map = serializer.serialize_map(Some(values.len()))?;
2010 for (key, value) in values {
2011 map.serialize_entry(key, value)?;
2012 }
2013 map.end()
2014 }
2015 }
2016 }
2017}
2018
2019pub fn encode_avro_value(value: &AvroValue) -> Result<PayloadEnvelope> {
2020 let datum = avro_value_to_datum(value)?;
2021 let datum = to_avro_datum(avro_value_ordered_map_encoding_schema()?, datum)
2022 .map_err(|err| Error::Codec(format!("avro_value_encode_failed: {err}")))?;
2023 let mut bytes = Vec::with_capacity(datum.len() + 10);
2024 bytes.extend_from_slice(&AVRO_SINGLE_OBJECT_MAGIC);
2025 bytes.extend_from_slice(&AVRO_VALUE_SCHEMA_FINGERPRINT);
2026 bytes.extend_from_slice(&datum);
2027 Ok(PayloadEnvelope {
2028 codec: DEFAULT_CODEC.to_string(),
2029 blob: BASE64.encode(bytes),
2030 })
2031}
2032
2033pub fn decode_avro_value(envelope: &PayloadEnvelope) -> Result<AvroValue> {
2034 if envelope.codec != DEFAULT_CODEC {
2035 return Err(unsupported_payload_codec(&envelope.codec));
2036 }
2037 decode_avro_value_blob(&envelope.blob)
2038}
2039
2040pub fn encode_payload<T: Serialize>(value: &T, codec: &str) -> Result<PayloadEnvelope> {
2041 let blob = match codec {
2042 DEFAULT_CODEC => encode_avro_value(&AvroValue::from_serialize(value)?)?.blob,
2043 other => return Err(unsupported_payload_codec(other)),
2044 };
2045
2046 Ok(PayloadEnvelope {
2047 codec: codec.to_string(),
2048 blob,
2049 })
2050}
2051
2052pub fn decode_payload<T: DeserializeOwned>(envelope: &PayloadEnvelope) -> Result<T> {
2053 match envelope.codec.as_str() {
2054 DEFAULT_CODEC => decode_avro_value(envelope)?.deserialize(),
2055 other => Err(unsupported_payload_codec(other)),
2056 }
2057}
2058
2059fn handler_type_error<T>(
2060 handler_kind: HandlerKind,
2061 handler_name: &str,
2062 value_kind: HandlerValueKind,
2063 message: impl Into<String>,
2064) -> Error {
2065 Error::HandlerType {
2066 handler_kind,
2067 handler_name: handler_name.to_string(),
2068 value_kind,
2069 rust_type: type_name::<T>(),
2070 message: message.into(),
2071 }
2072}
2073
2074fn decode_handler_input<T: DeserializeOwned>(
2075 arguments: AvroValue,
2076 handler_kind: HandlerKind,
2077 handler_name: &str,
2078) -> Result<T> {
2079 let argument = match arguments {
2080 AvroValue::Array(mut arguments) if arguments.len() == 1 => {
2081 arguments.pop().expect("one typed handler argument")
2082 }
2083 AvroValue::Array(arguments) if arguments.is_empty() => AvroValue::Null,
2084 AvroValue::Array(arguments) => {
2085 return Err(handler_type_error::<T>(
2086 handler_kind,
2087 handler_name,
2088 HandlerValueKind::Input,
2089 format!(
2090 "typed handlers accept one request value, but the task carried {} arguments",
2091 arguments.len()
2092 ),
2093 ));
2094 }
2095 argument => argument,
2096 };
2097
2098 argument.deserialize().map_err(|error| {
2099 handler_type_error::<T>(
2100 handler_kind,
2101 handler_name,
2102 HandlerValueKind::Input,
2103 error.to_string(),
2104 )
2105 })
2106}
2107
2108fn encode_handler_result<T: Serialize>(
2109 result: &T,
2110 handler_kind: HandlerKind,
2111 handler_name: &str,
2112) -> Result<AvroValue> {
2113 AvroValue::from_serialize(result).map_err(|error| {
2114 handler_type_error::<T>(
2115 handler_kind,
2116 handler_name,
2117 HandlerValueKind::Result,
2118 error.to_string(),
2119 )
2120 })
2121}
2122
2123fn decode_handler_result<T: DeserializeOwned>(
2124 result: AvroValue,
2125 handler_kind: HandlerKind,
2126 handler_name: &str,
2127) -> Result<T> {
2128 result.deserialize().map_err(|error| {
2129 handler_type_error::<T>(
2130 handler_kind,
2131 handler_name,
2132 HandlerValueKind::Result,
2133 error.to_string(),
2134 )
2135 })
2136}
2137
2138#[cfg(test)]
2139fn encode_value_envelope(value: &Value, codec: &str) -> Result<Value> {
2140 Ok(serde_json::to_value(encode_payload(value, codec)?)?)
2141}
2142
2143fn decode_wire_value(value: &Value, fallback_codec: &str) -> Result<Value> {
2144 validate_payload_codec(fallback_codec)?;
2145
2146 if value.is_null() {
2147 return Ok(Value::Null);
2148 }
2149
2150 if let Some((codec, blob)) = payload_envelope_parts(value)? {
2151 return decode_blob(blob, codec);
2152 }
2153
2154 if let Some(blob) = value.as_str() {
2155 return decode_blob(blob, fallback_codec);
2156 }
2157
2158 Err(untagged_payload_value())
2159}
2160
2161fn encode_typed_envelope(value: &AvroValue, codec: &str) -> Result<Value> {
2162 let envelope = match codec {
2163 DEFAULT_CODEC => encode_avro_value(value)?,
2164 other => return Err(unsupported_payload_codec(other)),
2165 };
2166 Ok(serde_json::to_value(envelope)?)
2167}
2168
2169fn decode_wire_avro_value(value: &Value, fallback_codec: &str) -> Result<AvroValue> {
2170 validate_payload_codec(fallback_codec)?;
2171
2172 if value.is_null() {
2173 return Ok(AvroValue::Null);
2174 }
2175
2176 if let Some((codec, blob)) = payload_envelope_parts(value)? {
2177 validate_payload_codec(codec)?;
2178 return decode_avro_value_blob(blob);
2179 }
2180
2181 if let Some(blob) = value.as_str() {
2182 return match fallback_codec {
2183 DEFAULT_CODEC => decode_avro_value_blob(blob),
2184 other => Err(unsupported_payload_codec(other)),
2185 };
2186 }
2187
2188 Err(untagged_payload_value())
2189}
2190
2191fn normalize_avro_arguments(value: AvroValue) -> AvroValue {
2192 match value {
2193 AvroValue::Null => AvroValue::Array(Vec::new()),
2194 AvroValue::Array(_) => value,
2195 other => AvroValue::Array(vec![other]),
2196 }
2197}
2198
2199fn decode_blob(blob: &str, codec: &str) -> Result<Value> {
2200 match codec {
2201 DEFAULT_CODEC => decode_avro_value_blob(blob)?.into_json(),
2202 other => Err(unsupported_payload_codec(other)),
2203 }
2204}
2205
2206fn validate_payload_codec(codec: &str) -> Result<()> {
2207 match codec {
2208 DEFAULT_CODEC => Ok(()),
2209 MISSING_TASK_PAYLOAD_CODEC => {
2210 Err(invalid_task_payload_codec("task payload_codec is missing"))
2211 }
2212 NULL_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec("task payload_codec is null")),
2213 NON_STRING_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec(
2214 "task payload_codec must be a string",
2215 )),
2216 other => Err(unsupported_payload_codec(other)),
2217 }
2218}
2219
2220fn invalid_task_payload_codec(reason: &str) -> Error {
2221 Error::Codec(format!(
2222 "unsupported_payload_codec: {reason}; Durable Workflow 2.0 requires an explicit string payload_codec=\"avro\" before worker task execution"
2223 ))
2224}
2225
2226fn payload_envelope_parts(value: &Value) -> Result<Option<(&str, &str)>> {
2227 let Some(object) = value.as_object() else {
2228 return Ok(None);
2229 };
2230 if !object.contains_key("codec") && !object.contains_key("blob") {
2231 return Ok(None);
2232 }
2233
2234 let codec = object
2235 .get("codec")
2236 .and_then(Value::as_str)
2237 .ok_or_else(invalid_payload_envelope)?;
2238 validate_payload_codec(codec)?;
2239 let blob = object
2240 .get("blob")
2241 .and_then(Value::as_str)
2242 .ok_or_else(invalid_payload_envelope)?;
2243 Ok(Some((codec, blob)))
2244}
2245
2246fn invalid_payload_envelope() -> Error {
2247 Error::Codec(
2248 "invalid_payload_envelope: durable payloads must use an object with string codec=\"avro\" and blob fields"
2249 .to_string(),
2250 )
2251}
2252
2253fn validate_workflow_task_commands(commands: &[Value]) -> Result<()> {
2254 for command in commands {
2255 let Some(command) = command.as_object() else {
2256 continue;
2257 };
2258 let Some(command_type) = command.get("type").and_then(Value::as_str) else {
2259 continue;
2260 };
2261 let Some(payload_field) = workflow_command_payload_field(command_type) else {
2262 continue;
2263 };
2264
2265 if let Some(codec) = command.get("payload_codec") {
2266 let codec = codec.as_str().ok_or_else(invalid_payload_envelope)?;
2267 validate_payload_codec(codec)?;
2268 }
2269
2270 let payload = command
2271 .get(payload_field)
2272 .ok_or_else(invalid_payload_envelope)?;
2273 validate_outbound_payload_envelope(payload)?;
2274 }
2275 Ok(())
2276}
2277
2278fn workflow_completion_protocol_version(commands: &[Value]) -> &'static str {
2279 if commands.iter().any(|command| {
2280 command.get("type").and_then(Value::as_str) == Some("open_condition_wait")
2281 && command
2282 .get("condition_wait_occurrence_id")
2283 .and_then(Value::as_str)
2284 .is_some_and(|occurrence_id| !occurrence_id.is_empty())
2285 }) {
2286 CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
2287 } else if commands.iter().any(|command| {
2288 command.get("type").and_then(Value::as_str) == Some("upsert_search_attributes")
2289 && command.get("attribute_types").is_some()
2290 }) {
2291 TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION
2292 } else if commands
2293 .iter()
2294 .any(|command| command.get("type").and_then(Value::as_str) == Some("upsert_memo"))
2295 {
2296 MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION
2297 } else if commands
2298 .iter()
2299 .any(|command| command.get("type").and_then(Value::as_str) == Some("open_condition_wait"))
2300 {
2301 CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION
2302 } else if commands.iter().any(|command| {
2303 command.get("type").and_then(Value::as_str) == Some("upsert_search_attributes")
2304 }) {
2305 SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION
2306 } else {
2307 WORKER_PROTOCOL_VERSION
2308 }
2309}
2310
2311fn workflow_completion_protocol_version_with_message_streams(
2312 commands: &[Value],
2313 has_message_stream_metadata: bool,
2314) -> &'static str {
2315 let command_protocol = workflow_completion_protocol_version(commands);
2316 if has_message_stream_metadata && !worker_protocol_supports_message_streams(command_protocol) {
2317 MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION
2318 } else {
2319 command_protocol
2320 }
2321}
2322
2323fn workflow_command_payload_field(command_type: &str) -> Option<&'static str> {
2324 match command_type {
2325 "complete_workflow" | "complete_update" | "record_side_effect" => Some("result"),
2326 "schedule_activity" | "start_child_workflow" | "continue_as_new" => Some("arguments"),
2327 "start_service_operation" => Some("request_payload"),
2328 "upsert_memo" => Some("entries"),
2329 _ => None,
2330 }
2331}
2332
2333fn validate_outbound_payload_envelope(value: &Value) -> Result<()> {
2334 let Some((codec, blob)) = payload_envelope_parts(value)? else {
2335 return Err(untagged_payload_value());
2336 };
2337 validate_payload_codec(codec)?;
2338 decode_avro_value_blob(blob)?;
2339 Ok(())
2340}
2341
2342fn unsupported_payload_codec(codec: &str) -> Error {
2343 Error::Codec(format!(
2344 "unsupported_payload_codec: workflow payload codec {codec:?} is not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
2345 ))
2346}
2347
2348fn untagged_payload_value() -> Error {
2349 Error::Codec(
2350 "unsupported_payload_codec: untagged durable payload values are not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
2351 .to_string(),
2352 )
2353}
2354
2355fn decode_avro_value_blob(blob: &str) -> Result<AvroValue> {
2356 let bytes = BASE64.decode(blob).map_err(|err| {
2357 Error::Codec(format!(
2358 "invalid_payload_framing: expected strict base64 Avro single-object bytes: {err}"
2359 ))
2360 })?;
2361
2362 if serde_json::from_slice::<Value>(&bytes).is_ok() {
2363 return Err(unsupported_payload_codec("json"));
2364 }
2365
2366 if bytes.len() < 10 || bytes[..2] != AVRO_SINGLE_OBJECT_MAGIC {
2367 return Err(Error::Codec(
2368 "invalid_payload_framing: expected Avro single-object magic c301".to_string(),
2369 ));
2370 }
2371
2372 let fingerprint: [u8; 8] = bytes[2..10]
2373 .try_into()
2374 .map_err(|_| Error::Codec("invalid Avro fingerprint length".to_string()))?;
2375 if fingerprint != AVRO_VALUE_SCHEMA_FINGERPRINT {
2376 return Err(Error::Codec(format!(
2377 "unsupported_payload_schema: unknown CRC-64-AVRO fingerprint {}",
2378 fingerprint
2379 .iter()
2380 .map(|byte| format!("{byte:02x}"))
2381 .collect::<String>()
2382 )));
2383 }
2384
2385 let mut datum_reader = StrictAvroDatumReader::new(&bytes[10..]);
2386 let datum = from_avro_datum(avro_value_schema()?, &mut datum_reader, None);
2390 if datum_reader.truncated {
2391 return Err(Error::Codec(
2392 "invalid_payload_framing: truncated Avro Value datum".to_string(),
2393 ));
2394 }
2395 let datum = datum.map_err(|err| {
2396 Error::Codec(format!(
2397 "invalid_payload_framing: malformed Avro Value datum: {err}"
2398 ))
2399 })?;
2400 if datum_reader.remaining() != 0 {
2401 return Err(Error::Codec(format!(
2402 "invalid_payload_framing: {} trailing bytes after Avro Value datum",
2403 datum_reader.remaining()
2404 )));
2405 }
2406 avro_value_from_datum(datum)
2407}
2408
2409struct StrictAvroDatumReader<'a> {
2410 bytes: &'a [u8],
2411 offset: usize,
2412 truncated: bool,
2413}
2414
2415impl<'a> StrictAvroDatumReader<'a> {
2416 fn new(bytes: &'a [u8]) -> Self {
2417 Self {
2418 bytes,
2419 offset: 0,
2420 truncated: false,
2421 }
2422 }
2423
2424 fn remaining(&self) -> usize {
2425 self.bytes.len() - self.offset
2426 }
2427}
2428
2429impl Read for StrictAvroDatumReader<'_> {
2430 fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
2431 let count = buffer.len().min(self.remaining());
2432 buffer[..count].copy_from_slice(&self.bytes[self.offset..self.offset + count]);
2433 self.offset += count;
2434 if count < buffer.len() {
2435 self.truncated = true;
2436 }
2437
2438 Ok(count)
2439 }
2440}
2441
2442fn avro_value_to_datum(value: &AvroValue) -> Result<AvroDatum> {
2443 let branch = match value {
2444 AvroValue::Null => AvroDatum::Union(0, Box::new(AvroDatum::Null)),
2445 AvroValue::Boolean(value) => AvroDatum::Union(
2446 1,
2447 Box::new(AvroDatum::Record(vec![(
2448 "boolean".to_string(),
2449 AvroDatum::Boolean(*value),
2450 )])),
2451 ),
2452 AvroValue::Long(value) => AvroDatum::Union(
2453 2,
2454 Box::new(AvroDatum::Record(vec![(
2455 "long".to_string(),
2456 AvroDatum::Long(*value),
2457 )])),
2458 ),
2459 AvroValue::Double(value) => {
2460 if !value.is_finite() {
2461 return Err(Error::Codec(
2462 "non_finite_float: Avro Value doubles must be finite".to_string(),
2463 ));
2464 }
2465 AvroDatum::Union(
2466 3,
2467 Box::new(AvroDatum::Record(vec![(
2468 "double".to_string(),
2469 AvroDatum::Double(*value),
2470 )])),
2471 )
2472 }
2473 AvroValue::Bytes(value) => AvroDatum::Union(
2474 4,
2475 Box::new(AvroDatum::Record(vec![(
2476 "bytes".to_string(),
2477 AvroDatum::Bytes(value.clone()),
2478 )])),
2479 ),
2480 AvroValue::String(value) => AvroDatum::Union(
2481 5,
2482 Box::new(AvroDatum::Record(vec![(
2483 "string".to_string(),
2484 AvroDatum::String(value.clone()),
2485 )])),
2486 ),
2487 AvroValue::Array(values) => AvroDatum::Union(
2488 6,
2489 Box::new(AvroDatum::Record(vec![(
2490 "items".to_string(),
2491 AvroDatum::Array(
2492 values
2493 .iter()
2494 .map(avro_value_to_datum)
2495 .collect::<Result<Vec<_>>>()?,
2496 ),
2497 )])),
2498 ),
2499 AvroValue::Map(values) => AvroDatum::Union(
2500 7,
2501 Box::new(AvroDatum::Record(vec![(
2502 "entries".to_string(),
2503 AvroDatum::Array(
2504 values
2505 .iter()
2506 .map(|(key, value)| {
2507 Ok(AvroDatum::Record(vec![
2508 ("key".to_string(), AvroDatum::String(key.clone())),
2509 ("value".to_string(), avro_value_to_datum(value)?),
2510 ]))
2511 })
2512 .collect::<Result<Vec<_>>>()?,
2513 ),
2514 )])),
2515 ),
2516 };
2517 Ok(AvroDatum::Record(vec![("value".to_string(), branch)]))
2518}
2519
2520fn avro_value_from_datum(datum: AvroDatum) -> Result<AvroValue> {
2521 let AvroDatum::Record(mut outer) = datum else {
2522 return Err(Error::Codec(
2523 "invalid_payload_framing: datum is not a Value record".to_string(),
2524 ));
2525 };
2526 let (_, branch) = outer
2527 .pop()
2528 .filter(|(name, _)| name == "value")
2529 .ok_or_else(|| Error::Codec("invalid_payload_framing: Value field missing".to_string()))?;
2530 let AvroDatum::Union(_, branch) = branch else {
2531 return Err(Error::Codec(
2532 "invalid_payload_framing: invalid Value union".to_string(),
2533 ));
2534 };
2535 match *branch {
2536 AvroDatum::Null => Ok(AvroValue::Null),
2537 AvroDatum::Record(mut fields) => {
2538 let (name, value) = fields.pop().ok_or_else(|| {
2539 Error::Codec("invalid_payload_framing: empty Value branch".to_string())
2540 })?;
2541 match (name.as_str(), value) {
2542 ("boolean", AvroDatum::Boolean(value)) => Ok(AvroValue::Boolean(value)),
2543 ("long", AvroDatum::Long(value)) => Ok(AvroValue::Long(value)),
2544 ("double", AvroDatum::Double(value)) if value.is_finite() => {
2545 Ok(AvroValue::Double(value))
2546 }
2547 ("bytes", AvroDatum::Bytes(value)) => Ok(AvroValue::Bytes(value)),
2548 ("string", AvroDatum::String(value)) => Ok(AvroValue::String(value)),
2549 ("items", AvroDatum::Array(values)) => values
2550 .into_iter()
2551 .map(avro_value_from_datum)
2552 .collect::<Result<Vec<_>>>()
2553 .map(AvroValue::Array),
2554 ("entries", AvroDatum::Map(values)) => values
2555 .into_iter()
2556 .map(|(key, value)| Ok((key, avro_value_from_datum(value)?)))
2557 .collect::<Result<BTreeMap<_, _>>>()
2558 .map(AvroValue::Map),
2559 _ => Err(Error::Codec(
2560 "invalid_payload_framing: unknown Value branch".to_string(),
2561 )),
2562 }
2563 }
2564 _ => Err(Error::Codec(
2565 "invalid_payload_framing: invalid Value branch".to_string(),
2566 )),
2567 }
2568}
2569
2570fn avro_value_schema() -> Result<&'static Schema> {
2571 match AVRO_VALUE_SCHEMA.get_or_init(|| {
2572 Schema::parse_str(AVRO_VALUE_SCHEMA_JSON)
2573 .map_err(|err| format!("could not parse Avro Value schema: {err}"))
2574 }) {
2575 Ok(schema) => Ok(schema),
2576 Err(message) => Err(Error::Codec(message.clone())),
2577 }
2578}
2579
2580fn avro_value_ordered_map_encoding_schema() -> Result<&'static Schema> {
2581 match AVRO_VALUE_ORDERED_MAP_ENCODING_SCHEMA.get_or_init(|| {
2582 let mut schema: Value = serde_json::from_str(AVRO_VALUE_SCHEMA_JSON)
2587 .map_err(|err| format!("could not read packaged Avro Value schema: {err}"))?;
2588 let entries_schema = schema
2589 .pointer_mut("/fields/0/type/7/fields/0/type")
2590 .ok_or_else(|| "packaged Avro Value map schema is missing".to_string())?;
2591 if *entries_schema != json!({"type": "map", "values": "Value"}) {
2592 return Err("packaged Avro Value map schema changed unexpectedly".to_string());
2593 }
2594 *entries_schema = json!({
2595 "type": "array",
2596 "items": {
2597 "type": "record",
2598 "name": "MapEntry",
2599 "fields": [
2600 {"name": "key", "type": "string"},
2601 {"name": "value", "type": "Value"}
2602 ]
2603 }
2604 });
2605 Schema::parse_str(&schema.to_string())
2606 .map_err(|err| format!("could not parse ordered-map Avro Value schema: {err}"))
2607 }) {
2608 Ok(schema) => Ok(schema),
2609 Err(message) => Err(Error::Codec(message.clone())),
2610 }
2611}
2612
2613#[derive(Clone, Debug)]
2614pub struct Client {
2615 http: reqwest::Client,
2616 base_url: String,
2617 token: Option<String>,
2618 control_token: Option<String>,
2619 worker_token: Option<String>,
2620 namespace: String,
2621}
2622
2623impl Client {
2624 pub fn new(base_url: impl Into<String>) -> Result<Self> {
2625 Self::builder(base_url).build()
2626 }
2627
2628 pub fn builder(base_url: impl Into<String>) -> ClientBuilder {
2629 ClientBuilder {
2630 base_url: base_url.into(),
2631 token: None,
2632 control_token: None,
2633 worker_token: None,
2634 namespace: "default".to_string(),
2635 timeout: Duration::from_secs(60),
2636 }
2637 }
2638
2639 pub async fn health(&self) -> Result<Value> {
2640 self.request_json(
2641 reqwest::Method::GET,
2642 "/health",
2643 RequestProtocol::ControlPlane,
2644 Option::<&Value>::None,
2645 )
2646 .await
2647 }
2648
2649 pub async fn cluster_info(&self) -> Result<Value> {
2650 self.request_json(
2651 reqwest::Method::GET,
2652 "/cluster/info",
2653 RequestProtocol::ControlPlane,
2654 Option::<&Value>::None,
2655 )
2656 .await
2657 }
2658
2659 pub async fn start_workflow<T: Serialize>(
2660 &self,
2661 workflow_type: &str,
2662 task_queue: &str,
2663 workflow_id: &str,
2664 input: T,
2665 ) -> Result<WorkflowHandle> {
2666 self.start_workflow_with_options(
2667 workflow_type,
2668 task_queue,
2669 workflow_id,
2670 WorkflowStartOptions::default(),
2671 input,
2672 )
2673 .await
2674 }
2675
2676 pub async fn start_workflow_with_options<T: Serialize>(
2679 &self,
2680 workflow_type: &str,
2681 task_queue: &str,
2682 workflow_id: &str,
2683 options: WorkflowStartOptions,
2684 input: T,
2685 ) -> Result<WorkflowHandle> {
2686 options.validate()?;
2687 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2688 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2689 let body = json!({
2690 "workflow_id": workflow_id,
2691 "workflow_type": workflow_type,
2692 "task_queue": task_queue,
2693 "input": input_envelope,
2694 "execution_timeout_seconds": options.execution_timeout_seconds,
2695 "run_timeout_seconds": options.run_timeout_seconds
2696 });
2697
2698 let data: Value = self
2699 .request_json(
2700 reqwest::Method::POST,
2701 "/workflows",
2702 RequestProtocol::ControlPlane,
2703 Some(&body),
2704 )
2705 .await?;
2706
2707 Ok(WorkflowHandle {
2708 client: self.clone(),
2709 workflow_id: data
2710 .get("workflow_id")
2711 .and_then(Value::as_str)
2712 .unwrap_or(workflow_id)
2713 .to_string(),
2714 run_id: data
2715 .get("run_id")
2716 .and_then(Value::as_str)
2717 .map(str::to_string),
2718 workflow_type: data
2719 .get("workflow_type")
2720 .and_then(Value::as_str)
2721 .unwrap_or(workflow_type)
2722 .to_string(),
2723 })
2724 }
2725
2726 pub async fn signal_workflow<T: Serialize>(
2727 &self,
2728 workflow_id: &str,
2729 signal_name: &str,
2730 input: T,
2731 ) -> Result<Value> {
2732 self.signal_workflow_target(workflow_id, None, signal_name, input)
2733 .await
2734 }
2735
2736 pub async fn append_message_stream<T: Serialize>(
2738 &self,
2739 workflow_id: &str,
2740 stream_name: &str,
2741 message_id: &str,
2742 input: T,
2743 ) -> Result<Value> {
2744 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2745 let body = json!({
2746 "message_id": message_id,
2747 "input": encode_typed_envelope(&input, DEFAULT_CODEC)?
2748 });
2749 self.request_json(
2750 reqwest::Method::POST,
2751 &format!("/workflows/{workflow_id}/message-streams/{stream_name}/messages"),
2752 RequestProtocol::ControlPlane,
2753 Some(&body),
2754 )
2755 .await
2756 }
2757
2758 pub async fn signal_workflow_run<T: Serialize>(
2760 &self,
2761 workflow_id: &str,
2762 run_id: &str,
2763 signal_name: &str,
2764 input: T,
2765 ) -> Result<Value> {
2766 self.signal_workflow_target(workflow_id, Some(run_id), signal_name, input)
2767 .await
2768 }
2769
2770 async fn signal_workflow_target<T: Serialize>(
2771 &self,
2772 workflow_id: &str,
2773 run_id: Option<&str>,
2774 signal_name: &str,
2775 input: T,
2776 ) -> Result<Value> {
2777 validate_user_signal_name(signal_name)?;
2778 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2779 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2780 let body = json!({
2781 "input": input_envelope
2782 });
2783 let path = match run_id {
2784 Some(run_id) => {
2785 format!("/workflows/{workflow_id}/runs/{run_id}/signal/{signal_name}")
2786 }
2787 None => format!("/workflows/{workflow_id}/signal/{signal_name}"),
2788 };
2789 self.request_json(
2790 reqwest::Method::POST,
2791 &path,
2792 RequestProtocol::ControlPlane,
2793 Some(&body),
2794 )
2795 .await
2796 }
2797
2798 pub async fn cancel_workflow(
2800 &self,
2801 workflow_id: &str,
2802 options: WorkflowCommandOptions,
2803 ) -> Result<WorkflowCommandResult> {
2804 self.workflow_command(workflow_id, None, WorkflowCommandKind::Cancel, options)
2805 .await
2806 }
2807
2808 pub async fn cancel_workflow_run(
2810 &self,
2811 workflow_id: &str,
2812 run_id: &str,
2813 options: WorkflowCommandOptions,
2814 ) -> Result<WorkflowCommandResult> {
2815 self.workflow_command(
2816 workflow_id,
2817 Some(run_id),
2818 WorkflowCommandKind::Cancel,
2819 options,
2820 )
2821 .await
2822 }
2823
2824 pub async fn terminate_workflow(
2826 &self,
2827 workflow_id: &str,
2828 options: WorkflowCommandOptions,
2829 ) -> Result<WorkflowCommandResult> {
2830 self.workflow_command(workflow_id, None, WorkflowCommandKind::Terminate, options)
2831 .await
2832 }
2833
2834 pub async fn terminate_workflow_run(
2836 &self,
2837 workflow_id: &str,
2838 run_id: &str,
2839 options: WorkflowCommandOptions,
2840 ) -> Result<WorkflowCommandResult> {
2841 self.workflow_command(
2842 workflow_id,
2843 Some(run_id),
2844 WorkflowCommandKind::Terminate,
2845 options,
2846 )
2847 .await
2848 }
2849
2850 async fn workflow_command(
2851 &self,
2852 workflow_id: &str,
2853 run_id: Option<&str>,
2854 command: WorkflowCommandKind,
2855 options: WorkflowCommandOptions,
2856 ) -> Result<WorkflowCommandResult> {
2857 let path = match run_id {
2858 Some(run_id) => format!(
2859 "/workflows/{workflow_id}/runs/{run_id}/{}",
2860 command.as_str()
2861 ),
2862 None => format!("/workflows/{workflow_id}/{}", command.as_str()),
2863 };
2864 let data = match self
2865 .request_json(
2866 reqwest::Method::POST,
2867 &path,
2868 RequestProtocol::ControlPlane,
2869 Some(&options),
2870 )
2871 .await
2872 {
2873 Ok(data) => data,
2874 Err(Error::Http { status, body }) => {
2875 return Err(Error::WorkflowCommandRejected(workflow_command_rejection(
2876 command,
2877 status,
2878 body,
2879 workflow_id,
2880 run_id,
2881 )));
2882 }
2883 Err(error) => return Err(error),
2884 };
2885
2886 Ok(workflow_command_result(command, data, workflow_id, run_id))
2887 }
2888
2889 pub async fn query_workflow<T: Serialize>(
2895 &self,
2896 workflow_id: &str,
2897 query_name: &str,
2898 input: T,
2899 ) -> Result<Value> {
2900 self.query_workflow_target(workflow_id, None, query_name, input)
2901 .await
2902 }
2903
2904 pub async fn query_workflow_run<T: Serialize>(
2906 &self,
2907 workflow_id: &str,
2908 run_id: &str,
2909 query_name: &str,
2910 input: T,
2911 ) -> Result<Value> {
2912 self.query_workflow_target(workflow_id, Some(run_id), query_name, input)
2913 .await
2914 }
2915
2916 pub async fn query_workflow_avro_value<T: Serialize>(
2918 &self,
2919 workflow_id: &str,
2920 query_name: &str,
2921 input: T,
2922 ) -> Result<AvroValue> {
2923 self.query_workflow_avro_value_target(workflow_id, None, query_name, input)
2924 .await
2925 }
2926
2927 pub async fn query_workflow_run_avro_value<T: Serialize>(
2929 &self,
2930 workflow_id: &str,
2931 run_id: &str,
2932 query_name: &str,
2933 input: T,
2934 ) -> Result<AvroValue> {
2935 self.query_workflow_avro_value_target(workflow_id, Some(run_id), query_name, input)
2936 .await
2937 }
2938
2939 async fn query_workflow_avro_value_target<T: Serialize>(
2940 &self,
2941 workflow_id: &str,
2942 run_id: Option<&str>,
2943 query_name: &str,
2944 input: T,
2945 ) -> Result<AvroValue> {
2946 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2947 let body = json!({"input": encode_typed_envelope(&input, DEFAULT_CODEC)?});
2948 let path = match run_id {
2949 Some(run_id) => {
2950 format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
2951 }
2952 None => format!("/workflows/{workflow_id}/query/{query_name}"),
2953 };
2954 let response: Value = match self
2955 .request_json(
2956 reqwest::Method::POST,
2957 &path,
2958 RequestProtocol::ControlPlane,
2959 Some(&body),
2960 )
2961 .await
2962 {
2963 Ok(response) => response,
2964 Err(Error::Http { status, body }) => {
2965 return Err(Error::QueryFailed(query_failure(status, body)));
2966 }
2967 Err(error) => return Err(error),
2968 };
2969
2970 let envelope = response
2971 .get("result_envelope")
2972 .filter(|envelope| !envelope.is_null())
2973 .ok_or_else(|| {
2974 Error::Codec(
2975 "missing_payload_envelope: typed query result requires result_envelope"
2976 .to_string(),
2977 )
2978 })?;
2979 decode_wire_avro_value(envelope, DEFAULT_CODEC)
2980 }
2981
2982 async fn query_workflow_target<T: Serialize>(
2983 &self,
2984 workflow_id: &str,
2985 run_id: Option<&str>,
2986 query_name: &str,
2987 input: T,
2988 ) -> Result<Value> {
2989 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2990 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2991 let body = json!({
2992 "input": input_envelope
2993 });
2994 let path = match run_id {
2995 Some(run_id) => {
2996 format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
2997 }
2998 None => format!("/workflows/{workflow_id}/query/{query_name}"),
2999 };
3000 let response: Value = match self
3001 .request_json(
3002 reqwest::Method::POST,
3003 &path,
3004 RequestProtocol::ControlPlane,
3005 Some(&body),
3006 )
3007 .await
3008 {
3009 Ok(response) => response,
3010 Err(Error::Http { status, body }) => {
3011 return Err(Error::QueryFailed(query_failure(status, body)));
3012 }
3013 Err(error) => return Err(error),
3014 };
3015
3016 if let Some(envelope) = response
3017 .get("result_envelope")
3018 .filter(|envelope| !envelope.is_null())
3019 {
3020 return decode_wire_value(envelope, DEFAULT_CODEC);
3021 }
3022
3023 Ok(response.get("result").cloned().unwrap_or(Value::Null))
3024 }
3025
3026 pub async fn update_workflow<T: Serialize>(
3028 &self,
3029 workflow_id: &str,
3030 update_name: &str,
3031 input: T,
3032 request_id: Option<&str>,
3033 ) -> Result<Value> {
3034 let response = self
3035 .update_workflow_response(workflow_id, update_name, input, request_id)
3036 .await?;
3037 if let Some(envelope) = response
3038 .get("result_envelope")
3039 .filter(|envelope| !envelope.is_null())
3040 {
3041 return decode_wire_value(envelope, DEFAULT_CODEC);
3042 }
3043 Ok(response.get("result").cloned().unwrap_or(response))
3044 }
3045
3046 pub async fn update_workflow_avro_value<T: Serialize>(
3048 &self,
3049 workflow_id: &str,
3050 update_name: &str,
3051 input: T,
3052 request_id: Option<&str>,
3053 ) -> Result<AvroValue> {
3054 let response = self
3055 .update_workflow_response(workflow_id, update_name, input, request_id)
3056 .await?;
3057 let envelope = response
3058 .get("result_envelope")
3059 .filter(|envelope| !envelope.is_null())
3060 .ok_or_else(|| {
3061 Error::Codec(
3062 "missing_payload_envelope: typed update result requires result_envelope"
3063 .to_string(),
3064 )
3065 })?;
3066 decode_wire_avro_value(envelope, DEFAULT_CODEC)
3067 }
3068
3069 async fn update_workflow_response<T: Serialize>(
3070 &self,
3071 workflow_id: &str,
3072 update_name: &str,
3073 input: T,
3074 request_id: Option<&str>,
3075 ) -> Result<Value> {
3076 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
3077 let mut body = json!({
3078 "input": encode_typed_envelope(&input, DEFAULT_CODEC)?,
3079 "wait_for": "completed",
3080 });
3081 if let Some(request_id) = request_id {
3082 body["request_id"] = json!(request_id);
3083 }
3084 self.request_json(
3085 reqwest::Method::POST,
3086 &format!("/workflows/{workflow_id}/update/{update_name}"),
3087 RequestProtocol::ControlPlane,
3088 Some(&body),
3089 )
3090 .await
3091 }
3092
3093 pub async fn describe_workflow(&self, workflow_id: &str) -> Result<WorkflowDescription> {
3094 let path = format!("/workflows/{workflow_id}");
3095 let mut data: WorkflowDescription = self
3096 .request_json(
3097 reqwest::Method::GET,
3098 &path,
3099 RequestProtocol::ControlPlane,
3100 Option::<&Value>::None,
3101 )
3102 .await?;
3103 data.decode_payloads()?;
3104 Ok(data)
3105 }
3106
3107 pub async fn describe_workflow_run(
3109 &self,
3110 workflow_id: &str,
3111 run_id: &str,
3112 ) -> Result<WorkflowDescription> {
3113 let path = format!("/workflows/{workflow_id}/runs/{run_id}");
3114 let mut data: WorkflowDescription = self
3115 .request_json(
3116 reqwest::Method::GET,
3117 &path,
3118 RequestProtocol::ControlPlane,
3119 Option::<&Value>::None,
3120 )
3121 .await?;
3122 data.decode_payloads()?;
3123 Ok(data)
3124 }
3125
3126 fn workflow_stream_path(workflow_id: &str, run_id: &str, stream_name: Option<&str>) -> String {
3127 let mut path = format!(
3128 "/workflows/{}/runs/{}/streams",
3129 percent_encode_path_segment(workflow_id),
3130 percent_encode_path_segment(run_id),
3131 );
3132 if let Some(stream_name) = stream_name {
3133 path.push('/');
3134 path.push_str(&percent_encode_path_segment(stream_name));
3135 }
3136 path
3137 }
3138
3139 pub async fn list_workflow_streams(
3141 &self,
3142 workflow_id: &str,
3143 run_id: &str,
3144 ) -> Result<Vec<WorkflowStreamDescription>> {
3145 let response: WorkflowStreamListResponse = self
3146 .request_json(
3147 reqwest::Method::GET,
3148 &Self::workflow_stream_path(workflow_id, run_id, None),
3149 RequestProtocol::ControlPlane,
3150 Option::<&Value>::None,
3151 )
3152 .await?;
3153 Ok(response.streams)
3154 }
3155
3156 pub async fn describe_workflow_stream(
3158 &self,
3159 workflow_id: &str,
3160 run_id: &str,
3161 stream_name: &str,
3162 ) -> Result<WorkflowStreamDescription> {
3163 let response: WorkflowStreamDescriptionResponse = self
3164 .request_json(
3165 reqwest::Method::GET,
3166 &Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3167 RequestProtocol::ControlPlane,
3168 Option::<&Value>::None,
3169 )
3170 .await?;
3171 Ok(response.stream)
3172 }
3173
3174 pub async fn subscribe_workflow_stream(
3181 &self,
3182 workflow_id: &str,
3183 run_id: &str,
3184 stream_name: &str,
3185 from_offset: u64,
3186 max_items: usize,
3187 wait: Duration,
3188 ) -> Result<WorkflowStreamPage> {
3189 let max_items = max_items.clamp(1, 500);
3190 let wait_seconds = wait.as_secs().min(MAX_LONG_POLL_TIMEOUT_SECONDS);
3191 let path = format!(
3192 "{}/items?from={from_offset}&max_items={max_items}&wait_seconds={wait_seconds}",
3193 Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3194 );
3195 let response: WorkflowStreamPageResponse = self
3196 .request_json_with_timeout(
3197 reqwest::Method::GET,
3198 &path,
3199 RequestProtocol::ControlPlane,
3200 Option::<&Value>::None,
3201 Duration::from_secs(wait_seconds.saturating_add(5).max(5)),
3202 )
3203 .await?;
3204
3205 let items = response
3206 .items
3207 .into_iter()
3208 .map(|raw| {
3209 let offset = raw.get("offset").and_then(Value::as_u64).unwrap_or(0);
3210 let envelope = raw.get("payload").cloned();
3211 let payload = envelope
3212 .as_ref()
3213 .filter(|value| value.get("blob").is_some())
3214 .map(|value| decode_wire_avro_value(value, DEFAULT_CODEC))
3215 .transpose()?
3216 .map(AvroValue::into_json)
3217 .transpose()?;
3218 Ok(WorkflowStreamItem {
3219 offset,
3220 payload,
3221 payload_envelope: envelope,
3222 payload_reference: raw
3223 .get("payload_reference")
3224 .and_then(Value::as_str)
3225 .map(str::to_string),
3226 payload_codec: raw
3227 .get("payload_codec")
3228 .and_then(Value::as_str)
3229 .map(str::to_string),
3230 idempotency_key: raw
3231 .get("idempotency_key")
3232 .and_then(Value::as_str)
3233 .map(str::to_string),
3234 item_type: raw
3235 .get("item_type")
3236 .and_then(Value::as_str)
3237 .map(str::to_string),
3238 content_type: raw
3239 .get("content_type")
3240 .and_then(Value::as_str)
3241 .map(str::to_string),
3242 origin: raw
3243 .get("origin")
3244 .and_then(Value::as_str)
3245 .map(str::to_string),
3246 origin_reference: raw
3247 .get("origin_reference")
3248 .and_then(Value::as_str)
3249 .map(str::to_string),
3250 emitted_at: raw
3251 .get("emitted_at")
3252 .and_then(Value::as_str)
3253 .map(str::to_string),
3254 raw,
3255 })
3256 })
3257 .collect::<Result<Vec<_>>>()?;
3258 Ok(WorkflowStreamPage {
3259 stream: response.stream,
3260 items,
3261 next_offset: response.next_offset,
3262 terminal: response.terminal,
3263 })
3264 }
3265
3266 pub async fn append_workflow_stream(
3268 &self,
3269 workflow_id: &str,
3270 run_id: &str,
3271 stream_name: &str,
3272 items: &[WorkflowStreamAppendItem],
3273 max_pending_items: Option<u64>,
3274 ) -> Result<WorkflowStreamAppendResult> {
3275 if items.is_empty() {
3276 return Err(Error::Codec(
3277 "workflow_stream_items_empty: append requires at least one item".to_string(),
3278 ));
3279 }
3280 let mut body = json!({
3281 "items": items
3282 .iter()
3283 .map(|item| item.wire_value(None))
3284 .collect::<Vec<_>>(),
3285 });
3286 if let Some(max_pending_items) = max_pending_items {
3287 if max_pending_items == 0 {
3288 return Err(Error::Codec(
3289 "workflow_stream_pending_limit_invalid: max_pending_items must be positive"
3290 .to_string(),
3291 ));
3292 }
3293 body["max_pending_items"] = json!(max_pending_items);
3294 }
3295 let response: WorkflowStreamAppendResponse = self
3296 .request_json(
3297 reqwest::Method::POST,
3298 &format!(
3299 "{}/items",
3300 Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3301 ),
3302 RequestProtocol::ControlPlane,
3303 Some(&body),
3304 )
3305 .await?;
3306 Ok(WorkflowStreamAppendResult {
3307 stream: response.stream,
3308 accepted_offsets: response.accepted_offsets,
3309 accepted: response.accepted,
3310 deduped: response.deduped,
3311 })
3312 }
3313
3314 pub async fn close_workflow_stream(
3316 &self,
3317 workflow_id: &str,
3318 run_id: &str,
3319 stream_name: &str,
3320 error_reason: Option<&str>,
3321 retention_seconds: Option<u64>,
3322 ) -> Result<WorkflowStreamDescription> {
3323 let mut body = json!({});
3324 if let Some(error_reason) = error_reason {
3325 body["error_reason"] = json!(error_reason);
3326 }
3327 if let Some(retention_seconds) = retention_seconds {
3328 if retention_seconds == 0 {
3329 return Err(Error::Codec(
3330 "workflow_stream_retention_invalid: retention_seconds must be positive"
3331 .to_string(),
3332 ));
3333 }
3334 body["retention_seconds"] = json!(retention_seconds);
3335 }
3336 let response: WorkflowStreamDescriptionResponse = self
3337 .request_json(
3338 reqwest::Method::POST,
3339 &format!(
3340 "{}/close",
3341 Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3342 ),
3343 RequestProtocol::ControlPlane,
3344 Some(&body),
3345 )
3346 .await?;
3347 Ok(response.stream)
3348 }
3349
3350 pub async fn register_worker(
3351 &self,
3352 worker_id: &str,
3353 task_queue: &str,
3354 supported_workflow_types: Vec<String>,
3355 supported_activity_types: Vec<String>,
3356 max_concurrent_workflow_tasks: usize,
3357 max_concurrent_activity_tasks: usize,
3358 ) -> Result<RegisterWorkerResponse> {
3359 self.register_worker_with_capabilities(
3360 worker_id,
3361 task_queue,
3362 supported_workflow_types,
3363 supported_activity_types,
3364 max_concurrent_workflow_tasks,
3365 max_concurrent_activity_tasks,
3366 Vec::new(),
3367 )
3368 .await
3369 }
3370
3371 pub async fn register_worker_with_capabilities(
3373 &self,
3374 worker_id: &str,
3375 task_queue: &str,
3376 supported_workflow_types: Vec<String>,
3377 supported_activity_types: Vec<String>,
3378 max_concurrent_workflow_tasks: usize,
3379 max_concurrent_activity_tasks: usize,
3380 capabilities: Vec<String>,
3381 ) -> Result<RegisterWorkerResponse> {
3382 self.register_worker_with_command_contracts(
3383 worker_id,
3384 task_queue,
3385 supported_workflow_types,
3386 supported_activity_types,
3387 max_concurrent_workflow_tasks,
3388 max_concurrent_activity_tasks,
3389 capabilities,
3390 Value::Object(serde_json::Map::new()),
3391 )
3392 .await
3393 }
3394
3395 #[allow(clippy::too_many_arguments)]
3402 pub async fn register_worker_with_command_contracts(
3403 &self,
3404 worker_id: &str,
3405 task_queue: &str,
3406 supported_workflow_types: Vec<String>,
3407 supported_activity_types: Vec<String>,
3408 max_concurrent_workflow_tasks: usize,
3409 max_concurrent_activity_tasks: usize,
3410 capabilities: Vec<String>,
3411 workflow_command_contracts: Value,
3412 ) -> Result<RegisterWorkerResponse> {
3413 if let Some(contracts) = workflow_command_contracts.as_object() {
3414 for (workflow_type, contract) in contracts {
3415 let Some(update_validators) = contract.get("update_validators") else {
3416 continue;
3417 };
3418 if !update_validators
3419 .as_array()
3420 .is_some_and(|validators| validators.is_empty())
3421 {
3422 return Err(Error::UnsupportedUpdateValidators {
3423 workflow_type: workflow_type.clone(),
3424 });
3425 }
3426 }
3427 }
3428
3429 let mut body = json!({
3430 "worker_id": worker_id,
3431 "task_queue": task_queue,
3432 "runtime": "rust",
3433 "sdk_version": SDK_VERSION,
3434 "supported_workflow_types": supported_workflow_types,
3435 "supported_activity_types": supported_activity_types,
3436 "capabilities": capabilities,
3437 "capability_manifest": portable_worker_affinity_capability_manifest(),
3438 "max_concurrent_workflow_tasks": max_concurrent_workflow_tasks,
3439 "max_concurrent_activity_tasks": max_concurrent_activity_tasks
3440 });
3441 if workflow_command_contracts
3442 .as_object()
3443 .is_some_and(|contracts| !contracts.is_empty())
3444 {
3445 body["workflow_command_contracts"] = workflow_command_contracts;
3446 }
3447
3448 self.request_json(
3449 reqwest::Method::POST,
3450 "/worker/register",
3451 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3452 Some(&body),
3453 )
3454 .await
3455 }
3456
3457 pub async fn deregister_worker_registration(
3463 &self,
3464 worker_id: &str,
3465 ) -> Result<WorkerDeregistrationEnvelope> {
3466 let path = format!(
3467 "/worker/registrations/{}",
3468 percent_encode_path_segment(worker_id)
3469 );
3470 self.request_json(
3471 reqwest::Method::DELETE,
3472 &path,
3473 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3474 Option::<&Value>::None,
3475 )
3476 .await
3477 }
3478
3479 pub async fn poll_query_task(
3481 &self,
3482 worker_id: &str,
3483 task_queue: &str,
3484 timeout: Duration,
3485 ) -> Result<Option<QueryTask>> {
3486 Ok(self
3487 .poll_query_task_response(worker_id, task_queue, timeout)
3488 .await?
3489 .task)
3490 }
3491
3492 pub async fn poll_query_task_response(
3494 &self,
3495 worker_id: &str,
3496 task_queue: &str,
3497 timeout: Duration,
3498 ) -> Result<PollQueryTaskResponse> {
3499 let poll_request_id = unique_request_id("rust-query-poll");
3500 self.poll_query_task_response_with_request_id(
3501 worker_id,
3502 task_queue,
3503 timeout,
3504 &poll_request_id,
3505 1,
3506 )
3507 .await
3508 }
3509
3510 async fn poll_query_task_response_with_request_id(
3511 &self,
3512 worker_id: &str,
3513 task_queue: &str,
3514 timeout: Duration,
3515 poll_request_id: &str,
3516 transport_retries: usize,
3517 ) -> Result<PollQueryTaskResponse> {
3518 let timeout_seconds = long_poll_timeout_seconds(timeout);
3519 let body = json!({
3520 "worker_id": worker_id,
3521 "task_queue": task_queue,
3522 "poll_request_id": poll_request_id,
3523 "timeout_seconds": timeout_seconds,
3524 });
3525 self.poll_request_json(
3526 "/worker/query-tasks/poll",
3527 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3528 &body,
3529 timeout + Duration::from_secs(5),
3530 transport_retries,
3531 )
3532 .await
3533 }
3534
3535 pub async fn complete_query_task<T: Serialize>(
3537 &self,
3538 query_task_id: &str,
3539 lease_owner: &str,
3540 query_task_attempt: u64,
3541 result: T,
3542 codec: &str,
3543 ) -> Result<Value> {
3544 let typed_result = AvroValue::from_serialize(&result)?;
3545 let result_envelope = encode_typed_envelope(&typed_result, codec)?;
3546 self.complete_query_task_with_envelope(
3547 query_task_id,
3548 lease_owner,
3549 query_task_attempt,
3550 typed_result.into_json()?,
3551 result_envelope,
3552 )
3553 .await
3554 }
3555
3556 async fn complete_query_task_with_envelope(
3557 &self,
3558 query_task_id: &str,
3559 lease_owner: &str,
3560 query_task_attempt: u64,
3561 result: Value,
3562 result_envelope: Value,
3563 ) -> Result<Value> {
3564 let body = json!({
3565 "lease_owner": lease_owner,
3566 "query_task_attempt": query_task_attempt,
3567 "result": result,
3568 "result_envelope": result_envelope,
3569 });
3570 let path = format!("/worker/query-tasks/{query_task_id}/complete");
3571 let response = self
3572 .request_json(
3573 reqwest::Method::POST,
3574 &path,
3575 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3576 Some(&body),
3577 )
3578 .await;
3579 query_task_response(response)
3580 }
3581
3582 pub async fn fail_query_task(
3584 &self,
3585 query_task_id: &str,
3586 lease_owner: &str,
3587 query_task_attempt: u64,
3588 message: impl Into<String>,
3589 reason: impl Into<String>,
3590 failure_type: impl Into<String>,
3591 ) -> Result<Value> {
3592 let body = json!({
3593 "lease_owner": lease_owner,
3594 "query_task_attempt": query_task_attempt,
3595 "failure": {
3596 "message": message.into(),
3597 "reason": reason.into(),
3598 "type": failure_type.into(),
3599 }
3600 });
3601 let path = format!("/worker/query-tasks/{query_task_id}/fail");
3602 let response = self
3603 .request_json(
3604 reqwest::Method::POST,
3605 &path,
3606 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3607 Some(&body),
3608 )
3609 .await;
3610 query_task_response(response)
3611 }
3612
3613 pub async fn heartbeat_worker(
3614 &self,
3615 worker_id: &str,
3616 workflow_available: usize,
3617 activity_available: usize,
3618 ) -> Result<Value> {
3619 let body = json!({
3620 "worker_id": worker_id,
3621 "task_slots": {
3622 "workflow_available": workflow_available,
3623 "activity_available": activity_available
3624 },
3625 "process_metrics": {
3626 "process_id": std::process::id(),
3627 "process_uptime_seconds": 0
3628 }
3629 });
3630
3631 self.request_json(
3632 reqwest::Method::POST,
3633 "/worker/heartbeat",
3634 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3635 Some(&body),
3636 )
3637 .await
3638 }
3639
3640 pub async fn poll_workflow_task(
3641 &self,
3642 worker_id: &str,
3643 task_queue: &str,
3644 timeout: Duration,
3645 ) -> Result<Option<WorkflowTask>> {
3646 Ok(self
3647 .poll_workflow_task_response(worker_id, task_queue, timeout)
3648 .await?
3649 .task)
3650 }
3651
3652 pub async fn poll_workflow_task_response(
3653 &self,
3654 worker_id: &str,
3655 task_queue: &str,
3656 timeout: Duration,
3657 ) -> Result<PollWorkflowTaskResponse> {
3658 let poll_request_id = unique_request_id("rust-workflow-poll");
3659 self.poll_workflow_task_response_with_request_id(
3660 worker_id,
3661 task_queue,
3662 timeout,
3663 &poll_request_id,
3664 1,
3665 )
3666 .await
3667 }
3668
3669 async fn poll_workflow_task_response_with_request_id(
3670 &self,
3671 worker_id: &str,
3672 task_queue: &str,
3673 timeout: Duration,
3674 poll_request_id: &str,
3675 transport_retries: usize,
3676 ) -> Result<PollWorkflowTaskResponse> {
3677 let body = json!({
3678 "worker_id": worker_id,
3679 "task_queue": task_queue,
3680 "poll_request_id": poll_request_id,
3681 "timeout_seconds": long_poll_timeout_seconds(timeout),
3682 });
3683 let mut data: PollWorkflowTaskResponse = self
3684 .poll_request_json(
3685 "/worker/workflow-tasks/poll",
3686 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3687 &body,
3688 timeout + Duration::from_secs(5),
3689 transport_retries,
3690 )
3691 .await?;
3692
3693 if let Some(task) = data.task.as_mut() {
3694 self.fetch_remaining_workflow_history(worker_id, task)
3695 .await?;
3696 }
3697
3698 Ok(data)
3699 }
3700
3701 async fn fetch_remaining_workflow_history(
3702 &self,
3703 worker_id: &str,
3704 task: &mut WorkflowTask,
3705 ) -> Result<()> {
3706 let mut next_token = task.next_history_page_token.clone();
3707
3708 while let Some(token) = next_token.take().filter(|token| !token.is_empty()) {
3709 let lease_owner = task
3710 .lease_owner
3711 .clone()
3712 .unwrap_or_else(|| worker_id.to_string());
3713 let page = self
3714 .workflow_task_history_page(
3715 &task.task_id,
3716 &lease_owner,
3717 task.workflow_task_attempt,
3718 &token,
3719 )
3720 .await?;
3721
3722 task.append_history_page(page);
3723
3724 if task.next_history_page_token.as_deref() == Some(token.as_str()) {
3725 return Err(Error::Codec(
3726 "workflow history pagination returned the same page token".to_string(),
3727 ));
3728 }
3729
3730 next_token = task.next_history_page_token.clone();
3731 }
3732
3733 Ok(())
3734 }
3735
3736 async fn workflow_task_history_page(
3737 &self,
3738 task_id: &str,
3739 lease_owner: &str,
3740 workflow_task_attempt: u64,
3741 next_history_page_token: &str,
3742 ) -> Result<WorkflowTaskHistoryPage> {
3743 let body = json!({
3744 "lease_owner": lease_owner,
3745 "workflow_task_attempt": workflow_task_attempt,
3746 "next_history_page_token": next_history_page_token
3747 });
3748 let path = format!("/worker/workflow-tasks/{task_id}/history");
3749
3750 self.request_json(
3751 reqwest::Method::POST,
3752 &path,
3753 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3754 Some(&body),
3755 )
3756 .await
3757 }
3758
3759 pub async fn complete_workflow_task(
3760 &self,
3761 task_id: &str,
3762 lease_owner: &str,
3763 workflow_task_attempt: u64,
3764 commands: Vec<Value>,
3765 ) -> Result<Value> {
3766 self.complete_workflow_task_with_message_streams(
3767 task_id,
3768 lease_owner,
3769 workflow_task_attempt,
3770 commands,
3771 Vec::new(),
3772 Vec::new(),
3773 )
3774 .await
3775 }
3776
3777 async fn complete_workflow_task_with_message_streams(
3778 &self,
3779 task_id: &str,
3780 lease_owner: &str,
3781 workflow_task_attempt: u64,
3782 commands: Vec<Value>,
3783 message_stream_cursors: Vec<Value>,
3784 message_stream_waits: Vec<Value>,
3785 ) -> Result<Value> {
3786 validate_workflow_task_commands(&commands)?;
3787 let has_message_stream_metadata =
3788 !message_stream_cursors.is_empty() || !message_stream_waits.is_empty();
3789 if has_message_stream_metadata
3790 && !worker_protocol_supports_message_streams(WORKER_PROTOCOL_VERSION)
3791 {
3792 return Err(Error::Codec(
3793 "message_streams_unavailable: message stream completion metadata requires worker protocol 1.15 or newer"
3794 .to_string(),
3795 ));
3796 }
3797 let protocol_version = workflow_completion_protocol_version_with_message_streams(
3798 &commands,
3799 has_message_stream_metadata,
3800 );
3801 let mut body = json!({
3802 "lease_owner": lease_owner,
3803 "workflow_task_attempt": workflow_task_attempt,
3804 "commands": commands
3805 });
3806 if !message_stream_cursors.is_empty() {
3807 body["message_stream_cursors"] = Value::Array(message_stream_cursors);
3808 }
3809 if !message_stream_waits.is_empty() {
3810 body["message_stream_waits"] = Value::Array(message_stream_waits);
3811 }
3812 let path = format!("/worker/workflow-tasks/{task_id}/complete");
3813 self.request_json(
3814 reqwest::Method::POST,
3815 &path,
3816 RequestProtocol::Worker(protocol_version),
3817 Some(&body),
3818 )
3819 .await
3820 }
3821
3822 pub async fn fail_workflow_task(
3823 &self,
3824 task_id: &str,
3825 lease_owner: &str,
3826 workflow_task_attempt: u64,
3827 message: impl Into<String>,
3828 ) -> Result<Value> {
3829 self.fail_workflow_task_with_type(
3830 task_id,
3831 lease_owner,
3832 workflow_task_attempt,
3833 message,
3834 "RustWorkflowTaskFailure",
3835 )
3836 .await
3837 }
3838
3839 async fn fail_workflow_task_with_type(
3840 &self,
3841 task_id: &str,
3842 lease_owner: &str,
3843 workflow_task_attempt: u64,
3844 message: impl Into<String>,
3845 failure_type: &str,
3846 ) -> Result<Value> {
3847 let body = json!({
3848 "lease_owner": lease_owner,
3849 "workflow_task_attempt": workflow_task_attempt,
3850 "failure": {
3851 "message": message.into(),
3852 "type": failure_type
3853 }
3854 });
3855 let path = format!("/worker/workflow-tasks/{task_id}/fail");
3856 self.request_json(
3857 reqwest::Method::POST,
3858 &path,
3859 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3860 Some(&body),
3861 )
3862 .await
3863 }
3864
3865 pub async fn poll_activity_task(
3866 &self,
3867 worker_id: &str,
3868 task_queue: &str,
3869 timeout: Duration,
3870 ) -> Result<Option<ActivityTask>> {
3871 Ok(self
3872 .poll_activity_task_response(worker_id, task_queue, timeout)
3873 .await?
3874 .task)
3875 }
3876
3877 pub async fn poll_activity_task_response(
3879 &self,
3880 worker_id: &str,
3881 task_queue: &str,
3882 timeout: Duration,
3883 ) -> Result<PollActivityTaskResponse> {
3884 let poll_request_id = unique_request_id("rust-activity-poll");
3885 self.poll_activity_task_response_with_request_id(
3886 worker_id,
3887 task_queue,
3888 timeout,
3889 &poll_request_id,
3890 1,
3891 )
3892 .await
3893 }
3894
3895 async fn poll_activity_task_response_with_request_id(
3896 &self,
3897 worker_id: &str,
3898 task_queue: &str,
3899 timeout: Duration,
3900 poll_request_id: &str,
3901 transport_retries: usize,
3902 ) -> Result<PollActivityTaskResponse> {
3903 let body = json!({
3904 "worker_id": worker_id,
3905 "task_queue": task_queue,
3906 "poll_request_id": poll_request_id,
3907 "timeout_seconds": long_poll_timeout_seconds(timeout),
3908 });
3909 let data: PollActivityTaskResponse = self
3910 .poll_request_json(
3911 "/worker/activity-tasks/poll",
3912 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3913 &body,
3914 timeout + Duration::from_secs(5),
3915 transport_retries,
3916 )
3917 .await?;
3918 Ok(data)
3919 }
3920
3921 pub async fn complete_activity_task<T: Serialize>(
3922 &self,
3923 task_id: &str,
3924 activity_attempt_id: &str,
3925 lease_owner: &str,
3926 result: T,
3927 codec: &str,
3928 ) -> Result<Value> {
3929 let result = encode_typed_envelope(&AvroValue::from_serialize(&result)?, codec)?;
3930 let body = json!({
3931 "activity_attempt_id": activity_attempt_id,
3932 "lease_owner": lease_owner,
3933 "result": result
3934 });
3935 let path = format!("/worker/activity-tasks/{task_id}/complete");
3936 activity_task_response(
3937 self.request_json(
3938 reqwest::Method::POST,
3939 &path,
3940 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3941 Some(&body),
3942 )
3943 .await,
3944 "complete",
3945 task_id,
3946 activity_attempt_id,
3947 )
3948 }
3949
3950 pub async fn fail_activity_task(
3951 &self,
3952 task_id: &str,
3953 activity_attempt_id: &str,
3954 lease_owner: &str,
3955 message: impl Into<String>,
3956 non_retryable: bool,
3957 ) -> Result<Value> {
3958 let body = json!({
3959 "activity_attempt_id": activity_attempt_id,
3960 "lease_owner": lease_owner,
3961 "failure": {
3962 "message": message.into(),
3963 "type": "RustActivityFailure",
3964 "non_retryable": non_retryable
3965 }
3966 });
3967 let path = format!("/worker/activity-tasks/{task_id}/fail");
3968 activity_task_response(
3969 self.request_json(
3970 reqwest::Method::POST,
3971 &path,
3972 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3973 Some(&body),
3974 )
3975 .await,
3976 "fail",
3977 task_id,
3978 activity_attempt_id,
3979 )
3980 }
3981
3982 pub async fn heartbeat_activity_task<T: Serialize>(
3983 &self,
3984 task_id: &str,
3985 activity_attempt_id: &str,
3986 lease_owner: &str,
3987 details: T,
3988 ) -> Result<ActivityHeartbeatResponse> {
3989 let details = encode_typed_envelope(&AvroValue::from_serialize(&details)?, DEFAULT_CODEC)?;
3990 let body = json!({
3991 "activity_attempt_id": activity_attempt_id,
3992 "lease_owner": lease_owner,
3993 "details": details
3994 });
3995 let path = format!("/worker/activity-tasks/{task_id}/heartbeat");
3996 activity_task_response(
3997 self.request_json(
3998 reqwest::Method::POST,
3999 &path,
4000 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
4001 Some(&body),
4002 )
4003 .await,
4004 "heartbeat",
4005 task_id,
4006 activity_attempt_id,
4007 )
4008 }
4009
4010 async fn request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
4011 &self,
4012 method: reqwest::Method,
4013 path: &str,
4014 protocol: RequestProtocol,
4015 body: Option<&B>,
4016 ) -> Result<T> {
4017 self.request_json_with_timeout(method, path, protocol, body, Duration::from_secs(60))
4018 .await
4019 }
4020
4021 async fn request_json_with_timeout<T: DeserializeOwned, B: Serialize + ?Sized>(
4022 &self,
4023 method: reqwest::Method,
4024 path: &str,
4025 protocol: RequestProtocol,
4026 body: Option<&B>,
4027 timeout: Duration,
4028 ) -> Result<T> {
4029 let auth_token = self.auth_token(protocol)?;
4030 let mut request = self
4031 .http
4032 .request(method, format!("{}/api{}", self.base_url, path))
4033 .timeout(timeout)
4034 .header(reqwest::header::ACCEPT, "application/json")
4035 .header(reqwest::header::CONTENT_TYPE, "application/json")
4036 .header("X-Namespace", &self.namespace);
4037
4038 match protocol {
4039 RequestProtocol::Worker(version) => {
4040 request = request.header("X-Durable-Workflow-Protocol-Version", version);
4041 }
4042 RequestProtocol::ControlPlane => {
4043 request = request.header(
4044 "X-Durable-Workflow-Control-Plane-Version",
4045 CONTROL_PLANE_VERSION,
4046 );
4047 }
4048 }
4049
4050 if let Some(token) = auth_token {
4051 request = request.bearer_auth(token);
4052 }
4053
4054 if let Some(body) = body {
4055 request = request.json(body);
4056 }
4057
4058 let response = request.send().await?;
4059 let status = response.status();
4060 let bytes = response.bytes().await?;
4061
4062 if !status.is_success() {
4063 let body = String::from_utf8_lossy(&bytes).to_string();
4064 if let Some(protocol) = protocol_failure(status, &body) {
4065 return Err(Error::Protocol(protocol));
4066 }
4067 return Err(Error::Http { status, body });
4068 }
4069
4070 if bytes.is_empty() {
4071 return Ok(serde_json::from_value(Value::Null)?);
4072 }
4073
4074 Ok(serde_json::from_slice(&bytes)?)
4075 }
4076
4077 async fn poll_request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
4078 &self,
4079 path: &str,
4080 protocol: RequestProtocol,
4081 body: &B,
4082 timeout: Duration,
4083 max_retries: usize,
4084 ) -> Result<T> {
4085 let mut retries = 0;
4086
4087 loop {
4088 let response = self
4089 .request_json_with_timeout(
4090 reqwest::Method::POST,
4091 path,
4092 protocol,
4093 Some(body),
4094 timeout,
4095 )
4096 .await;
4097
4098 match response {
4099 Err(Error::Transport(_)) if retries < max_retries => retries += 1,
4100 response => return worker_poll_response(response),
4101 }
4102 }
4103 }
4104
4105 fn auth_token(&self, protocol: RequestProtocol) -> Result<Option<&str>> {
4106 match protocol {
4107 RequestProtocol::Worker(_) => {
4108 if let Some(token) = self.worker_token.as_deref().or(self.token.as_deref()) {
4109 return Ok(Some(token));
4110 }
4111 if self.control_token.is_some() {
4112 return Err(Error::MissingRoleCredentials {
4113 role: "worker",
4114 opposite_role: "control",
4115 });
4116 }
4117 Ok(None)
4118 }
4119 RequestProtocol::ControlPlane => {
4120 if let Some(token) = self.control_token.as_deref().or(self.token.as_deref()) {
4121 return Ok(Some(token));
4122 }
4123 if self.worker_token.is_some() {
4124 return Err(Error::MissingRoleCredentials {
4125 role: "control",
4126 opposite_role: "worker",
4127 });
4128 }
4129 Ok(None)
4130 }
4131 }
4132 }
4133}
4134
4135fn query_failure(status: reqwest::StatusCode, raw_body: String) -> QueryFailure {
4136 let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
4137 let reason = body
4138 .get("reason")
4139 .and_then(Value::as_str)
4140 .unwrap_or("query_rejected")
4141 .to_string();
4142 let message = body
4143 .get("message")
4144 .or_else(|| body.get("error"))
4145 .and_then(Value::as_str)
4146 .unwrap_or("workflow query was rejected")
4147 .to_string();
4148
4149 QueryFailure {
4150 status: status.as_u16(),
4151 reason,
4152 message,
4153 body,
4154 }
4155}
4156
4157fn workflow_command_result(
4158 command: WorkflowCommandKind,
4159 data: Value,
4160 workflow_id: &str,
4161 run_id: Option<&str>,
4162) -> WorkflowCommandResult {
4163 WorkflowCommandResult {
4164 command,
4165 workflow_id: data
4166 .get("workflow_id")
4167 .and_then(Value::as_str)
4168 .unwrap_or(workflow_id)
4169 .to_string(),
4170 run_id: data
4171 .get("run_id")
4172 .and_then(Value::as_str)
4173 .or(run_id)
4174 .map(str::to_string),
4175 outcome: data
4176 .get("outcome")
4177 .and_then(Value::as_str)
4178 .map(str::to_string),
4179 reason: data
4180 .get("reason")
4181 .and_then(Value::as_str)
4182 .map(str::to_string),
4183 command_status: data
4184 .get("command_status")
4185 .and_then(Value::as_str)
4186 .map(str::to_string),
4187 raw: data,
4188 }
4189}
4190
4191fn workflow_command_rejection(
4192 command: WorkflowCommandKind,
4193 status: reqwest::StatusCode,
4194 raw_body: String,
4195 workflow_id: &str,
4196 run_id: Option<&str>,
4197) -> WorkflowCommandRejection {
4198 let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
4199 WorkflowCommandRejection {
4200 command,
4201 status: status.as_u16(),
4202 reason: body
4203 .get("reason")
4204 .and_then(Value::as_str)
4205 .unwrap_or("workflow_command_rejected")
4206 .to_string(),
4207 message: body
4208 .get("message")
4209 .or_else(|| body.get("error"))
4210 .and_then(Value::as_str)
4211 .unwrap_or("workflow lifecycle command was rejected")
4212 .to_string(),
4213 workflow_id: body
4214 .get("workflow_id")
4215 .and_then(Value::as_str)
4216 .unwrap_or(workflow_id)
4217 .to_string(),
4218 run_id: body
4219 .get("run_id")
4220 .and_then(Value::as_str)
4221 .or(run_id)
4222 .map(str::to_string),
4223 target_scope: body
4224 .get("target_scope")
4225 .and_then(Value::as_str)
4226 .map(str::to_string),
4227 body,
4228 }
4229}
4230
4231fn query_task_response(response: Result<Value>) -> Result<Value> {
4232 match response {
4233 Err(Error::Http { status, body }) => Err(Error::QueryFailed(query_failure(status, body))),
4234 response => response,
4235 }
4236}
4237
4238fn worker_poll_response<T: DeserializeOwned>(response: Result<T>) -> Result<T> {
4239 match response {
4240 Err(Error::Http { status, body })
4241 if status == reqwest::StatusCode::CONFLICT && worker_poll_body_is_stop(&body) =>
4242 {
4243 Ok(serde_json::from_str(&body)?)
4244 }
4245 response => response,
4246 }
4247}
4248
4249fn worker_poll_body_is_stop(body: &str) -> bool {
4250 serde_json::from_str::<Value>(body)
4251 .ok()
4252 .is_some_and(|body| {
4253 worker_poll_is_stop(
4254 body.get("poll_status").and_then(Value::as_str),
4255 body.get("reason").and_then(Value::as_str),
4256 )
4257 })
4258}
4259
4260fn worker_poll_is_stop(poll_status: Option<&str>, reason: Option<&str>) -> bool {
4261 matches!(poll_status, Some("draining" | "stopped"))
4262 || matches!(reason, Some("worker_draining" | "worker_stopped"))
4263}
4264
4265fn query_task_rejection_is_final(error: &Error) -> bool {
4266 matches!(
4267 error,
4268 Error::QueryFailed(failure)
4269 if QUERY_TASK_FINAL_REJECTION_REASONS.contains(&failure.reason.as_str())
4270 )
4271}
4272
4273fn activity_task_response<T>(
4274 response: Result<T>,
4275 operation: &str,
4276 task_id: &str,
4277 activity_attempt_id: &str,
4278) -> Result<T> {
4279 match response {
4280 Err(Error::Http { status, body }) => {
4281 let body = serde_json::from_str(&body).unwrap_or_else(|_| json!({"message": body}));
4282 Err(Error::ActivityTaskRejected(ActivityTaskRejection {
4283 operation: operation.to_string(),
4284 status: status.as_u16(),
4285 reason: body
4286 .get("reason")
4287 .and_then(Value::as_str)
4288 .unwrap_or("activity_task_rejected")
4289 .to_string(),
4290 task_id: body
4291 .get("task_id")
4292 .and_then(Value::as_str)
4293 .unwrap_or(task_id)
4294 .to_string(),
4295 activity_attempt_id: body
4296 .get("activity_attempt_id")
4297 .and_then(Value::as_str)
4298 .unwrap_or(activity_attempt_id)
4299 .to_string(),
4300 cancel_requested: body
4301 .get("cancel_requested")
4302 .and_then(Value::as_bool)
4303 .unwrap_or(false),
4304 can_continue: body.get("can_continue").and_then(Value::as_bool),
4305 run_closed_reason: body
4306 .get("run_closed_reason")
4307 .and_then(Value::as_str)
4308 .map(str::to_string),
4309 body,
4310 }))
4311 }
4312 response => response,
4313 }
4314}
4315
4316fn activity_task_rejection_is_final(error: &Error) -> bool {
4317 matches!(
4318 error,
4319 Error::ActivityTaskRejected(rejection)
4320 if matches!(
4321 rejection.reason.as_str(),
4322 "run_cancelled"
4323 | "run_terminated"
4324 | "attempt_closed"
4325 | "stale_attempt"
4326 | "activity_cancelled"
4327 | "task_cancelled"
4328 | "run_closed"
4329 | "activity_not_running"
4330 | "attempt_not_found"
4331 )
4332 )
4333}
4334
4335fn workflow_task_completion_is_terminal_timeout(
4336 error: &Error,
4337 task_id: &str,
4338 workflow_task_attempt: u64,
4339 run_id: Option<&str>,
4340) -> bool {
4341 let Error::Http { status, body } = error else {
4342 return false;
4343 };
4344 if *status != reqwest::StatusCode::CONFLICT {
4345 return false;
4346 }
4347
4348 let Some(run_id) = run_id else {
4349 return false;
4350 };
4351 let Ok(body) = serde_json::from_str::<Value>(body) else {
4352 return false;
4353 };
4354
4355 body.get("recorded").and_then(Value::as_bool) == Some(false)
4356 && body.get("reason").and_then(Value::as_str) == Some("run_timed_out")
4357 && body.get("run_status").and_then(Value::as_str) == Some("failed")
4358 && body.get("run_id").and_then(Value::as_str) == Some(run_id)
4359 && body.get("task_id").and_then(Value::as_str) == Some(task_id)
4360 && body.get("workflow_task_attempt").and_then(Value::as_u64) == Some(workflow_task_attempt)
4361}
4362
4363fn protocol_failure(status: reqwest::StatusCode, raw_body: &str) -> Option<ProtocolFailure> {
4364 let body: Value = serde_json::from_str(raw_body).ok()?;
4365 let reason = body.get("reason")?.as_str()?;
4366 if !matches!(
4367 reason,
4368 "missing_protocol_version"
4369 | "unsupported_protocol_version"
4370 | "missing_control_plane_version"
4371 | "unsupported_control_plane_version"
4372 ) {
4373 return None;
4374 }
4375
4376 Some(ProtocolFailure {
4377 status: status.as_u16(),
4378 reason: reason.to_string(),
4379 message: body
4380 .get("message")
4381 .or_else(|| body.get("error"))
4382 .and_then(Value::as_str)
4383 .unwrap_or("protocol version rejected")
4384 .to_string(),
4385 supported_version: body
4386 .get("supported_version")
4387 .and_then(Value::as_str)
4388 .map(str::to_string),
4389 requested_version: body
4390 .get("requested_version")
4391 .and_then(Value::as_str)
4392 .map(str::to_string),
4393 body,
4394 })
4395}
4396
4397fn long_poll_timeout_seconds(timeout: Duration) -> u64 {
4398 timeout
4399 .as_secs()
4400 .saturating_add(u64::from(timeout.subsec_nanos() > 0))
4401 .min(MAX_LONG_POLL_TIMEOUT_SECONDS)
4402}
4403
4404fn worker_operation_is_retryable(error: &Error) -> bool {
4405 if worker_poll_capacity_retry_after(error).is_some()
4406 || worker_operation_is_explicitly_non_retryable(error)
4407 {
4408 return false;
4409 }
4410
4411 match error {
4412 Error::Transport(error) => {
4413 error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()
4414 }
4415 Error::Http { status, .. } => {
4416 matches!(
4417 *status,
4418 reqwest::StatusCode::REQUEST_TIMEOUT | reqwest::StatusCode::TOO_MANY_REQUESTS
4419 ) || status.is_server_error()
4420 }
4421 _ => false,
4422 }
4423}
4424
4425fn worker_operation_is_explicitly_non_retryable(error: &Error) -> bool {
4426 let Error::Http { body, .. } = error else {
4427 return false;
4428 };
4429
4430 serde_json::from_str::<Value>(body)
4431 .ok()
4432 .and_then(|body| body.get("retryable").and_then(Value::as_bool))
4433 == Some(false)
4434}
4435
4436fn worker_poll_capacity_retry_after(error: &Error) -> Option<Duration> {
4437 let Error::Http { status, body } = error else {
4438 return None;
4439 };
4440 if *status != reqwest::StatusCode::TOO_MANY_REQUESTS {
4441 return None;
4442 }
4443
4444 let body = serde_json::from_str::<Value>(body).ok()?;
4445 let capacity_exhausted = body.get("poll_status").and_then(Value::as_str)
4446 == Some("long_poll_capacity_exhausted")
4447 || body.get("reason").and_then(Value::as_str) == Some("long_poll_capacity_exhausted");
4448 if !capacity_exhausted || body.get("retryable").and_then(Value::as_bool) != Some(true) {
4449 return None;
4450 }
4451
4452 Some(Duration::from_secs(
4453 body.get("retry_after_seconds")
4454 .and_then(Value::as_u64)
4455 .unwrap_or_default(),
4456 ))
4457}
4458
4459fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration {
4460 let exponent = retry.saturating_sub(1).min(31) as u32;
4461 policy
4462 .initial_backoff
4463 .saturating_mul(1_u32 << exponent)
4464 .min(policy.max_backoff)
4465}
4466
4467#[derive(Debug)]
4468pub struct ClientBuilder {
4469 base_url: String,
4470 token: Option<String>,
4471 control_token: Option<String>,
4472 worker_token: Option<String>,
4473 namespace: String,
4474 timeout: Duration,
4475}
4476
4477impl ClientBuilder {
4478 pub fn token(mut self, token: Option<String>) -> Self {
4479 self.token = token;
4480 self
4481 }
4482
4483 pub fn control_token(mut self, token: Option<String>) -> Self {
4484 self.control_token = token;
4485 self
4486 }
4487
4488 pub fn worker_token(mut self, token: Option<String>) -> Self {
4489 self.worker_token = token;
4490 self
4491 }
4492
4493 pub fn namespace(mut self, namespace: impl Into<String>) -> Self {
4494 self.namespace = namespace.into();
4495 self
4496 }
4497
4498 pub fn timeout(mut self, timeout: Duration) -> Self {
4499 self.timeout = timeout;
4500 self
4501 }
4502
4503 pub fn build(self) -> Result<Client> {
4504 let base_url = self.base_url.trim_end_matches('/').to_string();
4505 let has_sdk_api_suffix = reqwest::Url::parse(&base_url)
4506 .map(|url| url.path().trim_end_matches('/').ends_with("/api"))
4507 .unwrap_or_else(|_| base_url.ends_with("/api"));
4508
4509 if has_sdk_api_suffix {
4510 return Err(Error::InvalidBaseUrl);
4511 }
4512
4513 Ok(Client {
4514 http: reqwest::Client::builder().timeout(self.timeout).build()?,
4515 base_url,
4516 token: self.token,
4517 control_token: self.control_token,
4518 worker_token: self.worker_token,
4519 namespace: self.namespace,
4520 })
4521 }
4522}
4523
4524#[derive(Clone, Debug)]
4525pub struct WorkflowHandle {
4526 client: Client,
4527 pub workflow_id: String,
4528 pub run_id: Option<String>,
4529 pub workflow_type: String,
4530}
4531
4532impl WorkflowHandle {
4533 pub async fn describe(&self) -> Result<WorkflowDescription> {
4535 self.client.describe_workflow(&self.workflow_id).await
4536 }
4537
4538 pub async fn describe_selected_run(&self) -> Result<WorkflowDescription> {
4540 let run_id = self.run_id.as_deref().ok_or_else(|| {
4541 Error::Codec("run_id is required for selected-run description".to_string())
4542 })?;
4543 self.client
4544 .describe_workflow_run(&self.workflow_id, run_id)
4545 .await
4546 }
4547
4548 pub async fn signal<T: Serialize>(&self, signal_name: &str, input: T) -> Result<Value> {
4549 self.client
4550 .signal_workflow(&self.workflow_id, signal_name, input)
4551 .await
4552 }
4553
4554 pub async fn append_message<T: Serialize>(
4555 &self,
4556 stream_name: &str,
4557 message_id: &str,
4558 input: T,
4559 ) -> Result<Value> {
4560 self.client
4561 .append_message_stream(&self.workflow_id, stream_name, message_id, input)
4562 .await
4563 }
4564
4565 pub async fn signal_selected_run<T: Serialize>(
4567 &self,
4568 signal_name: &str,
4569 input: T,
4570 ) -> Result<Value> {
4571 let run_id = self.run_id.as_deref().ok_or_else(|| {
4572 Error::Codec("run_id is required for selected-run signaling".to_string())
4573 })?;
4574 self.client
4575 .signal_workflow_run(&self.workflow_id, run_id, signal_name, input)
4576 .await
4577 }
4578
4579 pub async fn cancel(&self, options: WorkflowCommandOptions) -> Result<WorkflowCommandResult> {
4581 self.client
4582 .cancel_workflow(&self.workflow_id, options)
4583 .await
4584 }
4585
4586 pub async fn cancel_selected_run(
4588 &self,
4589 options: WorkflowCommandOptions,
4590 ) -> Result<WorkflowCommandResult> {
4591 let run_id = self.run_id.as_deref().ok_or_else(|| {
4592 Error::Codec("run_id is required for selected-run cancellation".to_string())
4593 })?;
4594 self.client
4595 .cancel_workflow_run(&self.workflow_id, run_id, options)
4596 .await
4597 }
4598
4599 pub async fn terminate(
4601 &self,
4602 options: WorkflowCommandOptions,
4603 ) -> Result<WorkflowCommandResult> {
4604 self.client
4605 .terminate_workflow(&self.workflow_id, options)
4606 .await
4607 }
4608
4609 pub async fn terminate_selected_run(
4611 &self,
4612 options: WorkflowCommandOptions,
4613 ) -> Result<WorkflowCommandResult> {
4614 let run_id = self.run_id.as_deref().ok_or_else(|| {
4615 Error::Codec("run_id is required for selected-run termination".to_string())
4616 })?;
4617 self.client
4618 .terminate_workflow_run(&self.workflow_id, run_id, options)
4619 .await
4620 }
4621
4622 pub async fn query<T: Serialize>(&self, query_name: &str, input: T) -> Result<Value> {
4624 self.client
4625 .query_workflow(&self.workflow_id, query_name, input)
4626 .await
4627 }
4628
4629 pub async fn query_avro_value<T: Serialize>(
4630 &self,
4631 query_name: &str,
4632 input: T,
4633 ) -> Result<AvroValue> {
4634 self.client
4635 .query_workflow_avro_value(&self.workflow_id, query_name, input)
4636 .await
4637 }
4638
4639 pub async fn update<T: Serialize>(
4640 &self,
4641 update_name: &str,
4642 input: T,
4643 request_id: Option<&str>,
4644 ) -> Result<Value> {
4645 self.client
4646 .update_workflow(&self.workflow_id, update_name, input, request_id)
4647 .await
4648 }
4649
4650 pub async fn update_avro_value<T: Serialize>(
4651 &self,
4652 update_name: &str,
4653 input: T,
4654 request_id: Option<&str>,
4655 ) -> Result<AvroValue> {
4656 self.client
4657 .update_workflow_avro_value(&self.workflow_id, update_name, input, request_id)
4658 .await
4659 }
4660
4661 pub async fn query_selected_run<T: Serialize>(
4663 &self,
4664 query_name: &str,
4665 input: T,
4666 ) -> Result<Value> {
4667 let run_id = self
4668 .run_id
4669 .as_deref()
4670 .ok_or_else(|| Error::Codec("run_id is required for selected-run query".to_string()))?;
4671 self.client
4672 .query_workflow_run(&self.workflow_id, run_id, query_name, input)
4673 .await
4674 }
4675
4676 pub async fn result(&self, options: WorkflowResultOptions) -> Result<Value> {
4678 self.result_target(options, None).await
4679 }
4680
4681 pub async fn result_avro_value(&self, options: WorkflowResultOptions) -> Result<AvroValue> {
4683 self.result_avro_value_target(options, None).await
4684 }
4685
4686 pub async fn result_typed<T: DeserializeOwned>(
4688 &self,
4689 options: WorkflowResultOptions,
4690 ) -> Result<T> {
4691 let result = self.result_avro_value(options).await?;
4692 decode_handler_result(result, HandlerKind::Workflow, &self.workflow_type)
4693 }
4694
4695 pub async fn result_selected_run(&self, options: WorkflowResultOptions) -> Result<Value> {
4697 let run_id = self.run_id.as_deref().ok_or_else(|| {
4698 Error::Codec("run_id is required for selected-run result".to_string())
4699 })?;
4700 self.result_target(options, Some(run_id)).await
4701 }
4702
4703 pub async fn result_selected_run_avro_value(
4705 &self,
4706 options: WorkflowResultOptions,
4707 ) -> Result<AvroValue> {
4708 let run_id = self.run_id.as_deref().ok_or_else(|| {
4709 Error::Codec("run_id is required for selected-run result".to_string())
4710 })?;
4711 self.result_avro_value_target(options, Some(run_id)).await
4712 }
4713
4714 pub async fn result_selected_run_typed<T: DeserializeOwned>(
4716 &self,
4717 options: WorkflowResultOptions,
4718 ) -> Result<T> {
4719 let result = self.result_selected_run_avro_value(options).await?;
4720 decode_handler_result(result, HandlerKind::Workflow, &self.workflow_type)
4721 }
4722
4723 async fn result_avro_value_target(
4724 &self,
4725 options: WorkflowResultOptions,
4726 selected_run_id: Option<&str>,
4727 ) -> Result<AvroValue> {
4728 let started = Instant::now();
4729
4730 loop {
4731 let description = match selected_run_id {
4732 Some(run_id) => {
4733 self.client
4734 .describe_workflow_run(&self.workflow_id, run_id)
4735 .await?
4736 }
4737 None => self.describe().await?,
4738 };
4739 if description.is_completed() {
4740 return description.output_avro_value.ok_or_else(|| {
4741 Error::Codec(
4742 "missing_payload_envelope: typed workflow result requires output_envelope"
4743 .to_string(),
4744 )
4745 });
4746 }
4747 if description.is_terminal() {
4748 let outcome =
4749 workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
4750 return Err(match outcome.kind {
4751 WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
4752 WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
4753 WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
4754 WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
4755 });
4756 }
4757 if started.elapsed() >= options.timeout {
4758 return Err(Error::Timeout);
4759 }
4760 tokio::time::sleep(options.poll_interval).await;
4761 }
4762 }
4763
4764 async fn result_target(
4765 &self,
4766 options: WorkflowResultOptions,
4767 selected_run_id: Option<&str>,
4768 ) -> Result<Value> {
4769 let started = Instant::now();
4770
4771 loop {
4772 let description = match selected_run_id {
4773 Some(run_id) => {
4774 self.client
4775 .describe_workflow_run(&self.workflow_id, run_id)
4776 .await?
4777 }
4778 None => self.describe().await?,
4779 };
4780 if description.is_completed() {
4781 return Ok(description.output.unwrap_or(Value::Null));
4782 }
4783
4784 if description.is_terminal() {
4785 let outcome =
4786 workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
4787 return Err(match outcome.kind {
4788 WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
4789 WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
4790 WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
4791 WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
4792 });
4793 }
4794
4795 if started.elapsed() >= options.timeout {
4796 return Err(Error::WorkflowTimedOut(WorkflowTerminalOutcome {
4797 kind: WorkflowTerminalKind::TimedOut,
4798 workflow_id: description
4799 .workflow_id
4800 .clone()
4801 .unwrap_or_else(|| self.workflow_id.clone()),
4802 run_id: description
4803 .run_id
4804 .clone()
4805 .or_else(|| selected_run_id.map(str::to_string)),
4806 reason: "result_wait_timeout".to_string(),
4807 failure_category: Some("client_timeout".to_string()),
4808 failure_id: None,
4809 exception_type: None,
4810 exception_class: None,
4811 non_retryable: None,
4812 message: Some(format!(
4813 "workflow result was not terminal within {:?}",
4814 options.timeout
4815 )),
4816 exception: None,
4817 raw: description.raw_value(),
4818 }));
4819 }
4820
4821 tokio::time::sleep(options.poll_interval).await;
4822 }
4823 }
4824}
4825
4826#[derive(Clone, Copy, Debug)]
4827pub struct WorkflowResultOptions {
4828 pub poll_interval: Duration,
4829 pub timeout: Duration,
4830}
4831
4832impl Default for WorkflowResultOptions {
4833 fn default() -> Self {
4834 Self {
4835 poll_interval: Duration::from_millis(500),
4836 timeout: Duration::from_secs(30),
4837 }
4838 }
4839}
4840
4841#[derive(Clone, Debug, Deserialize)]
4842pub struct WorkflowDescription {
4843 pub workflow_id: Option<String>,
4844 pub run_id: Option<String>,
4845 pub workflow_type: Option<String>,
4846 pub status: Option<String>,
4847 #[serde(default)]
4848 pub closed_reason: Option<String>,
4849 #[serde(default)]
4850 pub error: Option<String>,
4851 #[serde(default)]
4852 pub failure: Option<Value>,
4853 #[serde(default)]
4854 pub exception: Option<Value>,
4855 #[serde(default)]
4856 pub failures: Vec<Value>,
4857 #[serde(default)]
4858 pub output: Option<Value>,
4859 #[serde(default)]
4860 pub output_envelope: Option<Value>,
4861 #[serde(skip)]
4862 pub output_avro_value: Option<AvroValue>,
4863 #[serde(flatten)]
4864 pub raw: HashMap<String, Value>,
4865}
4866
4867#[derive(Clone, Debug, Deserialize)]
4869pub struct WorkflowStreamDescription {
4870 pub stream_name: String,
4871 pub status: String,
4872 pub last_offset: i64,
4873 pub total_items: u64,
4874 pub pending_items: u64,
4875 #[serde(default)]
4876 pub opened_at: Option<String>,
4877 #[serde(default)]
4878 pub last_appended_at: Option<String>,
4879 #[serde(default)]
4880 pub closed_at: Option<String>,
4881 #[serde(default)]
4882 pub error_reason: Option<String>,
4883 #[serde(default)]
4884 pub retention_seconds: Option<u64>,
4885 #[serde(flatten)]
4886 pub raw: HashMap<String, Value>,
4887}
4888
4889impl WorkflowStreamDescription {
4890 pub fn is_terminal(&self) -> bool {
4891 matches!(self.status.as_str(), "closed" | "errored")
4892 }
4893}
4894
4895#[derive(Clone, Debug, Default)]
4897pub struct WorkflowStreamAppendItem {
4898 pub payload_envelope: Option<Value>,
4899 pub payload_reference: Option<String>,
4900 pub item_type: Option<String>,
4901 pub content_type: Option<String>,
4902 pub idempotency_key: Option<String>,
4903}
4904
4905impl WorkflowStreamAppendItem {
4906 pub fn new<T: Serialize>(payload: T) -> Result<Self> {
4908 let value = AvroValue::from_serialize(&payload)?;
4909 Ok(Self {
4910 payload_envelope: Some(encode_typed_envelope(&value, DEFAULT_CODEC)?),
4911 ..Self::default()
4912 })
4913 }
4914
4915 pub fn from_reference(reference: impl Into<String>) -> Self {
4917 Self {
4918 payload_reference: Some(reference.into()),
4919 ..Self::default()
4920 }
4921 }
4922
4923 pub fn item_type(mut self, item_type: impl Into<String>) -> Self {
4924 self.item_type = Some(item_type.into());
4925 self
4926 }
4927
4928 pub fn content_type(mut self, content_type: impl Into<String>) -> Self {
4929 self.content_type = Some(content_type.into());
4930 self
4931 }
4932
4933 pub fn idempotency_key(mut self, idempotency_key: impl Into<String>) -> Self {
4934 self.idempotency_key = Some(idempotency_key.into());
4935 self
4936 }
4937
4938 fn wire_value(&self, derived_idempotency_key: Option<String>) -> Value {
4939 let mut item = serde_json::Map::new();
4940 if let Some(payload) = &self.payload_envelope {
4941 item.insert("payload".to_string(), payload.clone());
4942 item.insert("payload_codec".to_string(), json!(DEFAULT_CODEC));
4943 }
4944 if let Some(reference) = &self.payload_reference {
4945 item.insert("payload_reference".to_string(), json!(reference));
4946 }
4947 if let Some(item_type) = &self.item_type {
4948 item.insert("item_type".to_string(), json!(item_type));
4949 }
4950 if let Some(content_type) = &self.content_type {
4951 item.insert("content_type".to_string(), json!(content_type));
4952 }
4953 if let Some(key) = derived_idempotency_key
4954 .as_ref()
4955 .or(self.idempotency_key.as_ref())
4956 {
4957 item.insert("idempotency_key".to_string(), json!(key));
4958 }
4959 Value::Object(item)
4960 }
4961}
4962
4963#[derive(Clone, Debug)]
4965pub struct WorkflowStreamItem {
4966 pub offset: u64,
4967 pub payload: Option<Value>,
4968 pub payload_envelope: Option<Value>,
4969 pub payload_reference: Option<String>,
4970 pub payload_codec: Option<String>,
4971 pub idempotency_key: Option<String>,
4972 pub item_type: Option<String>,
4973 pub content_type: Option<String>,
4974 pub origin: Option<String>,
4975 pub origin_reference: Option<String>,
4976 pub emitted_at: Option<String>,
4977 pub raw: Value,
4978}
4979
4980#[derive(Clone, Debug)]
4982pub struct WorkflowStreamPage {
4983 pub stream: WorkflowStreamDescription,
4984 pub items: Vec<WorkflowStreamItem>,
4985 pub next_offset: u64,
4986 pub terminal: bool,
4987}
4988
4989#[derive(Clone, Debug)]
4991pub struct WorkflowStreamAppendResult {
4992 pub stream: WorkflowStreamDescription,
4993 pub accepted_offsets: Vec<u64>,
4994 pub accepted: u64,
4995 pub deduped: u64,
4996}
4997
4998#[derive(Deserialize)]
4999struct WorkflowStreamListResponse {
5000 #[serde(default)]
5001 streams: Vec<WorkflowStreamDescription>,
5002}
5003
5004#[derive(Deserialize)]
5005struct WorkflowStreamDescriptionResponse {
5006 stream: WorkflowStreamDescription,
5007}
5008
5009#[derive(Deserialize)]
5010struct WorkflowStreamPageResponse {
5011 stream: WorkflowStreamDescription,
5012 #[serde(default)]
5013 items: Vec<Value>,
5014 next_offset: u64,
5015 terminal: bool,
5016}
5017
5018#[derive(Deserialize)]
5019struct WorkflowStreamAppendResponse {
5020 stream: WorkflowStreamDescription,
5021 #[serde(default)]
5022 accepted_offsets: Vec<u64>,
5023 accepted: u64,
5024 deduped: u64,
5025}
5026
5027impl WorkflowDescription {
5028 pub fn is_completed(&self) -> bool {
5029 matches!(self.status.as_deref(), Some("completed" | "Completed"))
5030 }
5031
5032 pub fn is_terminal(&self) -> bool {
5033 matches!(
5034 self.status.as_deref(),
5035 Some(
5036 "completed"
5037 | "Completed"
5038 | "failed"
5039 | "Failed"
5040 | "cancelled"
5041 | "Cancelled"
5042 | "terminated"
5043 | "Terminated"
5044 | "timed_out"
5045 | "TimedOut",
5046 )
5047 )
5048 }
5049
5050 fn decode_payloads(&mut self) -> Result<()> {
5051 if let Some(envelope) = &self.output_envelope {
5052 let value = decode_wire_avro_value(envelope, DEFAULT_CODEC)?;
5053 self.output = Some(value.clone().into_json()?);
5054 self.output_avro_value = Some(value);
5055 }
5056
5057 Ok(())
5058 }
5059
5060 fn raw_value(&self) -> Value {
5061 let mut data = self.raw.clone();
5062 data.insert(
5063 "workflow_id".to_string(),
5064 self.workflow_id
5065 .clone()
5066 .map(Value::String)
5067 .unwrap_or(Value::Null),
5068 );
5069 data.insert(
5070 "run_id".to_string(),
5071 self.run_id
5072 .clone()
5073 .map(Value::String)
5074 .unwrap_or(Value::Null),
5075 );
5076 data.insert(
5077 "workflow_type".to_string(),
5078 self.workflow_type
5079 .clone()
5080 .map(Value::String)
5081 .unwrap_or(Value::Null),
5082 );
5083 data.insert(
5084 "status".to_string(),
5085 self.status
5086 .clone()
5087 .map(Value::String)
5088 .unwrap_or(Value::Null),
5089 );
5090 data.insert(
5091 "closed_reason".to_string(),
5092 self.closed_reason
5093 .clone()
5094 .map(Value::String)
5095 .unwrap_or(Value::Null),
5096 );
5097 if let Some(failure) = &self.failure {
5098 data.insert("failure".to_string(), failure.clone());
5099 }
5100 if let Some(exception) = &self.exception {
5101 data.insert("exception".to_string(), exception.clone());
5102 }
5103 Value::Object(data.into_iter().collect())
5104 }
5105}
5106
5107fn workflow_terminal_outcome(
5108 description: &WorkflowDescription,
5109 workflow_id: &str,
5110 run_id: Option<&str>,
5111) -> WorkflowTerminalOutcome {
5112 let terminal_kind = description
5113 .closed_reason
5114 .as_deref()
5115 .or(description.status.as_deref())
5116 .unwrap_or("failed")
5117 .to_ascii_lowercase();
5118 let kind = match terminal_kind.as_str() {
5119 "cancelled" | "canceled" => WorkflowTerminalKind::Cancelled,
5120 "terminated" => WorkflowTerminalKind::Terminated,
5121 "timed_out" | "timedout" => WorkflowTerminalKind::TimedOut,
5122 _ => WorkflowTerminalKind::Failed,
5123 };
5124 let default_reason = match kind {
5125 WorkflowTerminalKind::Failed => "workflow_failed",
5126 WorkflowTerminalKind::Cancelled => "cancelled",
5127 WorkflowTerminalKind::Terminated => "terminated",
5128 WorkflowTerminalKind::TimedOut => "timed_out",
5129 };
5130 let failure = description
5131 .failure
5132 .as_ref()
5133 .filter(|value| value.is_object());
5134 let nested_failure = failure
5135 .and_then(|value| value.get("failures"))
5136 .and_then(Value::as_array)
5137 .and_then(|failures| failures.last())
5138 .or_else(|| description.failures.last());
5139 let exception = description
5140 .exception
5141 .clone()
5142 .or_else(|| failure.and_then(|value| value.get("exception")).cloned())
5143 .or_else(|| {
5144 nested_failure
5145 .and_then(|value| value.get("exception_payload"))
5146 .cloned()
5147 });
5148 let string_field = |name: &str| {
5149 failure
5150 .and_then(|value| value.get(name))
5151 .and_then(Value::as_str)
5152 .or_else(|| {
5153 nested_failure
5154 .and_then(|value| value.get(name))
5155 .and_then(Value::as_str)
5156 })
5157 .map(str::to_string)
5158 };
5159 let exception_field = |name: &str| {
5160 exception
5161 .as_ref()
5162 .and_then(|value| value.get(name))
5163 .and_then(Value::as_str)
5164 .map(str::to_string)
5165 };
5166 let message = description
5167 .error
5168 .clone()
5169 .or_else(|| string_field("message"))
5170 .or_else(|| exception_field("message"));
5171 let reason = description
5172 .raw
5173 .get("reason")
5174 .and_then(Value::as_str)
5175 .map(str::to_string)
5176 .or_else(|| {
5177 failure
5178 .and_then(|value| value.get("reason"))
5179 .and_then(Value::as_str)
5180 .map(str::to_string)
5181 })
5182 .or_else(|| description.closed_reason.clone())
5183 .unwrap_or_else(|| default_reason.to_string());
5184 let failure_id = string_field("failure_id").or_else(|| {
5185 nested_failure
5186 .and_then(|value| value.get("id"))
5187 .and_then(Value::as_str)
5188 .map(str::to_string)
5189 });
5190
5191 WorkflowTerminalOutcome {
5192 kind,
5193 workflow_id: description
5194 .workflow_id
5195 .clone()
5196 .unwrap_or_else(|| workflow_id.to_string()),
5197 run_id: description
5198 .run_id
5199 .clone()
5200 .or_else(|| run_id.map(str::to_string)),
5201 reason,
5202 failure_category: string_field("failure_category")
5203 .or_else(|| Some(default_reason.to_string())),
5204 failure_id,
5205 exception_type: string_field("exception_type").or_else(|| exception_field("type")),
5206 exception_class: string_field("exception_class").or_else(|| exception_field("class")),
5207 non_retryable: failure
5208 .and_then(|value| value.get("non_retryable"))
5209 .and_then(Value::as_bool)
5210 .or_else(|| {
5211 nested_failure
5212 .and_then(|value| value.get("non_retryable"))
5213 .and_then(Value::as_bool)
5214 }),
5215 message,
5216 exception,
5217 raw: description.raw_value(),
5218 }
5219}
5220
5221#[derive(Clone, Debug, Deserialize)]
5222pub struct RegisterWorkerResponse {
5223 pub worker_id: String,
5224 pub registered: bool,
5225 #[serde(default)]
5226 pub heartbeat_interval_seconds: Option<u64>,
5227 #[serde(default)]
5228 pub protocol_version: Option<String>,
5229 #[serde(default)]
5230 pub server_capabilities: Option<Value>,
5231}
5232
5233#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
5235pub struct WorkerDeregistrationEnvelope {
5236 pub worker_id: String,
5237 pub outcome: String,
5238 pub recovered_workflow_task_count: u64,
5239}
5240
5241#[derive(Clone, Debug, Deserialize)]
5242pub struct PollWorkflowTaskResponse {
5243 #[serde(default)]
5244 pub task: Option<WorkflowTask>,
5245 #[serde(default)]
5246 pub poll_status: Option<String>,
5247 #[serde(default)]
5248 pub reason: Option<String>,
5249 #[serde(default)]
5250 pub protocol_version: Option<String>,
5251 #[serde(default)]
5252 pub server_capabilities: Option<Value>,
5253}
5254
5255impl PollWorkflowTaskResponse {
5256 pub fn outcome(&self) -> WorkerPollOutcome {
5258 worker_poll_outcome(
5259 self.task.is_some(),
5260 self.poll_status.as_deref(),
5261 self.reason.as_deref(),
5262 )
5263 }
5264}
5265
5266fn runtime_supports_workflow_memo_updates(capabilities: Option<&Value>) -> bool {
5267 let Some(capabilities) = capabilities.and_then(Value::as_object) else {
5268 return false;
5269 };
5270 let supported = capabilities
5271 .get("workflow_memo_updates")
5272 .and_then(Value::as_object)
5273 .and_then(|memo| memo.get("supported"))
5274 .and_then(Value::as_bool)
5275 == Some(true);
5276 let command_advertised = capabilities
5277 .get("supported_workflow_task_commands")
5278 .and_then(Value::as_array)
5279 .is_some_and(|commands| {
5280 commands
5281 .iter()
5282 .any(|command| command.as_str() == Some("upsert_memo"))
5283 });
5284 supported && command_advertised
5285}
5286
5287fn commands_use_workflow_memo_updates(commands: &[Value]) -> bool {
5288 commands
5289 .iter()
5290 .any(|command| command.get("type").and_then(Value::as_str) == Some("upsert_memo"))
5291}
5292
5293#[derive(Clone, Debug, Deserialize)]
5294pub struct PollActivityTaskResponse {
5295 #[serde(default)]
5296 pub task: Option<ActivityTask>,
5297 #[serde(default)]
5298 pub poll_status: Option<String>,
5299 #[serde(default)]
5300 pub reason: Option<String>,
5301}
5302
5303impl PollActivityTaskResponse {
5304 pub fn outcome(&self) -> WorkerPollOutcome {
5306 worker_poll_outcome(
5307 self.task.is_some(),
5308 self.poll_status.as_deref(),
5309 self.reason.as_deref(),
5310 )
5311 }
5312}
5313
5314#[derive(Clone, Debug, Deserialize)]
5315pub struct PollQueryTaskResponse {
5316 #[serde(default)]
5317 pub task: Option<QueryTask>,
5318 #[serde(default)]
5319 pub poll_status: Option<String>,
5320 #[serde(default)]
5321 pub reason: Option<String>,
5322}
5323
5324impl PollQueryTaskResponse {
5325 pub fn outcome(&self) -> WorkerPollOutcome {
5327 worker_poll_outcome(
5328 self.task.is_some(),
5329 self.poll_status.as_deref(),
5330 self.reason.as_deref(),
5331 )
5332 }
5333}
5334
5335#[derive(Clone, Debug, PartialEq, Eq)]
5337pub enum WorkerPollOutcome {
5338 Task,
5340 Idle {
5342 poll_status: Option<String>,
5343 reason: Option<String>,
5344 },
5345 Stop {
5347 poll_status: Option<String>,
5348 reason: Option<String>,
5349 },
5350}
5351
5352impl WorkerPollOutcome {
5353 pub fn should_stop(&self) -> bool {
5354 matches!(self, Self::Stop { .. })
5355 }
5356}
5357
5358fn worker_poll_outcome(
5359 has_task: bool,
5360 poll_status: Option<&str>,
5361 reason: Option<&str>,
5362) -> WorkerPollOutcome {
5363 if worker_poll_is_stop(poll_status, reason) {
5364 return WorkerPollOutcome::Stop {
5365 poll_status: poll_status.map(str::to_string),
5366 reason: reason.map(str::to_string),
5367 };
5368 }
5369
5370 if has_task {
5371 WorkerPollOutcome::Task
5372 } else {
5373 WorkerPollOutcome::Idle {
5374 poll_status: poll_status.map(str::to_string),
5375 reason: reason.map(str::to_string),
5376 }
5377 }
5378}
5379
5380#[derive(Clone, Debug, Deserialize)]
5382pub struct QueryTask {
5383 pub query_task_id: String,
5384 #[serde(default = "default_workflow_task_attempt")]
5385 pub query_task_attempt: u64,
5386 #[serde(default)]
5387 pub lease_owner: Option<String>,
5388 #[serde(default)]
5389 pub workflow_id: Option<String>,
5390 #[serde(default)]
5391 pub run_id: Option<String>,
5392 pub workflow_type: String,
5393 pub query_name: String,
5394 #[serde(
5395 default = "missing_task_payload_codec",
5396 deserialize_with = "deserialize_task_payload_codec"
5397 )]
5398 pub payload_codec: String,
5399 #[serde(default)]
5400 pub workflow_arguments: Option<Value>,
5401 #[serde(default)]
5402 pub query_arguments: Option<Value>,
5403 #[serde(default)]
5404 pub history_events: Vec<HistoryEvent>,
5405 #[serde(default)]
5406 pub history_export: Option<Value>,
5407 #[serde(default)]
5408 pub run_status: Option<String>,
5409}
5410
5411#[derive(Clone, Debug, Deserialize)]
5412pub struct WorkflowTask {
5413 pub task_id: String,
5414 #[serde(default)]
5415 pub workflow_command_id: Option<String>,
5416 #[serde(default)]
5417 pub workflow_id: Option<String>,
5418 #[serde(default)]
5419 pub run_id: Option<String>,
5420 pub workflow_type: String,
5421 #[serde(default)]
5422 pub cancel_requested: bool,
5423 #[serde(
5424 default = "missing_task_payload_codec",
5425 deserialize_with = "deserialize_task_payload_codec"
5426 )]
5427 pub payload_codec: String,
5428 #[serde(default)]
5429 pub arguments: Option<Value>,
5430 #[serde(default)]
5431 pub history_events: Vec<HistoryEvent>,
5432 #[serde(default)]
5433 pub total_history_events: Option<u64>,
5434 #[serde(default)]
5435 pub history_size_bytes: Option<u64>,
5436 #[serde(default)]
5437 pub continue_as_new_recommended: Option<bool>,
5438 #[serde(default)]
5439 pub history_budget_pressure: Option<String>,
5440 #[serde(default)]
5441 pub next_history_page_token: Option<String>,
5442 #[serde(default = "default_workflow_task_attempt")]
5443 pub workflow_task_attempt: u64,
5444 #[serde(default)]
5445 pub workflow_signal_id: Option<String>,
5446 #[serde(default)]
5447 pub signal_name: Option<String>,
5448 #[serde(default)]
5449 pub signal_arguments: Option<Value>,
5450 #[serde(default)]
5451 pub workflow_update_id: Option<String>,
5452 #[serde(default)]
5453 pub update_name: Option<String>,
5454 #[serde(default)]
5455 pub lease_owner: Option<String>,
5456}
5457
5458impl WorkflowTask {
5459 fn append_history_page(&mut self, page: WorkflowTaskHistoryPage) {
5460 self.history_events.extend(page.history_events);
5461
5462 if page.total_history_events.is_some() {
5463 self.total_history_events = page.total_history_events;
5464 }
5465
5466 self.next_history_page_token = page
5467 .next_history_page_token
5468 .filter(|token| !token.is_empty());
5469 }
5470}
5471
5472#[derive(Clone, Debug, Deserialize)]
5473struct WorkflowTaskHistoryPage {
5474 #[serde(default)]
5475 history_events: Vec<HistoryEvent>,
5476 #[serde(default)]
5477 total_history_events: Option<u64>,
5478 #[serde(default)]
5479 next_history_page_token: Option<String>,
5480}
5481
5482#[derive(Clone, Debug, Deserialize)]
5483pub struct ActivityTask {
5484 pub task_id: String,
5485 #[serde(default)]
5486 pub activity_attempt_id: Option<String>,
5487 #[serde(default)]
5488 pub attempt_id: Option<String>,
5489 pub activity_type: String,
5490 #[serde(
5491 default = "missing_task_payload_codec",
5492 deserialize_with = "deserialize_task_payload_codec"
5493 )]
5494 pub payload_codec: String,
5495 #[serde(default)]
5496 pub arguments: Option<Value>,
5497 #[serde(default = "default_attempt_number")]
5498 pub attempt_number: u64,
5499 #[serde(default)]
5500 pub lease_owner: Option<String>,
5501}
5502
5503#[derive(Clone, Debug, Deserialize)]
5504pub struct HistoryEvent {
5505 #[serde(alias = "type")]
5506 pub event_type: String,
5507 #[serde(default)]
5508 pub payload: Value,
5509 #[serde(flatten)]
5510 pub raw: HashMap<String, Value>,
5511}
5512
5513#[derive(Clone, Debug, PartialEq)]
5515pub struct QuerySignal {
5516 pub id: Option<String>,
5517 pub name: String,
5518 pub arguments: Vec<Value>,
5519 avro_arguments: Vec<AvroValue>,
5520 pub workflow_sequence: Option<u64>,
5521}
5522
5523impl QuerySignal {
5524 pub fn arguments_avro_value(&self) -> &[AvroValue] {
5526 &self.avro_arguments
5527 }
5528}
5529
5530#[derive(Clone, Debug)]
5536pub struct QueryContext {
5537 pub workflow_id: Option<String>,
5538 pub run_id: Option<String>,
5539 pub workflow_type: String,
5540 pub run_status: Option<String>,
5541 workflow_input: Value,
5542 workflow_input_avro_value: AvroValue,
5543 history_events: Arc<Vec<HistoryEvent>>,
5544 signal_events: Arc<Vec<QuerySignal>>,
5545}
5546
5547impl QueryContext {
5548 pub fn workflow_input(&self) -> &Value {
5550 &self.workflow_input
5551 }
5552
5553 pub fn workflow_input_avro_value(&self) -> &AvroValue {
5555 &self.workflow_input_avro_value
5556 }
5557
5558 pub fn history_events(&self) -> &[HistoryEvent] {
5560 self.history_events.as_slice()
5561 }
5562
5563 pub fn signal_events(&self) -> &[QuerySignal] {
5565 self.signal_events.as_slice()
5566 }
5567
5568 pub fn signals(&self, signal_name: &str) -> Vec<Vec<Value>> {
5570 self.signal_events
5571 .iter()
5572 .filter(|signal| signal.name == signal_name)
5573 .map(|signal| signal.arguments.clone())
5574 .collect()
5575 }
5576
5577 pub fn signals_avro_value(&self, signal_name: &str) -> Vec<Vec<AvroValue>> {
5579 self.signal_events
5580 .iter()
5581 .filter(|signal| signal.name == signal_name)
5582 .map(|signal| signal.avro_arguments.clone())
5583 .collect()
5584 }
5585}
5586
5587#[derive(Clone, Debug, Deserialize)]
5588pub struct ActivityHeartbeatResponse {
5589 #[serde(default)]
5590 pub cancel_requested: bool,
5591 #[serde(default)]
5592 pub heartbeat_recorded: bool,
5593 #[serde(default)]
5594 pub can_continue: Option<bool>,
5595 #[serde(default)]
5596 pub reason: Option<String>,
5597 #[serde(default)]
5598 pub run_closed_reason: Option<String>,
5599 #[serde(default)]
5600 pub run_closed_at: Option<String>,
5601 #[serde(default)]
5602 pub lease_expires_at: Option<String>,
5603 #[serde(default)]
5604 pub last_heartbeat_at: Option<String>,
5605}
5606
5607impl ActivityHeartbeatResponse {
5608 pub fn should_stop(&self) -> bool {
5610 self.cancel_requested || self.can_continue == Some(false)
5611 }
5612}
5613
5614fn missing_task_payload_codec() -> String {
5615 MISSING_TASK_PAYLOAD_CODEC.to_string()
5616}
5617
5618fn deserialize_task_payload_codec<'de, D>(deserializer: D) -> std::result::Result<String, D::Error>
5619where
5620 D: Deserializer<'de>,
5621{
5622 Ok(match Value::deserialize(deserializer)? {
5623 Value::String(codec) => codec,
5624 Value::Null => NULL_TASK_PAYLOAD_CODEC.to_string(),
5625 _ => NON_STRING_TASK_PAYLOAD_CODEC.to_string(),
5626 })
5627}
5628
5629fn default_workflow_task_attempt() -> u64 {
5630 1
5631}
5632
5633fn default_attempt_number() -> u64 {
5634 1
5635}
5636
5637type WorkflowFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5638type WorkflowHandler = Arc<dyn Fn(WorkflowContext, AvroValue) -> WorkflowFuture + Send + Sync>;
5639type ErasedWorkflowState = Arc<dyn Any + Send + Sync>;
5640type WorkflowStateSnapshot = Arc<dyn Fn() -> Result<ErasedWorkflowState> + Send + Sync>;
5641type ReplayedWorkflowHandler =
5642 Arc<dyn Fn(WorkflowContext, AvroValue) -> ReplayedWorkflowInvocation + Send + Sync>;
5643type ActivityFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5644type ActivityHandler = Arc<dyn Fn(ActivityContext, AvroValue) -> ActivityFuture + Send + Sync>;
5645type QueryFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5646type QueryHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
5647type UpdateHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
5648type ReplayedQueryHandler = Arc<
5649 dyn Fn(QueryContext, ErasedWorkflowState, AvroValue) -> std::result::Result<QueryFuture, String>
5650 + Send
5651 + Sync,
5652>;
5653type WorkerHeartbeatObserver = Arc<dyn Fn(&WorkerHeartbeatObservation) + Send + Sync>;
5654
5655struct ReplayedWorkflowInvocation {
5656 future: WorkflowFuture,
5657 snapshot: WorkflowStateSnapshot,
5658}
5659
5660#[derive(Clone)]
5661struct RegisteredWorkflow {
5662 execute: WorkflowHandler,
5663 replay: Option<ReplayedWorkflowHandler>,
5664 state_type: Option<TypeId>,
5665}
5666
5667#[derive(Debug)]
5668struct WorkflowTaskDecision {
5669 commands: Vec<Value>,
5670 message_stream_cursors: Vec<Value>,
5671 message_stream_waits: Vec<Value>,
5672}
5673
5674impl WorkflowTaskDecision {
5675 fn without_message_streams(commands: Vec<Value>) -> Self {
5676 Self {
5677 commands,
5678 message_stream_cursors: Vec::new(),
5679 message_stream_waits: Vec::new(),
5680 }
5681 }
5682}
5683
5684#[derive(Clone)]
5685enum RegisteredQuery {
5686 Snapshot(QueryHandler),
5687 Replayed {
5688 state_type: TypeId,
5689 handler: ReplayedQueryHandler,
5690 },
5691}
5692
5693#[derive(Clone, Debug)]
5694pub struct WorkerHeartbeatObservation {
5695 pub worker_id: String,
5696 pub task_queue: String,
5697 pub acknowledged_at_unix_millis: u64,
5698 pub acknowledgement: Value,
5699}
5700
5701#[derive(Clone, Copy, Debug)]
5710pub struct WorkerRetryPolicy {
5711 pub max_retries: usize,
5713 pub initial_backoff: Duration,
5715 pub max_backoff: Duration,
5717}
5718
5719impl Default for WorkerRetryPolicy {
5720 fn default() -> Self {
5721 Self {
5722 max_retries: 5,
5723 initial_backoff: Duration::from_millis(100),
5724 max_backoff: Duration::from_secs(5),
5725 }
5726 }
5727}
5728
5729#[derive(Clone, Copy, Debug, PartialEq, Eq)]
5730enum ManagedPollOutcome {
5731 Idle,
5732 Handled,
5733 Stop,
5734}
5735
5736#[derive(Clone)]
5737pub struct Worker {
5738 client: Client,
5739 worker_id: String,
5740 task_queue: String,
5741 workflows: HashMap<String, RegisteredWorkflow>,
5742 activities: HashMap<String, ActivityHandler>,
5743 queries: HashMap<String, HashMap<String, RegisteredQuery>>,
5744 updates: HashMap<String, HashMap<String, UpdateHandler>>,
5745 max_concurrent_workflow_tasks: usize,
5746 max_concurrent_activity_tasks: usize,
5747 poll_timeout: Duration,
5748 heartbeat_interval: Duration,
5749 retry_policy: WorkerRetryPolicy,
5750 heartbeat_observer: Option<WorkerHeartbeatObserver>,
5751}
5752
5753impl Worker {
5754 pub fn new(client: Client, task_queue: impl Into<String>) -> Self {
5755 Self {
5756 client,
5757 worker_id: default_worker_id(),
5758 task_queue: task_queue.into(),
5759 workflows: HashMap::new(),
5760 activities: HashMap::new(),
5761 queries: HashMap::new(),
5762 updates: HashMap::new(),
5763 max_concurrent_workflow_tasks: 10,
5764 max_concurrent_activity_tasks: 10,
5765 poll_timeout: Duration::from_secs(30),
5766 heartbeat_interval: Duration::from_secs(60),
5767 retry_policy: WorkerRetryPolicy::default(),
5768 heartbeat_observer: None,
5769 }
5770 }
5771
5772 pub fn worker_id(mut self, worker_id: impl Into<String>) -> Self {
5773 self.worker_id = worker_id.into();
5774 self
5775 }
5776
5777 pub fn poll_timeout(mut self, timeout: Duration) -> Self {
5778 self.poll_timeout = timeout;
5779 self
5780 }
5781
5782 pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
5783 self.heartbeat_interval = interval;
5784 self
5785 }
5786
5787 pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self {
5789 self.retry_policy = policy;
5790 self
5791 }
5792
5793 pub fn on_worker_heartbeat<F>(mut self, observer: F) -> Self
5794 where
5795 F: Fn(&WorkerHeartbeatObservation) + Send + Sync + 'static,
5796 {
5797 self.heartbeat_observer = Some(Arc::new(observer));
5798 self
5799 }
5800
5801 pub fn max_concurrent_workflow_tasks(mut self, count: usize) -> Self {
5802 self.max_concurrent_workflow_tasks = count.max(1);
5803 self
5804 }
5805
5806 pub fn max_concurrent_activity_tasks(mut self, count: usize) -> Self {
5807 self.max_concurrent_activity_tasks = count.max(1);
5808 self
5809 }
5810
5811 pub fn register_workflow<F, Fut>(&mut self, workflow_type: impl Into<String>, handler: F)
5818 where
5819 F: Fn(WorkflowContext, Value) -> Fut + Send + Sync + 'static,
5820 Fut: Future<Output = Result<Value>> + Send + 'static,
5821 {
5822 let handler = Arc::new(handler);
5823 self.workflows.insert(
5824 workflow_type.into(),
5825 RegisteredWorkflow {
5826 execute: Arc::new(move |ctx, input| {
5827 let handler = Arc::clone(&handler);
5828 Box::pin(async move {
5829 let result = handler(ctx, input.into_json()?).await?;
5830 AvroValue::from_serialize(&result)
5831 })
5832 }),
5833 replay: None,
5834 state_type: None,
5835 },
5836 );
5837 }
5838
5839 pub fn register_typed_workflow<I, O, F, Fut>(
5850 &mut self,
5851 workflow_type: impl Into<String>,
5852 handler: F,
5853 ) where
5854 I: DeserializeOwned + Send + 'static,
5855 O: Serialize + Send + 'static,
5856 F: Fn(WorkflowContext, I) -> Fut + Send + Sync + 'static,
5857 Fut: Future<Output = Result<O>> + Send + 'static,
5858 {
5859 let workflow_type = workflow_type.into();
5860 let handler_name = workflow_type.clone();
5861 let handler = Arc::new(handler);
5862 self.workflows.insert(
5863 workflow_type,
5864 RegisteredWorkflow {
5865 execute: Arc::new(move |ctx, input| {
5866 let handler = Arc::clone(&handler);
5867 let handler_name = handler_name.clone();
5868 Box::pin(async move {
5869 let input =
5870 decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
5871 let result = handler(ctx, input).await?;
5872 encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
5873 })
5874 }),
5875 replay: None,
5876 state_type: None,
5877 },
5878 );
5879 }
5880
5881 pub fn register_workflow_avro_value<F, Fut>(
5883 &mut self,
5884 workflow_type: impl Into<String>,
5885 handler: F,
5886 ) where
5887 F: Fn(WorkflowContext, AvroValue) -> Fut + Send + Sync + 'static,
5888 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
5889 {
5890 self.workflows.insert(
5891 workflow_type.into(),
5892 RegisteredWorkflow {
5893 execute: Arc::new(move |ctx, input| Box::pin(handler(ctx, input))),
5894 replay: None,
5895 state_type: None,
5896 },
5897 );
5898 }
5899
5900 pub fn register_replayed_workflow<S, Factory, F, Fut>(
5908 &mut self,
5909 workflow_type: impl Into<String>,
5910 state_factory: Factory,
5911 handler: F,
5912 ) where
5913 S: Clone + Send + Sync + 'static,
5914 Factory: Fn() -> S + Send + Sync + 'static,
5915 F: Fn(WorkflowContext, Value, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
5916 Fut: Future<Output = Result<Value>> + Send + 'static,
5917 {
5918 let state_factory = Arc::new(state_factory);
5919 let handler = Arc::new(handler);
5920
5921 let execute_factory = Arc::clone(&state_factory);
5922 let execute_handler = Arc::clone(&handler);
5923 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
5924 let state = WorkflowInstance::new(execute_factory());
5925 let handler = Arc::clone(&execute_handler);
5926 Box::pin(async move {
5927 let result = handler(ctx, input.into_json()?, state).await?;
5928 AvroValue::from_serialize(&result)
5929 }) as WorkflowFuture
5930 });
5931
5932 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
5933 let state = WorkflowInstance::new(state_factory());
5934 let snapshot_state = state.clone();
5935 let snapshot: WorkflowStateSnapshot =
5936 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
5937 let replay_handler = Arc::clone(&handler);
5938 let future = async move {
5939 let result = replay_handler(ctx, input.into_json()?, state).await?;
5940 AvroValue::from_serialize(&result)
5941 };
5942 ReplayedWorkflowInvocation {
5943 future: Box::pin(future),
5944 snapshot,
5945 }
5946 });
5947
5948 self.workflows.insert(
5949 workflow_type.into(),
5950 RegisteredWorkflow {
5951 execute,
5952 replay: Some(replay),
5953 state_type: Some(TypeId::of::<S>()),
5954 },
5955 );
5956 }
5957
5958 pub fn register_typed_replayed_workflow<I, O, S, Factory, F, Fut>(
5964 &mut self,
5965 workflow_type: impl Into<String>,
5966 state_factory: Factory,
5967 handler: F,
5968 ) where
5969 I: DeserializeOwned + Send + 'static,
5970 O: Serialize + Send + 'static,
5971 S: Clone + Send + Sync + 'static,
5972 Factory: Fn() -> S + Send + Sync + 'static,
5973 F: Fn(WorkflowContext, I, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
5974 Fut: Future<Output = Result<O>> + Send + 'static,
5975 {
5976 let workflow_type = workflow_type.into();
5977 let state_factory = Arc::new(state_factory);
5978 let handler = Arc::new(handler);
5979
5980 let execute_name = workflow_type.clone();
5981 let execute_factory = Arc::clone(&state_factory);
5982 let execute_handler = Arc::clone(&handler);
5983 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
5984 let state = WorkflowInstance::new(execute_factory());
5985 let handler = Arc::clone(&execute_handler);
5986 let handler_name = execute_name.clone();
5987 Box::pin(async move {
5988 let input = decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
5989 let result = handler(ctx, input, state).await?;
5990 encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
5991 }) as WorkflowFuture
5992 });
5993
5994 let replay_name = workflow_type.clone();
5995 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
5996 let state = WorkflowInstance::new(state_factory());
5997 let snapshot_state = state.clone();
5998 let snapshot: WorkflowStateSnapshot =
5999 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6000 let handler = Arc::clone(&handler);
6001 let handler_name = replay_name.clone();
6002 let future = async move {
6003 let input = decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
6004 let result = handler(ctx, input, state).await?;
6005 encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
6006 };
6007 ReplayedWorkflowInvocation {
6008 future: Box::pin(future),
6009 snapshot,
6010 }
6011 });
6012
6013 self.workflows.insert(
6014 workflow_type,
6015 RegisteredWorkflow {
6016 execute,
6017 replay: Some(replay),
6018 state_type: Some(TypeId::of::<S>()),
6019 },
6020 );
6021 }
6022
6023 pub fn register_replayed_workflow_avro_value<S, Factory, F, Fut>(
6025 &mut self,
6026 workflow_type: impl Into<String>,
6027 state_factory: Factory,
6028 handler: F,
6029 ) where
6030 S: Clone + Send + Sync + 'static,
6031 Factory: Fn() -> S + Send + Sync + 'static,
6032 F: Fn(WorkflowContext, AvroValue, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
6033 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6034 {
6035 let state_factory = Arc::new(state_factory);
6036 let handler = Arc::new(handler);
6037
6038 let execute_factory = Arc::clone(&state_factory);
6039 let execute_handler = Arc::clone(&handler);
6040 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6041 let state = WorkflowInstance::new(execute_factory());
6042 Box::pin(execute_handler(ctx, input, state)) as WorkflowFuture
6043 });
6044
6045 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6046 let state = WorkflowInstance::new(state_factory());
6047 let snapshot_state = state.clone();
6048 let snapshot: WorkflowStateSnapshot =
6049 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6050 ReplayedWorkflowInvocation {
6051 future: Box::pin(handler(ctx, input, state)),
6052 snapshot,
6053 }
6054 });
6055
6056 self.workflows.insert(
6057 workflow_type.into(),
6058 RegisteredWorkflow {
6059 execute,
6060 replay: Some(replay),
6061 state_type: Some(TypeId::of::<S>()),
6062 },
6063 );
6064 }
6065
6066 pub fn register_activity<F, Fut>(&mut self, activity_type: impl Into<String>, handler: F)
6067 where
6068 F: Fn(ActivityContext, Value) -> Fut + Send + Sync + 'static,
6069 Fut: Future<Output = Result<Value>> + Send + 'static,
6070 {
6071 let handler = Arc::new(handler);
6072 self.activities.insert(
6073 activity_type.into(),
6074 Arc::new(move |ctx, args| {
6075 let handler = Arc::clone(&handler);
6076 Box::pin(async move {
6077 let result = handler(ctx, args.into_json()?).await?;
6078 AvroValue::from_serialize(&result)
6079 })
6080 }),
6081 );
6082 }
6083
6084 pub fn register_typed_activity<I, O, F, Fut>(
6090 &mut self,
6091 activity_type: impl Into<String>,
6092 handler: F,
6093 ) where
6094 I: DeserializeOwned + Send + 'static,
6095 O: Serialize + Send + 'static,
6096 F: Fn(ActivityContext, I) -> Fut + Send + Sync + 'static,
6097 Fut: Future<Output = Result<O>> + Send + 'static,
6098 {
6099 let activity_type = activity_type.into();
6100 let handler_name = activity_type.clone();
6101 let handler = Arc::new(handler);
6102 self.activities.insert(
6103 activity_type,
6104 Arc::new(move |ctx, input| {
6105 let handler = Arc::clone(&handler);
6106 let handler_name = handler_name.clone();
6107 Box::pin(async move {
6108 let input =
6109 decode_handler_input::<I>(input, HandlerKind::Activity, &handler_name)?;
6110 let result = handler(ctx, input).await?;
6111 encode_handler_result(&result, HandlerKind::Activity, &handler_name)
6112 })
6113 }),
6114 );
6115 }
6116
6117 pub fn register_activity_avro_value<F, Fut>(
6119 &mut self,
6120 activity_type: impl Into<String>,
6121 handler: F,
6122 ) where
6123 F: Fn(ActivityContext, AvroValue) -> Fut + Send + Sync + 'static,
6124 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6125 {
6126 self.activities.insert(
6127 activity_type.into(),
6128 Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
6129 );
6130 }
6131
6132 pub fn register_query<F, Fut>(
6138 &mut self,
6139 workflow_type: impl Into<String>,
6140 query_name: impl Into<String>,
6141 handler: F,
6142 ) where
6143 F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
6144 Fut: Future<Output = Result<Value>> + Send + 'static,
6145 {
6146 let handler = Arc::new(handler);
6147 self.queries
6148 .entry(workflow_type.into())
6149 .or_default()
6150 .insert(
6151 query_name.into(),
6152 RegisteredQuery::Snapshot(Arc::new(move |ctx, args| {
6153 let handler = Arc::clone(&handler);
6154 Box::pin(async move {
6155 let result = handler(ctx, args.into_json()?).await?;
6156 AvroValue::from_serialize(&result)
6157 })
6158 })),
6159 );
6160 }
6161
6162 pub fn register_query_avro_value<F, Fut>(
6164 &mut self,
6165 workflow_type: impl Into<String>,
6166 query_name: impl Into<String>,
6167 handler: F,
6168 ) where
6169 F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
6170 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6171 {
6172 self.queries
6173 .entry(workflow_type.into())
6174 .or_default()
6175 .insert(
6176 query_name.into(),
6177 RegisteredQuery::Snapshot(Arc::new(move |ctx, args| Box::pin(handler(ctx, args)))),
6178 );
6179 }
6180
6181 pub fn register_replayed_query<S, F, Fut>(
6188 &mut self,
6189 workflow_type: impl Into<String>,
6190 query_name: impl Into<String>,
6191 handler: F,
6192 ) where
6193 S: Clone + Send + Sync + 'static,
6194 F: Fn(QueryContext, Arc<S>, Value) -> Fut + Send + Sync + 'static,
6195 Fut: Future<Output = Result<Value>> + Send + 'static,
6196 {
6197 let handler = Arc::new(handler);
6198 let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
6199 let state = state.downcast::<S>().map_err(|_| {
6200 "registered query state type does not match the replayed workflow state".to_string()
6201 })?;
6202 let handler = Arc::clone(&handler);
6203 Ok(Box::pin(async move {
6204 let result = handler(ctx, state, args.into_json()?).await?;
6205 AvroValue::from_serialize(&result)
6206 }))
6207 });
6208
6209 self.queries
6210 .entry(workflow_type.into())
6211 .or_default()
6212 .insert(
6213 query_name.into(),
6214 RegisteredQuery::Replayed {
6215 state_type: TypeId::of::<S>(),
6216 handler: erased_handler,
6217 },
6218 );
6219 }
6220
6221 pub fn register_replayed_query_avro_value<S, F, Fut>(
6223 &mut self,
6224 workflow_type: impl Into<String>,
6225 query_name: impl Into<String>,
6226 handler: F,
6227 ) where
6228 S: Clone + Send + Sync + 'static,
6229 F: Fn(QueryContext, Arc<S>, AvroValue) -> Fut + Send + Sync + 'static,
6230 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6231 {
6232 let handler = Arc::new(handler);
6233 let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
6234 let state = state.downcast::<S>().map_err(|_| {
6235 "registered query state type does not match the replayed workflow state".to_string()
6236 })?;
6237 Ok(Box::pin(handler(ctx, state, args)))
6238 });
6239
6240 self.queries
6241 .entry(workflow_type.into())
6242 .or_default()
6243 .insert(
6244 query_name.into(),
6245 RegisteredQuery::Replayed {
6246 state_type: TypeId::of::<S>(),
6247 handler: erased_handler,
6248 },
6249 );
6250 }
6251
6252 pub fn register_update<F, Fut>(
6254 &mut self,
6255 workflow_type: impl Into<String>,
6256 update_name: impl Into<String>,
6257 handler: F,
6258 ) where
6259 F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
6260 Fut: Future<Output = Result<Value>> + Send + 'static,
6261 {
6262 let handler = Arc::new(handler);
6263 self.updates
6264 .entry(workflow_type.into())
6265 .or_default()
6266 .insert(
6267 update_name.into(),
6268 Arc::new(move |ctx, args| {
6269 let handler = Arc::clone(&handler);
6270 Box::pin(async move {
6271 let result = handler(ctx, args.into_json()?).await?;
6272 AvroValue::from_serialize(&result)
6273 })
6274 }),
6275 );
6276 }
6277
6278 pub fn register_update_avro_value<F, Fut>(
6280 &mut self,
6281 workflow_type: impl Into<String>,
6282 update_name: impl Into<String>,
6283 handler: F,
6284 ) where
6285 F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
6286 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6287 {
6288 self.updates
6289 .entry(workflow_type.into())
6290 .or_default()
6291 .insert(
6292 update_name.into(),
6293 Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
6294 );
6295 }
6296
6297 pub async fn register(&self) -> Result<RegisterWorkerResponse> {
6298 let mut command_contracts = serde_json::Map::new();
6299 for workflow_type in self.workflows.keys() {
6300 let mut queries = self
6301 .queries
6302 .get(workflow_type)
6303 .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
6304 .unwrap_or_default();
6305 queries.sort();
6306 let mut updates = self
6307 .updates
6308 .get(workflow_type)
6309 .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
6310 .unwrap_or_default();
6311 updates.sort();
6312 command_contracts.insert(
6313 workflow_type.clone(),
6314 json!({
6315 "queries": queries,
6316 "query_contracts": [],
6317 "signals": [],
6318 "signal_contracts": [],
6319 "updates": updates,
6320 "update_contracts": [],
6321 "update_validators": [],
6322 }),
6323 );
6324 }
6325
6326 self.client
6327 .register_worker_with_command_contracts(
6328 &self.worker_id,
6329 &self.task_queue,
6330 self.workflows.keys().cloned().collect(),
6331 self.activities.keys().cloned().collect(),
6332 self.max_concurrent_workflow_tasks,
6333 self.max_concurrent_activity_tasks,
6334 [
6335 Some(CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY.to_string()),
6336 Some(DURABLE_SELECTION_CAPABILITY.to_string()),
6337 Some(MEMO_UPSERTS_CAPABILITY.to_string()),
6338 Some(TYPED_SEARCH_ATTRIBUTES_CAPABILITY.to_string()),
6339 (!self.queries.is_empty()).then(|| QUERY_TASKS_CAPABILITY.to_string()),
6340 (!self.updates.is_empty()).then(|| WORKFLOW_UPDATES_CAPABILITY.to_string()),
6341 worker_protocol_supports_message_streams(WORKER_PROTOCOL_VERSION)
6342 .then(|| MESSAGE_STREAMS_CAPABILITY.to_string()),
6343 ]
6344 .into_iter()
6345 .flatten()
6346 .collect(),
6347 Value::Object(command_contracts),
6348 )
6349 .await
6350 }
6351
6352 pub async fn run(&self) -> Result<()> {
6358 self.run_until(std::future::pending::<()>()).await
6359 }
6360
6361 pub async fn run_until<F>(&self, shutdown: F) -> Result<()>
6365 where
6366 F: Future<Output = ()>,
6367 {
6368 let registration = self.register().await?;
6369 if !registration.registered {
6370 return Err(Error::WorkerLoop(format!(
6371 "worker registration for {:?} was not accepted",
6372 self.worker_id
6373 )));
6374 }
6375 let registered_worker_id = registration.worker_id.clone();
6376 let primary = self.run_registered_until(shutdown, registration).await;
6377 let deregistration = self
6378 .client
6379 .deregister_worker_registration(®istered_worker_id)
6380 .await;
6381
6382 match (primary, deregistration) {
6383 (Ok(()), Ok(_)) => Ok(()),
6384 (Ok(()), Err(deregistration)) => Err(deregistration),
6385 (Err(primary), Ok(_)) => Err(primary),
6386 (Err(primary), Err(deregistration)) => Err(Error::WorkerShutdown {
6387 primary: Box::new(primary),
6388 deregistration: Box::new(deregistration),
6389 }),
6390 }
6391 }
6392
6393 async fn run_registered_until<F>(
6394 &self,
6395 shutdown: F,
6396 registration: RegisterWorkerResponse,
6397 ) -> Result<()>
6398 where
6399 F: Future<Output = ()>,
6400 {
6401 let heartbeat_interval = Duration::from_secs(
6402 registration
6403 .heartbeat_interval_seconds
6404 .unwrap_or(self.heartbeat_interval.as_secs().max(1)),
6405 );
6406 let heartbeat = tokio::time::sleep(Duration::ZERO);
6412 tokio::pin!(heartbeat);
6413 tokio::pin!(shutdown);
6414 let stop = Arc::new(AtomicBool::new(false));
6415 let mut workflow_poller = (!self.workflows.is_empty()).then(|| {
6419 let worker = self.clone();
6420 let stop = Arc::clone(&stop);
6421 tokio::spawn(async move { worker.poll_workflows_until_stopped(stop).await })
6422 });
6423 let mut activity_poller = (!self.activities.is_empty()).then(|| {
6424 let worker = self.clone();
6425 let stop = Arc::clone(&stop);
6426 tokio::spawn(async move { worker.poll_activities_until_stopped(stop).await })
6427 });
6428 let mut query_poller = (!self.queries.is_empty()).then(|| {
6429 let worker = self.clone();
6430 let stop = Arc::clone(&stop);
6431 tokio::spawn(async move { worker.poll_queries_until_stopped(stop).await })
6432 });
6433
6434 loop {
6435 tokio::select! {
6436 _ = &mut shutdown => {
6437 stop.store(true, Ordering::SeqCst);
6438 break;
6439 }
6440 _ = &mut heartbeat => {
6441 let result = self.retry_worker_operation(|| {
6442 self.client.heartbeat_worker(
6443 &self.worker_id,
6444 self.max_concurrent_workflow_tasks,
6445 self.max_concurrent_activity_tasks,
6446 )
6447 }).await;
6448 heartbeat
6449 .as_mut()
6450 .reset(tokio::time::Instant::now() + heartbeat_interval);
6451 match result {
6452 Ok(acknowledgement) => {
6453 if let Some(observer) = &self.heartbeat_observer {
6454 observer(&WorkerHeartbeatObservation {
6455 worker_id: self.worker_id.clone(),
6456 task_queue: self.task_queue.clone(),
6457 acknowledged_at_unix_millis: SystemTime::now()
6458 .duration_since(UNIX_EPOCH)
6459 .unwrap_or_default()
6460 .as_millis()
6461 .min(u64::MAX as u128)
6462 as u64,
6463 acknowledgement,
6464 });
6465 }
6466 }
6467 Err(error) => {
6468 stop.store(true, Ordering::SeqCst);
6469 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await?;
6470 return Err(error);
6471 }
6472 }
6473 }
6474 result = OptionFuture::from(workflow_poller.as_mut()), if workflow_poller.is_some() => {
6475 workflow_poller = None;
6476 let stopped_by_server = stop.load(Ordering::SeqCst);
6477 stop.store(true, Ordering::SeqCst);
6478 let poller_result = optional_poller_result("workflow", result);
6479 let join_result =
6480 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6481 poller_result?;
6482 join_result?;
6483 if stopped_by_server {
6484 return Ok(());
6485 }
6486 return Err(Error::WorkerLoop(
6487 "workflow poller stopped unexpectedly".to_string(),
6488 ));
6489 }
6490 result = OptionFuture::from(activity_poller.as_mut()), if activity_poller.is_some() => {
6491 activity_poller = None;
6492 let stopped_by_server = stop.load(Ordering::SeqCst);
6493 stop.store(true, Ordering::SeqCst);
6494 let poller_result = optional_poller_result("activity", result);
6495 let join_result =
6496 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6497 poller_result?;
6498 join_result?;
6499 if stopped_by_server {
6500 return Ok(());
6501 }
6502 return Err(Error::WorkerLoop(
6503 "activity poller stopped unexpectedly".to_string(),
6504 ));
6505 }
6506 result = OptionFuture::from(query_poller.as_mut()), if query_poller.is_some() => {
6507 query_poller = None;
6508 let stopped_by_server = stop.load(Ordering::SeqCst);
6509 stop.store(true, Ordering::SeqCst);
6510 let poller_result = optional_poller_result("query", result);
6511 let join_result =
6512 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6513 poller_result?;
6514 join_result?;
6515 if stopped_by_server {
6516 return Ok(());
6517 }
6518 return Err(Error::WorkerLoop(
6519 "query poller stopped unexpectedly".to_string(),
6520 ));
6521 }
6522 }
6523 }
6524
6525 join_pollers(
6526 workflow_poller.take(),
6527 activity_poller.take(),
6528 query_poller.take(),
6529 )
6530 .await
6531 }
6532
6533 pub async fn run_once(&self) -> Result<usize> {
6547 let mut handled = 0;
6548 match self.poll_workflow_once().await? {
6549 ManagedPollOutcome::Handled => handled += 1,
6550 ManagedPollOutcome::Stop => return Ok(handled),
6551 ManagedPollOutcome::Idle => {}
6552 }
6553 match self.poll_activity_once().await? {
6554 ManagedPollOutcome::Handled => handled += 1,
6555 ManagedPollOutcome::Stop => return Ok(handled),
6556 ManagedPollOutcome::Idle => {}
6557 }
6558 if !self.queries.is_empty() {
6559 match self.poll_query_once().await? {
6560 ManagedPollOutcome::Handled => handled += 1,
6561 ManagedPollOutcome::Stop => return Ok(handled),
6562 ManagedPollOutcome::Idle => {}
6563 }
6564 }
6565 Ok(handled)
6566 }
6567
6568 async fn poll_workflow_once(&self) -> Result<ManagedPollOutcome> {
6569 let poll_request_id = unique_request_id("rust-workflow-poll");
6570 let response = self
6571 .retry_worker_operation(|| {
6572 self.client.poll_workflow_task_response_with_request_id(
6573 &self.worker_id,
6574 &self.task_queue,
6575 self.poll_timeout,
6576 &poll_request_id,
6577 0,
6578 )
6579 })
6580 .await;
6581 let Some(response) = self.settle_worker_poll_response(response).await? else {
6582 return Ok(ManagedPollOutcome::Idle);
6583 };
6584 if response.outcome().should_stop() {
6585 return Ok(ManagedPollOutcome::Stop);
6586 }
6587 let memo_updates_supported =
6588 runtime_supports_workflow_memo_updates(response.server_capabilities.as_ref());
6589 let Some(task) = response.task else {
6590 return Ok(ManagedPollOutcome::Idle);
6591 };
6592
6593 let task_id = task.task_id.clone();
6594 let attempt = task.workflow_task_attempt;
6595 let run_id = task.run_id.clone();
6596 let lease_owner = task
6597 .lease_owner
6598 .clone()
6599 .unwrap_or_else(|| self.worker_id.clone());
6600
6601 match self.execute_workflow_task_decision(task) {
6602 Ok(decision)
6603 if commands_use_workflow_memo_updates(&decision.commands)
6604 && !memo_updates_supported =>
6605 {
6606 self.client
6607 .fail_workflow_task(
6608 &task_id,
6609 &lease_owner,
6610 attempt,
6611 Error::WorkflowMemoUpdatesUnavailable.to_string(),
6612 )
6613 .await?;
6614 }
6615 Ok(decision) if decision.commands.is_empty() => {
6616 self.client
6622 .fail_workflow_task_with_type(
6623 &task_id,
6624 &lease_owner,
6625 attempt,
6626 WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE,
6627 WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE,
6628 )
6629 .await?;
6630 }
6631 Ok(decision) => {
6632 let completion = self
6633 .client
6634 .complete_workflow_task_with_message_streams(
6635 &task_id,
6636 &lease_owner,
6637 attempt,
6638 decision.commands,
6639 decision.message_stream_cursors,
6640 decision.message_stream_waits,
6641 )
6642 .await;
6643 if let Err(error) = completion {
6644 if !workflow_task_completion_is_terminal_timeout(
6645 &error,
6646 &task_id,
6647 attempt,
6648 run_id.as_deref(),
6649 ) {
6650 return Err(error);
6651 }
6652 }
6653 }
6654 Err(error) => {
6655 self.client
6656 .fail_workflow_task(&task_id, &lease_owner, attempt, error.to_string())
6657 .await?;
6658 }
6659 }
6660
6661 Ok(ManagedPollOutcome::Handled)
6662 }
6663
6664 async fn poll_workflows_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
6665 while !stop.load(Ordering::SeqCst) {
6666 if self.poll_workflow_once().await? == ManagedPollOutcome::Stop {
6667 stop.store(true, Ordering::SeqCst);
6668 break;
6669 }
6670 }
6671
6672 Ok(())
6673 }
6674
6675 async fn poll_activity_once(&self) -> Result<ManagedPollOutcome> {
6676 let poll_request_id = unique_request_id("rust-activity-poll");
6677 let response = self
6678 .retry_worker_operation(|| {
6679 self.client.poll_activity_task_response_with_request_id(
6680 &self.worker_id,
6681 &self.task_queue,
6682 self.poll_timeout,
6683 &poll_request_id,
6684 0,
6685 )
6686 })
6687 .await;
6688 let Some(response) = self.settle_worker_poll_response(response).await? else {
6689 return Ok(ManagedPollOutcome::Idle);
6690 };
6691 if response.outcome().should_stop() {
6692 return Ok(ManagedPollOutcome::Stop);
6693 }
6694 let Some(task) = response.task else {
6695 return Ok(ManagedPollOutcome::Idle);
6696 };
6697
6698 let task_id = task.task_id.clone();
6699 let attempt_id = task
6700 .activity_attempt_id
6701 .clone()
6702 .or(task.attempt_id.clone())
6703 .unwrap_or_default();
6704 let lease_owner = task
6705 .lease_owner
6706 .clone()
6707 .unwrap_or_else(|| self.worker_id.clone());
6708 let codec = task.payload_codec.clone();
6709 let result = self.execute_activity_task(task).await;
6710 match result {
6711 Ok(value) => {
6712 let completion = self
6713 .client
6714 .complete_activity_task(&task_id, &attempt_id, &lease_owner, value, &codec)
6715 .await;
6716 if let Err(error) = completion {
6717 if !activity_task_rejection_is_final(&error) {
6718 return Err(error);
6719 }
6720 }
6721 }
6722 Err(error) => {
6723 let failure = self
6724 .client
6725 .fail_activity_task(
6726 &task_id,
6727 &attempt_id,
6728 &lease_owner,
6729 error.to_string(),
6730 false,
6731 )
6732 .await;
6733 if let Err(error) = failure {
6734 if !activity_task_rejection_is_final(&error) {
6735 return Err(error);
6736 }
6737 }
6738 }
6739 }
6740
6741 Ok(ManagedPollOutcome::Handled)
6742 }
6743
6744 async fn poll_activities_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
6745 while !stop.load(Ordering::SeqCst) {
6746 if self.poll_activity_once().await? == ManagedPollOutcome::Stop {
6747 stop.store(true, Ordering::SeqCst);
6748 break;
6749 }
6750 }
6751
6752 Ok(())
6753 }
6754
6755 async fn poll_query_once(&self) -> Result<ManagedPollOutcome> {
6756 let poll_request_id = unique_request_id("rust-query-poll");
6757 let response = self
6758 .retry_worker_operation(|| {
6759 self.client.poll_query_task_response_with_request_id(
6760 &self.worker_id,
6761 &self.task_queue,
6762 self.poll_timeout,
6763 &poll_request_id,
6764 0,
6765 )
6766 })
6767 .await;
6768 let Some(response) = self.settle_worker_poll_response(response).await? else {
6769 return Ok(ManagedPollOutcome::Idle);
6770 };
6771 if response.outcome().should_stop() {
6772 return Ok(ManagedPollOutcome::Stop);
6773 }
6774 let Some(task) = response.task else {
6775 return Ok(ManagedPollOutcome::Idle);
6776 };
6777
6778 let query_task_id = task.query_task_id.clone();
6779 let attempt = task.query_task_attempt;
6780 let lease_owner = task
6781 .lease_owner
6782 .clone()
6783 .unwrap_or_else(|| self.worker_id.clone());
6784 let codec = task.payload_codec.clone();
6785
6786 match self.execute_query_task(task).await {
6787 Ok(value) => {
6788 let result_envelope = match encode_typed_envelope(&value, &codec) {
6789 Ok(result_envelope) => result_envelope,
6790 Err(error) => {
6791 let failure = self
6792 .client
6793 .fail_query_task(
6794 &query_task_id,
6795 &lease_owner,
6796 attempt,
6797 error.to_string(),
6798 "query_result_encode_failed",
6799 "QueryResultEncodeFailed",
6800 )
6801 .await;
6802 if let Err(error) = failure {
6803 if !query_task_rejection_is_final(&error) {
6804 return Err(error);
6805 }
6806 }
6807 return Ok(ManagedPollOutcome::Handled);
6808 }
6809 };
6810
6811 if let Err(error) = self
6812 .client
6813 .complete_query_task_with_envelope(
6814 &query_task_id,
6815 &lease_owner,
6816 attempt,
6817 value.clone().into_json()?,
6818 result_envelope,
6819 )
6820 .await
6821 {
6822 if !query_task_rejection_is_final(&error) {
6823 return Err(error);
6824 }
6825 }
6826 }
6827 Err(failure) => {
6828 let result = self
6829 .client
6830 .fail_query_task(
6831 &query_task_id,
6832 &lease_owner,
6833 attempt,
6834 failure.message,
6835 failure.reason,
6836 failure.failure_type,
6837 )
6838 .await;
6839 if let Err(error) = result {
6840 if !query_task_rejection_is_final(&error) {
6841 return Err(error);
6842 }
6843 }
6844 }
6845 }
6846
6847 Ok(ManagedPollOutcome::Handled)
6848 }
6849
6850 async fn poll_queries_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
6851 while !stop.load(Ordering::SeqCst) {
6852 if self.poll_query_once().await? == ManagedPollOutcome::Stop {
6853 stop.store(true, Ordering::SeqCst);
6854 break;
6855 }
6856 }
6857
6858 Ok(())
6859 }
6860
6861 async fn retry_worker_operation<T, F, Fut>(&self, mut operation: F) -> Result<T>
6862 where
6863 F: FnMut() -> Fut,
6864 Fut: Future<Output = Result<T>>,
6865 {
6866 let mut retries = 0;
6867
6868 loop {
6869 match operation().await {
6870 Err(error)
6871 if worker_operation_is_retryable(&error)
6872 && retries < self.retry_policy.max_retries =>
6873 {
6874 retries += 1;
6875 tokio::time::sleep(worker_retry_delay(self.retry_policy, retries)).await;
6876 }
6877 result => return result,
6878 }
6879 }
6880 }
6881
6882 async fn settle_worker_poll_response<T>(&self, response: Result<T>) -> Result<Option<T>> {
6883 match response {
6884 Ok(response) => Ok(Some(response)),
6885 Err(error) => {
6886 let Some(advertised_delay) = worker_poll_capacity_retry_after(&error) else {
6887 return Err(error);
6888 };
6889 let minimum_delay = self
6890 .retry_policy
6891 .initial_backoff
6892 .max(Duration::from_millis(1));
6893 let maximum_delay = self.retry_policy.max_backoff.max(minimum_delay);
6894 tokio::time::sleep(advertised_delay.max(minimum_delay).min(maximum_delay)).await;
6895 Ok(None)
6896 }
6897 }
6898 }
6899
6900 async fn execute_query_task(
6901 &self,
6902 mut task: QueryTask,
6903 ) -> std::result::Result<AvroValue, QueryTaskExecutionFailure> {
6904 validate_query_task_payloads(&task).map_err(|error| {
6905 QueryTaskExecutionFailure::new(
6906 "query_payload_decode_failed",
6907 error.to_string(),
6908 "QueryPayloadDecodeFailed",
6909 )
6910 })?;
6911
6912 if !self.workflows.contains_key(&task.workflow_type) {
6913 return Err(QueryTaskExecutionFailure::new(
6914 "query_workflow_type_not_registered",
6915 format!("no workflow registered for type {:?}", task.workflow_type),
6916 "WorkflowTypeNotRegistered",
6917 ));
6918 }
6919
6920 let Some(handlers) = self.queries.get(&task.workflow_type) else {
6921 return Err(QueryTaskExecutionFailure::new(
6922 "query_handler_unavailable",
6923 format!(
6924 "query handlers are unavailable for workflow type {:?}",
6925 task.workflow_type
6926 ),
6927 "QueryHandlerUnavailable",
6928 ));
6929 };
6930 let Some(query) = handlers.get(&task.query_name) else {
6931 return Err(QueryTaskExecutionFailure::new(
6932 "rejected_unknown_query",
6933 format!("unknown query {:?}", task.query_name),
6934 "QueryFailed",
6935 ));
6936 };
6937
6938 let args = decode_task_avro_arguments(task.query_arguments.as_ref(), &task.payload_codec)
6939 .map_err(|error| {
6940 QueryTaskExecutionFailure::new(
6941 "query_payload_decode_failed",
6942 format!("cannot decode query arguments: {error}"),
6943 "QueryPayloadDecodeFailed",
6944 )
6945 })?;
6946 let workflow_input_typed =
6947 decode_task_avro_arguments(task.workflow_arguments.as_ref(), &task.payload_codec)
6948 .map_err(|error| {
6949 QueryTaskExecutionFailure::new(
6950 "query_workflow_state_unavailable",
6951 format!("cannot decode workflow start input: {error}"),
6952 "QueryWorkflowStateUnavailable",
6953 )
6954 })?;
6955 let workflow_input = workflow_input_typed.clone().into_json().map_err(|error| {
6956 QueryTaskExecutionFailure::new(
6957 "query_workflow_state_unavailable",
6958 format!("cannot project workflow start input: {error}"),
6959 "QueryWorkflowStateUnavailable",
6960 )
6961 })?;
6962 hydrate_query_history_from_export(&mut task).map_err(|error| {
6963 QueryTaskExecutionFailure::new(
6964 "query_workflow_state_unavailable",
6965 format!("cannot restore query history snapshot: {error}"),
6966 "QueryWorkflowStateUnavailable",
6967 )
6968 })?;
6969 enrich_query_history_from_export(&mut task).map_err(|error| {
6970 QueryTaskExecutionFailure::new(
6971 "query_workflow_state_unavailable",
6972 format!("cannot restore compact query history payloads: {error}"),
6973 "QueryWorkflowStateUnavailable",
6974 )
6975 })?;
6976 let signal_events = query_signal_events(&task).map_err(|error| {
6977 QueryTaskExecutionFailure::new(
6978 "query_workflow_state_unavailable",
6979 format!("cannot decode committed workflow signals: {error}"),
6980 "QueryWorkflowStateUnavailable",
6981 )
6982 })?;
6983 let history_events = Arc::new(std::mem::take(&mut task.history_events));
6984 let context = QueryContext {
6985 workflow_id: task.workflow_id,
6986 run_id: task.run_id,
6987 workflow_type: task.workflow_type.clone(),
6988 run_status: task.run_status,
6989 workflow_input,
6990 workflow_input_avro_value: workflow_input_typed.clone(),
6991 history_events: Arc::clone(&history_events),
6992 signal_events: Arc::new(signal_events),
6993 };
6994
6995 let future = match query {
6996 RegisteredQuery::Snapshot(handler) => handler(context, args),
6997 RegisteredQuery::Replayed {
6998 state_type,
6999 handler,
7000 } => {
7001 let workflow = self
7002 .workflows
7003 .get(&task.workflow_type)
7004 .expect("workflow registration was checked above");
7005 if workflow.state_type != Some(*state_type) {
7006 return Err(QueryTaskExecutionFailure::new(
7007 "query_workflow_state_unavailable",
7008 "replayed query state type does not match its workflow registration",
7009 "QueryWorkflowStateUnavailable",
7010 ));
7011 }
7012 let replay = workflow.replay.as_ref().ok_or_else(|| {
7013 QueryTaskExecutionFailure::new(
7014 "query_workflow_state_unavailable",
7015 format!(
7016 "workflow type {:?} is not registered for instance-state replay",
7017 task.workflow_type
7018 ),
7019 "QueryWorkflowStateUnavailable",
7020 )
7021 })?;
7022 let workflow_state = Arc::new(Mutex::new(
7023 WorkflowState::new_with_identity(
7024 history_events.as_ref().clone(),
7025 context.workflow_id.clone(),
7026 context.run_id.clone(),
7027 self.task_queue.clone(),
7028 task.payload_codec,
7029 None,
7030 )
7031 .map_err(|error| {
7032 QueryTaskExecutionFailure::new(
7033 "query_workflow_state_unavailable",
7034 format!("workflow replay failed before query: {error}"),
7035 "QueryWorkflowStateUnavailable",
7036 )
7037 })?,
7038 ));
7039 let workflow_context = WorkflowContext {
7040 state: workflow_state,
7041 };
7042 let mut invocation = replay(workflow_context.clone(), workflow_input_typed.clone());
7043 let mut cx = TaskContext::from_waker(noop_waker_ref());
7044 match invocation.future.as_mut().poll(&mut cx) {
7045 Poll::Ready(Ok(_)) => {
7046 workflow_context
7047 .ensure_history_consumed()
7048 .map_err(|error| {
7049 QueryTaskExecutionFailure::new(
7050 "query_workflow_state_unavailable",
7051 format!("workflow replay failed before query: {error}"),
7052 "QueryWorkflowStateUnavailable",
7053 )
7054 })?;
7055 }
7056 Poll::Ready(Err(error)) => {
7057 return Err(QueryTaskExecutionFailure::new(
7058 "query_workflow_state_unavailable",
7059 format!("workflow replay failed before query: {error}"),
7060 "QueryWorkflowStateUnavailable",
7061 ));
7062 }
7063 Poll::Pending => {
7064 let commands = workflow_context.take_commands().map_err(|error| {
7065 QueryTaskExecutionFailure::new(
7066 "query_workflow_state_unavailable",
7067 format!("workflow replay failed before query: {error}"),
7068 "QueryWorkflowStateUnavailable",
7069 )
7070 })?;
7071 if commands.is_empty()
7072 && !workflow_context
7073 .matched_recorded_pending()
7074 .map_err(|error| {
7075 QueryTaskExecutionFailure::new(
7076 "query_workflow_state_unavailable",
7077 format!("workflow replay failed before query: {error}"),
7078 "QueryWorkflowStateUnavailable",
7079 )
7080 })?
7081 {
7082 return Err(QueryTaskExecutionFailure::new(
7083 "query_workflow_state_unavailable",
7084 "workflow replay yielded without a durable command",
7085 "QueryWorkflowStateUnavailable",
7086 ));
7087 }
7088 }
7089 }
7090 let state = (invocation.snapshot)().map_err(|error| {
7091 QueryTaskExecutionFailure::new(
7092 "query_workflow_state_unavailable",
7093 format!("cannot snapshot replayed workflow state: {error}"),
7094 "QueryWorkflowStateUnavailable",
7095 )
7096 })?;
7097 handler(context, state, args).map_err(|message| {
7098 QueryTaskExecutionFailure::new(
7099 "query_workflow_state_unavailable",
7100 message,
7101 "QueryWorkflowStateUnavailable",
7102 )
7103 })?
7104 }
7105 };
7106
7107 future.await.map_err(|error| {
7108 QueryTaskExecutionFailure::new("query_rejected", error.to_string(), "QueryFailed")
7109 })
7110 }
7111
7112 #[cfg(test)]
7113 fn execute_workflow_task(&self, task: WorkflowTask) -> Result<Vec<Value>> {
7114 Ok(self.execute_workflow_task_decision(task)?.commands)
7115 }
7116
7117 fn execute_workflow_task_decision(&self, task: WorkflowTask) -> Result<WorkflowTaskDecision> {
7118 validate_workflow_task_payloads(&task)?;
7119
7120 if let Some(update_id) = task
7121 .workflow_update_id
7122 .as_deref()
7123 .filter(|update_id| !update_id.is_empty())
7124 {
7125 return self
7126 .execute_update_task(&task, update_id)
7127 .map(WorkflowTaskDecision::without_message_streams);
7128 }
7129
7130 let workflow = self
7131 .workflows
7132 .get(&task.workflow_type)
7133 .ok_or_else(|| Error::WorkflowNotRegistered(task.workflow_type.clone()))?;
7134 let input = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
7135 let resume_signal = decode_resume_signal(&task)?;
7136 let history_budget = WorkflowHistoryBudget {
7137 event_count: task
7138 .total_history_events
7139 .unwrap_or_else(|| u64::try_from(task.history_events.len()).unwrap_or(u64::MAX)),
7140 size_bytes: task.history_size_bytes,
7141 continue_as_new_recommended: task.continue_as_new_recommended.unwrap_or(false),
7142 pressure: task.history_budget_pressure.clone(),
7143 };
7144 let workflow_command_identity = task
7145 .workflow_command_id
7146 .clone()
7147 .filter(|identity| !identity.is_empty())
7148 .unwrap_or_default();
7149 let mut workflow_state = WorkflowState::new_with_identity(
7150 task.history_events,
7151 task.workflow_id,
7152 task.run_id,
7153 self.task_queue.clone(),
7154 task.payload_codec.clone(),
7155 resume_signal,
7156 )?;
7157 workflow_state.history_budget = history_budget;
7158 workflow_state.workflow_command_identity = workflow_command_identity;
7159 workflow_state.cancel_requested = task.cancel_requested;
7160 let state = Arc::new(Mutex::new(workflow_state));
7161 let ctx = WorkflowContext { state };
7162 let mut future = (workflow.execute)(ctx.clone(), input);
7163 let mut cx = TaskContext::from_waker(noop_waker_ref());
7164
7165 match future.as_mut().poll(&mut cx) {
7166 Poll::Ready(Ok(result)) => {
7167 ctx.ensure_history_consumed()?;
7168 let result = encode_typed_envelope(&result, &task.payload_codec)?;
7169 let mut commands = ctx.take_commands()?;
7170 commands.push(json!({
7171 "type": "complete_workflow",
7172 "result": result
7173 }));
7174 self.message_stream_decision(&ctx, commands)
7175 }
7176 Poll::Ready(Err(error)) => {
7177 if let Error::ContinueAsNew(request) = error {
7178 let mut commands = ctx.take_commands()?;
7179 if let Some(command) = ctx.continue_as_new_command(request)? {
7180 commands.push(command);
7181 }
7182 ctx.ensure_history_consumed()?;
7183 return self.message_stream_decision(&ctx, commands);
7184 }
7185 if workflow_task_integrity_error(&error) {
7186 return Err(error);
7191 }
7192 ctx.ensure_history_consumed()?;
7195 let mut commands = ctx.take_commands()?;
7196 commands.push(workflow_failure_command(&error));
7197 self.message_stream_decision(&ctx, commands)
7198 }
7199 Poll::Pending => {
7200 let commands = ctx.take_commands()?;
7201 if commands.is_empty() && !ctx.matched_recorded_pending()? {
7202 Err(Error::WorkflowYieldedWithoutCommand)
7203 } else {
7204 self.message_stream_decision(&ctx, commands)
7205 }
7206 }
7207 }
7208 }
7209
7210 fn message_stream_decision(
7211 &self,
7212 ctx: &WorkflowContext,
7213 commands: Vec<Value>,
7214 ) -> Result<WorkflowTaskDecision> {
7215 let (message_stream_cursors, message_stream_waits) = ctx.message_stream_metadata()?;
7216 Ok(WorkflowTaskDecision {
7217 commands,
7218 message_stream_cursors,
7219 message_stream_waits,
7220 })
7221 }
7222
7223 fn execute_update_task(&self, task: &WorkflowTask, update_id: &str) -> Result<Vec<Value>> {
7224 if !self.workflows.contains_key(&task.workflow_type) {
7225 return Err(Error::WorkflowNotRegistered(task.workflow_type.clone()));
7226 }
7227
7228 let accepted = task.history_events.iter().rev().find_map(|event| {
7229 (event.event_type == "UpdateAccepted"
7230 && event.payload.get("update_id").and_then(Value::as_str) == Some(update_id))
7231 .then_some(&event.payload)
7232 });
7233 let update_name = accepted
7234 .and_then(|payload| payload.get("update_name"))
7235 .and_then(Value::as_str)
7236 .or(task.update_name.as_deref())
7237 .unwrap_or_default();
7238 let Some(handler) = self
7239 .updates
7240 .get(&task.workflow_type)
7241 .and_then(|handlers| handlers.get(update_name))
7242 else {
7243 return Ok(vec![json!({
7244 "type": "fail_update",
7245 "update_id": update_id,
7246 "message": format!(
7247 "no update handler is registered for {}.{update_name}",
7248 task.workflow_type
7249 ),
7250 "exception_type": "UnknownUpdate",
7251 "non_retryable": true,
7252 })]);
7253 };
7254 let arguments = accepted
7255 .and_then(|payload| payload.get("arguments"))
7256 .or(task.arguments.as_ref());
7257 let arguments = decode_task_avro_arguments(arguments, &task.payload_codec)?;
7258 let context = QueryContext {
7259 workflow_id: task.workflow_id.clone(),
7260 run_id: task.run_id.clone(),
7261 workflow_type: task.workflow_type.clone(),
7262 run_status: Some("running".to_string()),
7263 workflow_input: Value::Null,
7264 workflow_input_avro_value: AvroValue::Null,
7265 history_events: Arc::new(task.history_events.clone()),
7266 signal_events: Arc::new(Vec::new()),
7267 };
7268 let mut future = handler(context, arguments);
7269 let mut cx = TaskContext::from_waker(noop_waker_ref());
7270
7271 match future.as_mut().poll(&mut cx) {
7272 Poll::Ready(Ok(result)) => Ok(vec![json!({
7273 "type": "complete_update",
7274 "update_id": update_id,
7275 "result": encode_typed_envelope(&result, &task.payload_codec)?,
7276 })]),
7277 Poll::Ready(Err(error)) => Ok(vec![json!({
7278 "type": "fail_update",
7279 "update_id": update_id,
7280 "message": error.to_string(),
7281 "exception_type": "UpdateFailed",
7282 "non_retryable": true,
7283 })]),
7284 Poll::Pending => Err(Error::WorkflowYieldedWithoutCommand),
7285 }
7286 }
7287
7288 async fn execute_activity_task(&self, task: ActivityTask) -> Result<AvroValue> {
7289 validate_activity_task_payloads(&task)?;
7290
7291 let handler = self
7292 .activities
7293 .get(&task.activity_type)
7294 .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?;
7295 let args = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
7296 let attempt_id = task
7297 .activity_attempt_id
7298 .clone()
7299 .or(task.attempt_id.clone())
7300 .unwrap_or_default();
7301 let lease_owner = task
7302 .lease_owner
7303 .clone()
7304 .unwrap_or_else(|| self.worker_id.clone());
7305 let ctx = ActivityContext {
7306 client: self.client.clone(),
7307 task_id: task.task_id,
7308 activity_attempt_id: attempt_id,
7309 lease_owner,
7310 activity_type: task.activity_type,
7311 attempt_number: task.attempt_number,
7312 task_queue: self.task_queue.clone(),
7313 worker_id: self.worker_id.clone(),
7314 };
7315
7316 handler(ctx, args).await
7317 }
7318}
7319
7320fn poller_result(
7321 kind: &str,
7322 result: std::result::Result<Result<()>, tokio::task::JoinError>,
7323) -> Result<()> {
7324 match result {
7325 Ok(result) => result,
7326 Err(error) => Err(Error::WorkerLoop(format!(
7327 "{kind} poller join error: {error}"
7328 ))),
7329 }
7330}
7331
7332fn optional_poller_result(
7333 kind: &str,
7334 result: Option<std::result::Result<Result<()>, tokio::task::JoinError>>,
7335) -> Result<()> {
7336 match result {
7337 Some(result) => poller_result(kind, result),
7338 None => Ok(()),
7339 }
7340}
7341
7342async fn join_pollers(
7343 workflow_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7344 activity_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7345 query_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7346) -> Result<()> {
7347 let mut first_error = None;
7348
7349 if let Some(handle) = workflow_poller {
7350 if let Err(error) = poller_result("workflow", handle.await) {
7351 first_error.get_or_insert(error);
7352 }
7353 }
7354
7355 if let Some(handle) = activity_poller {
7356 if let Err(error) = poller_result("activity", handle.await) {
7357 first_error.get_or_insert(error);
7358 }
7359 }
7360
7361 if let Some(handle) = query_poller {
7362 if let Err(error) = poller_result("query", handle.await) {
7363 first_error.get_or_insert(error);
7364 }
7365 }
7366
7367 if let Some(error) = first_error {
7368 Err(error)
7369 } else {
7370 Ok(())
7371 }
7372}
7373
7374fn default_worker_id() -> String {
7375 let millis = SystemTime::now()
7376 .duration_since(UNIX_EPOCH)
7377 .unwrap_or_default()
7378 .as_millis();
7379 format!("rust-worker-{}-{millis}", std::process::id())
7380}
7381
7382fn percent_encode_path_segment(segment: &str) -> String {
7383 const HEX: &[u8; 16] = b"0123456789ABCDEF";
7384 let mut encoded = String::with_capacity(segment.len());
7385
7386 for byte in segment.bytes() {
7387 if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') {
7388 encoded.push(char::from(byte));
7389 } else {
7390 encoded.push('%');
7391 encoded.push(char::from(HEX[(byte >> 4) as usize]));
7392 encoded.push(char::from(HEX[(byte & 0x0f) as usize]));
7393 }
7394 }
7395
7396 encoded
7397}
7398
7399fn unique_request_id(prefix: &str) -> String {
7400 let nanos = SystemTime::now()
7401 .duration_since(UNIX_EPOCH)
7402 .unwrap_or_default()
7403 .as_nanos();
7404 format!("{prefix}-{}-{nanos}", std::process::id())
7405}
7406
7407#[derive(Debug)]
7408struct QueryTaskExecutionFailure {
7409 reason: String,
7410 message: String,
7411 failure_type: String,
7412}
7413
7414impl QueryTaskExecutionFailure {
7415 fn new(
7416 reason: impl Into<String>,
7417 message: impl Into<String>,
7418 failure_type: impl Into<String>,
7419 ) -> Self {
7420 Self {
7421 reason: reason.into(),
7422 message: message.into(),
7423 failure_type: failure_type.into(),
7424 }
7425 }
7426}
7427
7428#[derive(Clone, Debug)]
7434pub struct WorkflowInstance<S> {
7435 state: Arc<Mutex<S>>,
7436}
7437
7438impl<S> WorkflowInstance<S> {
7439 fn new(state: S) -> Self {
7440 Self {
7441 state: Arc::new(Mutex::new(state)),
7442 }
7443 }
7444
7445 pub fn read<R>(&self, reader: impl FnOnce(&S) -> R) -> Result<R> {
7447 let state = self
7448 .state
7449 .lock()
7450 .map_err(|_| Error::WorkflowStatePoisoned)?;
7451 Ok(reader(&state))
7452 }
7453
7454 pub fn update<R>(&self, transition: impl FnOnce(&mut S) -> R) -> Result<R> {
7456 let mut state = self
7457 .state
7458 .lock()
7459 .map_err(|_| Error::WorkflowStatePoisoned)?;
7460 Ok(transition(&mut state))
7461 }
7462}
7463
7464impl<S: Clone> WorkflowInstance<S> {
7465 fn snapshot(&self) -> Result<S> {
7466 self.read(Clone::clone)
7467 }
7468}
7469
7470#[derive(Clone, Debug, PartialEq)]
7471pub struct MessageStreamMessage {
7472 pub stream_name: String,
7473 pub message_id: String,
7474 pub position: u64,
7475 pub arguments: Vec<AvroValue>,
7476}
7477
7478#[derive(Clone, Debug)]
7479pub struct MessageStream {
7480 ctx: WorkflowContext,
7481 name: String,
7482}
7483
7484impl MessageStream {
7485 pub async fn receive(&self, max_items: usize) -> Result<Vec<MessageStreamMessage>> {
7487 if !(1..=MESSAGE_STREAM_MAX_BATCH).contains(&max_items) {
7488 return Err(Error::Codec(format!(
7489 "message stream max_items must be between 1 and {MESSAGE_STREAM_MAX_BATCH}"
7490 )));
7491 }
7492 loop {
7493 if let Some(batch) = self.ctx.take_message_stream_batch(&self.name, max_items)? {
7494 return Ok(batch);
7495 }
7496
7497 self.ctx.record_message_stream_wait(&self.name)?;
7498 let replay_wait_sequence = self.ctx.next_message_stream_wait_sequence()?;
7499 let arguments = self.ctx.wait_runtime_signal(MESSAGE_STREAM_SIGNAL).await?;
7500 self.ctx.buffer_message_stream_delivery(arguments)?;
7501 if let Some(sequence) = replay_wait_sequence {
7502 self.ctx.buffer_message_stream_history_for_wait(sequence)?;
7503 }
7504 }
7505 }
7506
7507 pub async fn receive_one(&self) -> Result<MessageStreamMessage> {
7508 self.receive(1)
7509 .await?
7510 .into_iter()
7511 .next()
7512 .ok_or_else(|| Error::Codec("message stream resumed without a message".to_string()))
7513 }
7514}
7515
7516#[derive(Clone, Debug)]
7517pub struct WorkflowContext {
7518 state: Arc<Mutex<WorkflowState>>,
7519}
7520
7521fn valid_memo_key(key: &str) -> bool {
7522 let numeric_candidate = key.strip_prefix('-').unwrap_or(key);
7523
7524 !key.is_empty()
7525 && key.len() <= 64
7526 && (numeric_candidate.is_empty()
7527 || !numeric_candidate.bytes().all(|byte| byte.is_ascii_digit()))
7528 && key
7529 .bytes()
7530 .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b':' | b'-'))
7531}
7532
7533fn avro_encoded_size(value: &AvroValue) -> Result<usize> {
7534 BASE64
7535 .decode(encode_avro_value(value)?.blob)
7536 .map(|bytes| bytes.len())
7537 .map_err(|error| Error::Codec(format!("memo Avro encoding was not strict base64: {error}")))
7538}
7539
7540fn canonical_memo_entries(value: AvroValue, require_entries: bool) -> Result<AvroValue> {
7541 let AvroValue::Map(entries) = value else {
7542 return Err(Error::InvalidMemoUpdate(
7543 "entries must serialize to an Avro string-keyed map".to_string(),
7544 ));
7545 };
7546 if require_entries && entries.is_empty() {
7547 return Err(Error::InvalidMemoUpdate(
7548 "at least one entry is required".to_string(),
7549 ));
7550 }
7551 if entries.len() > MAX_MEMO_ENTRIES {
7552 return Err(Error::InvalidMemoUpdate(format!(
7553 "at most {MAX_MEMO_ENTRIES} entries are allowed"
7554 )));
7555 }
7556
7557 for (key, value) in &entries {
7558 if !valid_memo_key(&key) {
7559 return Err(Error::InvalidMemoUpdate(
7560 "keys must match ^(?!-?[0-9]+$)[A-Za-z0-9_.:-]{1,64}$".to_string(),
7561 ));
7562 }
7563 if avro_encoded_size(value)? > MAX_MEMO_VALUE_SIZE_BYTES {
7564 return Err(Error::InvalidMemoUpdate(format!(
7565 "value {key:?} exceeds the {MAX_MEMO_VALUE_SIZE_BYTES}-byte limit"
7566 )));
7567 }
7568 }
7569
7570 let value = AvroValue::Map(entries);
7571 if avro_encoded_size(&value)? > MAX_MEMO_TOTAL_SIZE_BYTES {
7572 return Err(Error::InvalidMemoUpdate(format!(
7573 "update exceeds the {MAX_MEMO_TOTAL_SIZE_BYTES}-byte total limit"
7574 )));
7575 }
7576 Ok(value)
7577}
7578
7579fn decode_memo_history_map(envelope: &Value, require_entries: bool) -> Result<AvroValue> {
7580 let object = envelope.as_object().ok_or_else(|| {
7581 Error::InvalidMemoUpdate(
7582 "history field must use the public {codec, blob} payload envelope".to_string(),
7583 )
7584 })?;
7585 if object.len() != 2 || !object.contains_key("codec") || !object.contains_key("blob") {
7586 return Err(Error::InvalidMemoUpdate(
7587 "history field must use exactly the public {codec, blob} payload envelope".to_string(),
7588 ));
7589 }
7590
7591 canonical_memo_entries(
7592 decode_wire_avro_value(envelope, DEFAULT_CODEC)?,
7593 require_entries,
7594 )
7595}
7596
7597impl WorkflowContext {
7598 pub fn message_stream(&self, name: impl Into<String>) -> Result<MessageStream> {
7599 let name = name.into();
7600 if name.is_empty()
7601 || name.len() > 128
7602 || !name.bytes().all(|byte| {
7603 byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b':' | b'-')
7604 })
7605 {
7606 return Err(Error::Codec(
7607 "message stream names must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens"
7608 .to_string(),
7609 ));
7610 }
7611 Ok(MessageStream {
7612 ctx: self.clone(),
7613 name,
7614 })
7615 }
7616
7617 fn record_message_stream_wait(&self, name: &str) -> Result<()> {
7618 let mut state = self
7619 .state
7620 .lock()
7621 .map_err(|_| Error::WorkflowStatePoisoned)?;
7622 let position = state.message_stream_cursors.get(name).copied().unwrap_or(0);
7623 state
7624 .message_stream_waits
7625 .insert(name.to_string(), position);
7626 Ok(())
7627 }
7628
7629 fn buffer_message_stream(&self, message: MessageStreamMessage) -> Result<()> {
7630 let mut state = self
7631 .state
7632 .lock()
7633 .map_err(|_| Error::WorkflowStatePoisoned)?;
7634 let cursor = state
7635 .message_stream_cursors
7636 .get(&message.stream_name)
7637 .copied()
7638 .unwrap_or(0);
7639 if message.position <= cursor {
7640 return Ok(());
7641 }
7642 let pending = state
7643 .message_stream_messages
7644 .entry(message.stream_name.clone())
7645 .or_default();
7646 if pending.iter().any(|candidate| {
7647 candidate.position == message.position || candidate.message_id == message.message_id
7648 }) {
7649 return Ok(());
7650 }
7651 pending.push(message);
7652 pending.sort_by_key(|candidate| candidate.position);
7653 Ok(())
7654 }
7655
7656 fn buffer_message_stream_delivery(&self, arguments: Vec<Value>) -> Result<Option<String>> {
7657 if let Some(delivery) = decode_message_stream_delivery(arguments)? {
7658 match delivery {
7659 MessageStreamDelivery::Message(message) => {
7660 let stream_name = message.stream_name.clone();
7661 self.buffer_message_stream(message)?;
7662 return Ok(Some(stream_name));
7663 }
7664 MessageStreamDelivery::Cursor {
7665 stream_name,
7666 through_position,
7667 } => self.apply_message_stream_cursor(&stream_name, through_position)?,
7668 }
7669 }
7670 Ok(None)
7671 }
7672
7673 fn next_message_stream_wait_sequence(&self) -> Result<Option<u64>> {
7674 let state = self
7675 .state
7676 .lock()
7677 .map_err(|_| Error::WorkflowStatePoisoned)?;
7678 Ok(match state.recorded_commands.get(state.command_cursor) {
7679 Some(RecordedCommand::SignalWait {
7680 sequence,
7681 signal_name,
7682 ..
7683 }) if signal_name == MESSAGE_STREAM_SIGNAL => Some(*sequence),
7684 _ => None,
7685 })
7686 }
7687
7688 fn buffer_message_stream_history_for_wait(&self, wait_sequence: u64) -> Result<()> {
7689 let (history, payload_codec) = {
7690 let state = self
7691 .state
7692 .lock()
7693 .map_err(|_| Error::WorkflowStatePoisoned)?;
7694 (
7695 Arc::clone(&state.history_events),
7696 state.payload_codec.clone(),
7697 )
7698 };
7699
7700 let Some(opened_index) = history.iter().position(|event| {
7701 event.event_type == "SignalWaitOpened"
7702 && durable_event_sequence(event) == Some(wait_sequence)
7703 && event.payload.get("signal_name").and_then(Value::as_str)
7704 == Some(MESSAGE_STREAM_SIGNAL)
7705 }) else {
7706 return Ok(());
7707 };
7708 let boundary_index = history
7709 .iter()
7710 .enumerate()
7711 .skip(opened_index + 1)
7712 .find_map(|(index, event)| {
7713 (durable_event_sequence(event).is_some_and(|sequence| sequence > wait_sequence)
7714 && is_authored_command_open_event(event))
7715 .then_some(index)
7716 })
7717 .unwrap_or(history.len());
7718
7719 for event in history[opened_index + 1..boundary_index]
7720 .iter()
7721 .filter(|event| {
7722 event.event_type == "SignalReceived"
7723 && event.payload.get("signal_name").and_then(Value::as_str)
7724 == Some(MESSAGE_STREAM_SIGNAL)
7725 })
7726 {
7727 let arguments = decode_signal_event_arguments(event, &payload_codec)?
7728 .into_iter()
7729 .map(AvroValue::into_json)
7730 .collect::<Result<Vec<_>>>()?;
7731 self.buffer_message_stream_delivery(arguments)?;
7732 }
7733 Ok(())
7734 }
7735
7736 fn apply_message_stream_cursor(&self, name: &str, through_position: u64) -> Result<()> {
7737 let mut state = self
7738 .state
7739 .lock()
7740 .map_err(|_| Error::WorkflowStatePoisoned)?;
7741 let cursor = state
7742 .message_stream_cursors
7743 .entry(name.to_string())
7744 .or_default();
7745 *cursor = (*cursor).max(through_position);
7746 if let Some(pending) = state.message_stream_messages.get_mut(name) {
7747 pending.retain(|message| message.position > through_position);
7748 }
7749 Ok(())
7750 }
7751
7752 fn take_message_stream_batch(
7753 &self,
7754 name: &str,
7755 max_items: usize,
7756 ) -> Result<Option<Vec<MessageStreamMessage>>> {
7757 let mut state = self
7758 .state
7759 .lock()
7760 .map_err(|_| Error::WorkflowStatePoisoned)?;
7761 let cursor = state.message_stream_cursors.get(name).copied().unwrap_or(0);
7762 let pending = state
7763 .message_stream_messages
7764 .entry(name.to_string())
7765 .or_default();
7766 let count = contiguous_message_stream_count(pending, cursor, max_items);
7767 if count == 0 {
7768 return Ok(None);
7769 }
7770 let batch = pending.drain(..count).collect::<Vec<_>>();
7771 let position = batch.last().map(|message| message.position).unwrap_or(0);
7772 state
7773 .message_stream_cursors
7774 .insert(name.to_string(), position);
7775 state.message_stream_waits.remove(name);
7776 Ok(Some(batch))
7777 }
7778
7779 fn message_stream_metadata(&self) -> Result<(Vec<Value>, Vec<Value>)> {
7780 let state = self
7781 .state
7782 .lock()
7783 .map_err(|_| Error::WorkflowStatePoisoned)?;
7784 let mut cursors = state.message_stream_cursors.iter().collect::<Vec<_>>();
7785 cursors.sort_by_key(|(name, _)| *name);
7786 let mut waits = state.message_stream_waits.iter().collect::<Vec<_>>();
7787 waits.sort_by_key(|(name, _)| *name);
7788 Ok((
7789 cursors
7790 .into_iter()
7791 .map(|(name, position)| json!({"stream_name": name, "through_position": position}))
7792 .collect(),
7793 waits
7794 .into_iter()
7795 .map(|(name, position)| json!({"stream_name": name, "after_position": position}))
7796 .collect(),
7797 ))
7798 }
7799 pub fn workflow_identity(&self) -> Result<WorkflowIdentity> {
7801 let state = self
7802 .state
7803 .lock()
7804 .map_err(|_| Error::WorkflowStatePoisoned)?;
7805 Ok(WorkflowIdentity {
7806 workflow_id: state.workflow_id.clone(),
7807 run_id: state.run_id.clone(),
7808 })
7809 }
7810
7811 pub fn history_budget(&self) -> Result<WorkflowHistoryBudget> {
7813 let state = self
7814 .state
7815 .lock()
7816 .map_err(|_| Error::WorkflowStatePoisoned)?;
7817 Ok(state.history_budget.clone())
7818 }
7819
7820 pub fn continue_as_new<T: Serialize>(&self, args: T) -> Result<Value> {
7826 self.continue_as_new_with_options(ContinueAsNewOptions::new(), args)
7827 }
7828
7829 pub fn continue_as_new_with_options<T: Serialize>(
7831 &self,
7832 options: ContinueAsNewOptions,
7833 args: T,
7834 ) -> Result<Value> {
7835 options.validate()?;
7836 Err(Error::ContinueAsNew(ContinueAsNewRequest {
7837 arguments: normalize_avro_arguments(AvroValue::from_serialize(&args)?),
7838 options,
7839 }))
7840 }
7841
7842 pub fn activity<T: Serialize>(
7843 &self,
7844 activity_type: impl Into<String>,
7845 args: T,
7846 ) -> ActivityCall {
7847 self.activity_with_options(activity_type, ActivityOptions::new(), args)
7848 }
7849
7850 pub fn activity_on_queue<T, Q>(
7851 &self,
7852 activity_type: impl Into<String>,
7853 task_queue: Option<Q>,
7854 args: T,
7855 ) -> ActivityCall
7856 where
7857 T: Serialize,
7858 Q: Into<String>,
7859 {
7860 let mut options = ActivityOptions::new();
7861 options.task_queue = task_queue.map(Into::into);
7862 self.activity_with_options(activity_type, options, args)
7863 }
7864
7865 pub fn activity_with_options<T: Serialize>(
7903 &self,
7904 activity_type: impl Into<String>,
7905 options: ActivityOptions,
7906 args: T,
7907 ) -> ActivityCall {
7908 ActivityCall {
7909 ctx: self.clone(),
7910 activity_type: activity_type.into(),
7911 options,
7912 args: Some(AvroValue::from_serialize(&args)),
7913 scheduled: false,
7914 parallel_group_path: Vec::new(),
7915 }
7916 }
7917
7918 pub async fn activity_avro_value<T: Serialize>(
7919 &self,
7920 activity_type: impl Into<String>,
7921 args: T,
7922 ) -> Result<AvroValue> {
7923 let mut call = self.activity(activity_type, args);
7924 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
7925 }
7926
7927 pub async fn activity_avro_value_with_options<T: Serialize>(
7928 &self,
7929 activity_type: impl Into<String>,
7930 options: ActivityOptions,
7931 args: T,
7932 ) -> Result<AvroValue> {
7933 let mut call = self.activity_with_options(activity_type, options, args);
7934 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
7935 }
7936
7937 pub async fn activity_typed<I, O>(&self, activity_type: impl Into<String>, args: I) -> Result<O>
7939 where
7940 I: Serialize,
7941 O: DeserializeOwned,
7942 {
7943 self.activity_typed_with_options(activity_type, ActivityOptions::new(), args)
7944 .await
7945 }
7946
7947 pub async fn activity_typed_with_options<I, O>(
7953 &self,
7954 activity_type: impl Into<String>,
7955 options: ActivityOptions,
7956 args: I,
7957 ) -> Result<O>
7958 where
7959 I: Serialize,
7960 O: DeserializeOwned,
7961 {
7962 let activity_type = activity_type.into();
7963 let encoded = AvroValue::from_serialize(&args).map_err(|error| {
7964 handler_type_error::<I>(
7965 HandlerKind::Activity,
7966 &activity_type,
7967 HandlerValueKind::Input,
7968 error.to_string(),
7969 )
7970 });
7971 let mut call = ActivityCall {
7972 ctx: self.clone(),
7973 activity_type: activity_type.clone(),
7974 options,
7975 args: Some(encoded),
7976 scheduled: false,
7977 parallel_group_path: Vec::new(),
7978 };
7979 let result = std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await?;
7980 decode_handler_result(result, HandlerKind::Activity, &activity_type)
7981 }
7982
7983 pub fn parallel(&self, operations: Vec<ParallelOperation>) -> ParallelCall {
7990 ParallelCall::new(self.clone(), operations)
7991 }
7992
7993 pub fn join(&self, operations: Vec<ParallelOperation>) -> ParallelCall {
7995 self.parallel(operations)
7996 }
7997
7998 pub async fn parallel_avro_value(
8000 &self,
8001 operations: Vec<ParallelOperation>,
8002 ) -> Result<Vec<ParallelAvroResult>> {
8003 let mut call = self.parallel(operations);
8004 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8005 }
8006
8007 pub fn select(&self, operations: Vec<ParallelOperation>) -> SelectCall {
8010 let operations = operations
8011 .into_iter()
8012 .enumerate()
8013 .map(|(index, operation)| (SelectionKey::Index(index), operation))
8014 .collect();
8015 SelectCall::new(self.clone(), operations)
8016 }
8017
8018 pub fn select_keyed<K>(&self, operations: Vec<(K, ParallelOperation)>) -> SelectCall
8020 where
8021 K: Into<SelectionKey>,
8022 {
8023 SelectCall::new(
8024 self.clone(),
8025 operations
8026 .into_iter()
8027 .map(|(key, operation)| (key.into(), operation))
8028 .collect(),
8029 )
8030 }
8031
8032 pub fn saga(&self) -> Saga {
8034 Saga::new(self.clone())
8035 }
8036
8037 pub fn is_cancellation_requested(&self) -> Result<bool> {
8039 let state = self
8040 .state
8041 .lock()
8042 .map_err(|_| Error::WorkflowStatePoisoned)?;
8043 Ok(state.cancel_requested)
8044 }
8045
8046 pub fn throw_if_cancellation_requested(&self) -> Result<()> {
8051 if self.is_cancellation_requested()? {
8052 return Err(Error::WorkflowCancellationRequested(
8053 WorkflowCancellationRequested,
8054 ));
8055 }
8056 Ok(())
8057 }
8058
8059 pub fn wait_signal(&self, signal_name: impl Into<String>) -> SignalCall {
8060 SignalCall {
8061 ctx: self.clone(),
8062 signal_name: signal_name.into(),
8063 runtime_reserved_allowed: false,
8064 opened_wait: false,
8065 matched_pending: false,
8066 parallel_group_path: Vec::new(),
8067 }
8068 }
8069
8070 fn wait_runtime_signal(&self, signal_name: impl Into<String>) -> SignalCall {
8071 SignalCall {
8072 ctx: self.clone(),
8073 signal_name: signal_name.into(),
8074 runtime_reserved_allowed: true,
8075 opened_wait: false,
8076 matched_pending: false,
8077 parallel_group_path: Vec::new(),
8078 }
8079 }
8080
8081 pub async fn wait_signal_avro_value(
8082 &self,
8083 signal_name: impl Into<String>,
8084 ) -> Result<Vec<AvroValue>> {
8085 let mut call = self.wait_signal(signal_name);
8086 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8087 }
8088
8089 pub fn signals(&self, signal_name: &str) -> Result<Vec<Vec<Value>>> {
8095 self.signals_avro_value(signal_name)?
8096 .into_iter()
8097 .map(|arguments| {
8098 arguments
8099 .into_iter()
8100 .map(AvroValue::into_json)
8101 .collect::<Result<Vec<_>>>()
8102 })
8103 .collect()
8104 }
8105
8106 pub fn signals_avro_value(&self, signal_name: &str) -> Result<Vec<Vec<AvroValue>>> {
8108 let state = self
8109 .state
8110 .lock()
8111 .map_err(|_| Error::WorkflowStatePoisoned)?;
8112 state
8113 .history_events
8114 .iter()
8115 .filter(|event| {
8116 event.event_type == "SignalReceived"
8117 && event.payload.get("signal_name").and_then(Value::as_str) == Some(signal_name)
8118 })
8119 .map(|event| decode_signal_event_arguments(event, &state.payload_codec))
8120 .collect()
8121 }
8122
8123 pub fn updates(&self, update_name: &str) -> Result<Vec<Vec<Value>>> {
8129 self.updates_avro_value(update_name)?
8130 .into_iter()
8131 .map(|arguments| {
8132 arguments
8133 .into_iter()
8134 .map(AvroValue::into_json)
8135 .collect::<Result<Vec<_>>>()
8136 })
8137 .collect()
8138 }
8139
8140 pub fn updates_avro_value(&self, update_name: &str) -> Result<Vec<Vec<AvroValue>>> {
8142 let state = self
8143 .state
8144 .lock()
8145 .map_err(|_| Error::WorkflowStatePoisoned)?;
8146 let mut seen = Vec::new();
8147 let mut updates = Vec::new();
8148 for event in state.history_events.iter() {
8149 if !matches!(
8150 event.event_type.as_str(),
8151 "UpdateAccepted" | "UpdateApplied"
8152 ) || event.payload.get("update_name").and_then(Value::as_str) != Some(update_name)
8153 || event.payload.get("arguments").is_none()
8154 {
8155 continue;
8156 }
8157 if let Some(update_id) = event.payload.get("update_id").and_then(Value::as_str) {
8158 if seen.iter().any(|recorded| recorded == update_id) {
8159 continue;
8160 }
8161 seen.push(update_id.to_string());
8162 }
8163 updates.push(decode_update_event_arguments(event, &state.payload_codec)?);
8164 }
8165 Ok(updates)
8166 }
8167
8168 pub fn wait_condition<F>(
8176 &self,
8177 options: ConditionWaitOptions,
8178 predicate: F,
8179 ) -> ConditionWaitCall
8180 where
8181 F: Fn() -> Result<bool> + Send + 'static,
8182 {
8183 ConditionWaitCall {
8184 ctx: self.clone(),
8185 options,
8186 predicate: Box::new(predicate),
8187 occurrence_id: None,
8188 opened_wait: false,
8189 parallel_group_path: Vec::new(),
8190 }
8191 }
8192
8193 pub fn sleep(&self, duration: Duration) -> TimerCall {
8214 let delay_seconds = duration
8215 .as_secs()
8216 .checked_add(u64::from(duration.subsec_nanos() > 0));
8217 TimerCall {
8218 ctx: self.clone(),
8219 delay_seconds,
8220 scheduled: false,
8221 matched_pending: false,
8222 parallel_group_path: Vec::new(),
8223 }
8224 }
8225
8226 pub fn start_timer(&self, duration: Duration) -> TimerCall {
8228 self.sleep(duration)
8229 }
8230
8231 pub fn side_effect<T, F>(&self, callback: F) -> Result<T>
8238 where
8239 T: Serialize + DeserializeOwned,
8240 F: FnOnce() -> T,
8241 {
8242 {
8243 let mut state = self
8244 .state
8245 .lock()
8246 .map_err(|_| Error::WorkflowStatePoisoned)?;
8247 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8248 return match recorded {
8249 RecordedCommand::SideEffect { sequence, value } => {
8250 state.command_cursor += 1;
8251 value.deserialize().map_err(|error| {
8252 Error::NonDeterministicReplay(ReplayFailure::new(
8253 "side_effect_type_mismatch",
8254 Some(sequence),
8255 Some(std::any::type_name::<T>().to_string()),
8256 Some(error.to_string()),
8257 "recorded side-effect value is incompatible with the requested Rust type",
8258 ))
8259 })
8260 }
8261 other => Err(command_mismatch(&other, "side effect")),
8262 };
8263 }
8264 }
8265
8266 let value = callback();
8267 let avro_value = AvroValue::from_serialize(&value)?;
8268 let mut state = self
8269 .state
8270 .lock()
8271 .map_err(|_| Error::WorkflowStatePoisoned)?;
8272 let result = encode_typed_envelope(&avro_value, &state.payload_codec)?;
8273 state.commands.push(json!({
8274 "type": "record_side_effect",
8275 "result": result,
8276 }));
8277 Ok(value)
8278 }
8279
8280 pub fn side_effect_avro_value<F>(&self, callback: F) -> Result<AvroValue>
8282 where
8283 F: FnOnce() -> AvroValue,
8284 {
8285 {
8286 let mut state = self
8287 .state
8288 .lock()
8289 .map_err(|_| Error::WorkflowStatePoisoned)?;
8290 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8291 return match recorded {
8292 RecordedCommand::SideEffect { value, .. } => {
8293 state.command_cursor += 1;
8294 Ok(value)
8295 }
8296 other => Err(command_mismatch(&other, "side effect")),
8297 };
8298 }
8299 }
8300
8301 let value = callback();
8302 let mut state = self
8303 .state
8304 .lock()
8305 .map_err(|_| Error::WorkflowStatePoisoned)?;
8306 let result = encode_typed_envelope(&value, &state.payload_codec)?;
8307 state.commands.push(json!({
8308 "type": "record_side_effect",
8309 "result": result,
8310 }));
8311 Ok(value)
8312 }
8313
8314 pub fn append_workflow_stream(
8320 &self,
8321 stream_name: impl Into<String>,
8322 items: &[WorkflowStreamAppendItem],
8323 max_pending_items: Option<u64>,
8324 ) -> Result<()> {
8325 if items.is_empty() {
8326 return Err(Error::Codec(
8327 "workflow_stream_items_empty: append requires at least one item".to_string(),
8328 ));
8329 }
8330 if max_pending_items == Some(0) {
8331 return Err(Error::Codec(
8332 "workflow_stream_pending_limit_invalid: max_pending_items must be positive"
8333 .to_string(),
8334 ));
8335 }
8336 let stream_name = stream_name.into();
8337 if stream_name.is_empty() {
8338 return Err(Error::Codec(
8339 "workflow_stream_name_invalid: stream name must not be empty".to_string(),
8340 ));
8341 }
8342
8343 let mut state = self
8344 .state
8345 .lock()
8346 .map_err(|_| Error::WorkflowStatePoisoned)?;
8347 let command_ordinal = state.workflow_stream_command_counter;
8348 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8349 state.workflow_stream_command_counter += 1;
8350 return match recorded {
8351 RecordedCommand::SideEffect { .. } => {
8352 state.command_cursor += 1;
8353 Ok(())
8354 }
8355 other => Err(command_mismatch(&other, "workflow stream append")),
8356 };
8357 }
8358
8359 let identity = Self::workflow_stream_command_identity(&state)?.to_string();
8360 state.workflow_stream_command_counter += 1;
8361 let wire_items = items
8362 .iter()
8363 .enumerate()
8364 .map(|(item_index, item)| {
8365 item.wire_value(Some(format!(
8366 "dw-stream:{identity}:{command_ordinal}:{item_index}"
8367 )))
8368 })
8369 .collect::<Vec<_>>();
8370 let mut directive = json!({
8371 "operation": "append",
8372 "stream_name": stream_name,
8373 "command_identity": identity,
8374 "command_ordinal": command_ordinal,
8375 "items": wire_items,
8376 });
8377 if let Some(max_pending_items) = max_pending_items {
8378 directive["max_pending_items"] = json!(max_pending_items);
8379 }
8380 let result = encode_typed_envelope(&AvroValue::Null, &state.payload_codec)?;
8381 state.commands.push(json!({
8382 "type": "record_side_effect",
8383 "result": result,
8384 "workflow_stream": directive,
8385 }));
8386 Ok(())
8387 }
8388
8389 pub fn close_workflow_stream(
8391 &self,
8392 stream_name: impl Into<String>,
8393 retention_seconds: Option<u64>,
8394 ) -> Result<()> {
8395 self.finish_workflow_stream(stream_name.into(), None, retention_seconds)
8396 }
8397
8398 pub fn error_workflow_stream(
8400 &self,
8401 stream_name: impl Into<String>,
8402 error_reason: impl Into<String>,
8403 retention_seconds: Option<u64>,
8404 ) -> Result<()> {
8405 let error_reason = error_reason.into();
8406 if error_reason.is_empty() {
8407 return Err(Error::Codec(
8408 "workflow_stream_error_invalid: error reason must not be empty".to_string(),
8409 ));
8410 }
8411 self.finish_workflow_stream(stream_name.into(), Some(error_reason), retention_seconds)
8412 }
8413
8414 fn finish_workflow_stream(
8415 &self,
8416 stream_name: String,
8417 error_reason: Option<String>,
8418 retention_seconds: Option<u64>,
8419 ) -> Result<()> {
8420 if stream_name.is_empty() {
8421 return Err(Error::Codec(
8422 "workflow_stream_name_invalid: stream name must not be empty".to_string(),
8423 ));
8424 }
8425 if retention_seconds == Some(0) {
8426 return Err(Error::Codec(
8427 "workflow_stream_retention_invalid: retention_seconds must be positive".to_string(),
8428 ));
8429 }
8430 let mut state = self
8431 .state
8432 .lock()
8433 .map_err(|_| Error::WorkflowStatePoisoned)?;
8434 let command_ordinal = state.workflow_stream_command_counter;
8435 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8436 state.workflow_stream_command_counter += 1;
8437 return match recorded {
8438 RecordedCommand::SideEffect { .. } => {
8439 state.command_cursor += 1;
8440 Ok(())
8441 }
8442 other => Err(command_mismatch(&other, "workflow stream close")),
8443 };
8444 }
8445 let identity = Self::workflow_stream_command_identity(&state)?.to_string();
8446 state.workflow_stream_command_counter += 1;
8447 let mut directive = json!({
8448 "operation": if error_reason.is_some() { "error" } else { "close" },
8449 "stream_name": stream_name,
8450 "command_identity": identity,
8451 "command_ordinal": command_ordinal,
8452 });
8453 if let Some(error_reason) = error_reason {
8454 directive["error_reason"] = json!(error_reason);
8455 }
8456 if let Some(retention_seconds) = retention_seconds {
8457 directive["retention_seconds"] = json!(retention_seconds);
8458 }
8459 let result = encode_typed_envelope(&AvroValue::Null, &state.payload_codec)?;
8460 state.commands.push(json!({
8461 "type": "record_side_effect",
8462 "result": result,
8463 "workflow_stream": directive,
8464 }));
8465 Ok(())
8466 }
8467
8468 fn workflow_stream_command_identity(state: &WorkflowState) -> Result<&str> {
8469 let identity = state.workflow_command_identity.as_str();
8470 if identity.is_empty() {
8471 return Err(Error::MissingWorkflowCommandIdentity);
8472 }
8473 Ok(identity)
8474 }
8475
8476 pub fn upsert_search_attributes(&self, update: SearchAttributeUpdate) -> Result<()> {
8482 update.validate()?;
8483 let (attributes, attribute_types) = update.into_wire_parts();
8484 let mut state = self
8485 .state
8486 .lock()
8487 .map_err(|_| Error::WorkflowStatePoisoned)?;
8488
8489 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8490 return match recorded {
8491 RecordedCommand::SearchAttributes {
8492 sequence,
8493 attributes: recorded_attributes,
8494 attribute_types: recorded_attribute_types,
8495 } => {
8496 if recorded_attributes != attributes {
8497 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8498 "search_attribute_value_mismatch",
8499 Some(sequence),
8500 Some(recorded_attributes.to_string()),
8501 Some(attributes.to_string()),
8502 "search-attribute values differ from the recorded durable command",
8503 )));
8504 }
8505 if let RecordedSnapshotValue::Known(recorded_types) = recorded_attribute_types {
8506 if recorded_types != attribute_types {
8507 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8508 "search_attribute_type_mismatch",
8509 Some(sequence),
8510 Some(json!(recorded_types).to_string()),
8511 Some(json!(attribute_types).to_string()),
8512 "search-attribute declared types differ from the recorded durable command",
8513 )));
8514 }
8515 }
8516 state.command_cursor += 1;
8517 Ok(())
8518 }
8519 other => Err(command_mismatch(&other, "search-attribute update")),
8520 };
8521 }
8522
8523 let mut command = serde_json::Map::from_iter([
8524 ("type".to_string(), json!("upsert_search_attributes")),
8525 ("attributes".to_string(), attributes),
8526 ]);
8527 if !attribute_types.is_empty() {
8528 command.insert("attribute_types".to_string(), json!(attribute_types));
8529 }
8530 state.commands.push(Value::Object(command));
8531 Ok(())
8532 }
8533
8534 pub fn uuid_v4(&self) -> Result<Uuid> {
8536 self.side_effect(Uuid::new_v4)
8537 }
8538
8539 pub fn get_version(
8542 &self,
8543 change_id: impl Into<String>,
8544 min_supported: i32,
8545 max_supported: i32,
8546 ) -> Result<i32> {
8547 let change_id = change_id.into();
8548 if change_id.trim().is_empty() {
8549 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8550 "version_change_id_invalid",
8551 None,
8552 Some("non-empty change ID".to_string()),
8553 Some(change_id),
8554 "version markers require a stable non-empty change ID",
8555 )));
8556 }
8557 if min_supported > max_supported {
8558 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8559 "version_range_invalid",
8560 None,
8561 Some("min_supported <= max_supported".to_string()),
8562 Some(format!("{min_supported}..={max_supported}")),
8563 "version marker supported range is invalid",
8564 )));
8565 }
8566
8567 let mut state = self
8568 .state
8569 .lock()
8570 .map_err(|_| Error::WorkflowStatePoisoned)?;
8571 if let Some((version, sequence)) = state.version_markers.get(&change_id).copied() {
8572 ensure_version_supported(&change_id, version, min_supported, max_supported, sequence)?;
8573 return Ok(version);
8574 }
8575
8576 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8577 return match recorded {
8578 RecordedCommand::VersionMarker {
8579 sequence,
8580 change_id: recorded_change_id,
8581 version,
8582 ..
8583 } => {
8584 if recorded_change_id != change_id {
8585 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8586 "version_change_id_mismatch",
8587 Some(sequence),
8588 Some(recorded_change_id),
8589 Some(change_id),
8590 "recorded version marker change ID differs from current workflow code",
8591 )));
8592 }
8593 ensure_version_supported(
8594 &change_id,
8595 version,
8596 min_supported,
8597 max_supported,
8598 sequence,
8599 )?;
8600 state.command_cursor += 1;
8601 state.version_markers.insert(change_id, (version, sequence));
8602 Ok(version)
8603 }
8604 other => Err(command_mismatch(
8605 &other,
8606 format!("version marker:{change_id}"),
8607 )),
8608 };
8609 }
8610
8611 let version = max_supported;
8612 state.commands.push(json!({
8613 "type": "record_version_marker",
8614 "change_id": change_id,
8615 "version": version,
8616 "min_supported": min_supported,
8617 "max_supported": max_supported,
8618 }));
8619 state.version_markers.insert(change_id, (version, 0));
8622 Ok(version)
8623 }
8624
8625 pub fn patched(&self, change_id: impl Into<String>) -> Result<bool> {
8627 Ok(self.get_version(change_id, -1, 1)? == 1)
8628 }
8629
8630 pub fn deprecate_patch(&self, change_id: impl Into<String>) -> Result<()> {
8632 self.get_version(change_id, -1, 1).map(|_| ())
8633 }
8634
8635 pub fn upsert_memo<T: Serialize>(&self, entries: T) -> Result<()> {
8640 let entries = canonical_memo_entries(AvroValue::from_serialize(&entries)?, true)?;
8641 let mut state = self
8642 .state
8643 .lock()
8644 .map_err(|_| Error::WorkflowStatePoisoned)?;
8645
8646 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8647 return match recorded {
8648 RecordedCommand::Memo {
8649 sequence,
8650 entries: recorded_entries,
8651 } => {
8652 if recorded_entries != entries {
8653 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8654 "memo_update_mismatch",
8655 Some(sequence),
8656 Some(format!("{recorded_entries:?}")),
8657 Some(format!("{entries:?}")),
8658 "recorded memo entries differ from the current workflow update",
8659 )));
8660 }
8661 state.command_cursor += 1;
8662 Ok(())
8663 }
8664 other => Err(command_mismatch(&other, "memo upsert")),
8665 };
8666 }
8667
8668 let entries_envelope = encode_typed_envelope(&entries, DEFAULT_CODEC)?;
8669 state.commands.push(json!({
8670 "type": "upsert_memo",
8671 "entries": entries_envelope,
8672 }));
8673 Ok(())
8674 }
8675
8676 pub fn start_child_workflow<T: Serialize>(
8703 &self,
8704 workflow_type: impl Into<String>,
8705 options: ChildWorkflowOptions,
8706 args: T,
8707 ) -> ChildWorkflowCall {
8708 ChildWorkflowCall {
8709 ctx: self.clone(),
8710 workflow_type: workflow_type.into(),
8711 options,
8712 args: Some(AvroValue::from_serialize(&args)),
8713 scheduled: false,
8714 matched_pending: false,
8715 parallel_group_path: Vec::new(),
8716 }
8717 }
8718
8719 pub async fn start_child_workflow_avro_value<T: Serialize>(
8720 &self,
8721 workflow_type: impl Into<String>,
8722 options: ChildWorkflowOptions,
8723 args: T,
8724 ) -> Result<ChildWorkflowAvroResult> {
8725 let mut call = self.start_child_workflow(workflow_type, options, args);
8726 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8727 }
8728
8729 fn take_commands(&self) -> Result<Vec<Value>> {
8730 let mut state = self
8731 .state
8732 .lock()
8733 .map_err(|_| Error::WorkflowStatePoisoned)?;
8734 Ok(std::mem::take(&mut state.commands))
8735 }
8736
8737 fn continue_as_new_command(&self, request: ContinueAsNewRequest) -> Result<Option<Value>> {
8738 let mut state = self
8739 .state
8740 .lock()
8741 .map_err(|_| Error::WorkflowStatePoisoned)?;
8742
8743 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8744 return Err(command_mismatch(&recorded, "continue as new"));
8745 }
8746 if state.recorded_continue_as_new_sequence.is_some() {
8747 state.continue_as_new_consumed = true;
8748 return Ok(None);
8749 }
8750
8751 let arguments = encode_typed_envelope(&request.arguments, &state.payload_codec)?;
8752 let mut command = serde_json::Map::from_iter([
8753 ("type".to_string(), json!("continue_as_new")),
8754 ("arguments".to_string(), arguments),
8755 ("queue".to_string(), json!(state.task_queue.clone())),
8756 ]);
8757 if let Some(workflow_type) = request.options.workflow_type {
8758 command.insert("workflow_type".to_string(), json!(workflow_type));
8759 }
8760 if let Some(task_queue) = request.options.task_queue {
8761 command.insert("queue".to_string(), json!(task_queue));
8762 }
8763 Ok(Some(Value::Object(command)))
8764 }
8765
8766 fn matched_recorded_pending(&self) -> Result<bool> {
8767 let state = self
8768 .state
8769 .lock()
8770 .map_err(|_| Error::WorkflowStatePoisoned)?;
8771 Ok(state.matched_recorded_pending)
8772 }
8773
8774 fn ensure_history_consumed(&self) -> Result<()> {
8775 let state = self
8776 .state
8777 .lock()
8778 .map_err(|_| Error::WorkflowStatePoisoned)?;
8779 if let Some(command) = state.recorded_commands.get(state.command_cursor) {
8780 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8781 "recorded_commands_unconsumed",
8782 Some(command.sequence()),
8783 Some(command.shape().to_string()),
8784 Some("workflow completion".to_string()),
8785 "workflow completed before consuming all recorded durable commands",
8786 )));
8787 }
8788 if let Some(sequence) = state
8789 .recorded_continue_as_new_sequence
8790 .filter(|_| !state.continue_as_new_consumed)
8791 {
8792 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8793 "recorded_continue_as_new_unconsumed",
8794 Some(sequence),
8795 Some("continue as new".to_string()),
8796 Some("workflow completion".to_string()),
8797 "workflow completed without consuming its recorded continue-as-new transition",
8798 )));
8799 }
8800 Ok(())
8801 }
8802}
8803
8804fn contiguous_message_stream_count(
8805 pending: &[MessageStreamMessage],
8806 cursor: u64,
8807 max_items: usize,
8808) -> usize {
8809 pending
8810 .iter()
8811 .take(max_items)
8812 .enumerate()
8813 .take_while(|(offset, message)| {
8814 u64::try_from(*offset)
8815 .ok()
8816 .and_then(|offset| cursor.checked_add(offset + 1))
8817 == Some(message.position)
8818 })
8819 .count()
8820}
8821
8822fn is_authored_command_open_event(event: &HistoryEvent) -> bool {
8823 matches!(
8824 event.event_type.as_str(),
8825 "ActivityScheduled"
8826 | "TimerScheduled"
8827 | "ChildWorkflowScheduled"
8828 | "SignalWaitOpened"
8829 | "ConditionWaitOpened"
8830 | "SearchAttributesUpserted"
8831 | "SideEffectRecorded"
8832 | "VersionMarkerRecorded"
8833 | "MemoUpserted"
8834 | "WorkflowContinuedAsNew"
8835 )
8836}
8837
8838#[derive(Debug)]
8839struct WorkflowState {
8840 workflow_id: Option<String>,
8841 run_id: Option<String>,
8842 task_queue: String,
8843 payload_codec: String,
8844 history_events: Arc<Vec<HistoryEvent>>,
8845 history_budget: WorkflowHistoryBudget,
8846 cancel_requested: bool,
8847 resume_signal: Option<ResumeSignal>,
8848 recorded_commands: Vec<RecordedCommand>,
8849 selection_markers: Vec<SelectionMarker>,
8850 selection_marker_cursor: usize,
8851 cancelled_selection_members: Vec<SelectionCancellation>,
8852 recorded_continue_as_new_sequence: Option<u64>,
8853 continue_as_new_consumed: bool,
8854 command_cursor: usize,
8855 condition_wait_occurrence_counter: u64,
8856 matched_recorded_pending: bool,
8857 version_markers: HashMap<String, (i32, u64)>,
8858 workflow_command_identity: String,
8859 workflow_stream_command_counter: u64,
8860 commands: Vec<Value>,
8861 message_stream_messages: HashMap<String, Vec<MessageStreamMessage>>,
8862 message_stream_cursors: HashMap<String, u64>,
8863 message_stream_waits: HashMap<String, u64>,
8864}
8865
8866impl WorkflowState {
8867 #[cfg(test)]
8868 fn new(
8869 history: Vec<HistoryEvent>,
8870 task_queue: String,
8871 payload_codec: String,
8872 resume_signal: Option<ResumeSignal>,
8873 ) -> Result<Self> {
8874 Self::new_with_identity(
8875 history,
8876 None,
8877 None,
8878 task_queue,
8879 payload_codec,
8880 resume_signal,
8881 )
8882 }
8883
8884 fn new_with_identity(
8885 history: Vec<HistoryEvent>,
8886 workflow_id: Option<String>,
8887 run_id: Option<String>,
8888 task_queue: String,
8889 payload_codec: String,
8890 resume_signal: Option<ResumeSignal>,
8891 ) -> Result<Self> {
8892 let recorded_commands = recorded_commands(
8893 &history,
8894 &payload_codec,
8895 WorkflowIdentity {
8896 workflow_id: workflow_id.clone(),
8897 run_id: run_id.clone(),
8898 },
8899 )?;
8900 let selection_markers = recorded_selection_markers(&history)?;
8901 let cancelled_selection_members = recorded_selection_cancellations(&history)?;
8902 let recorded_continue_as_new = history
8903 .iter()
8904 .filter(|event| event.event_type == "WorkflowContinuedAsNew")
8905 .collect::<Vec<_>>();
8906 if recorded_continue_as_new.len() > 1 {
8907 return Err(invalid_recorded_history(
8908 "duplicate_continue_as_new_transition",
8909 recorded_continue_as_new
8910 .last()
8911 .and_then(|event| durable_event_sequence(event))
8912 .unwrap_or(0),
8913 "one WorkflowContinuedAsNew event",
8914 &format!(
8915 "{} WorkflowContinuedAsNew events",
8916 recorded_continue_as_new.len()
8917 ),
8918 "workflow history records one continue-as-new transition more than once",
8919 ));
8920 }
8921 let recorded_continue_as_new_sequence = recorded_continue_as_new
8922 .first()
8923 .map(|event| {
8924 durable_event_sequence(event).ok_or_else(|| {
8925 Error::NonDeterministicReplay(ReplayFailure::new(
8926 "continue_as_new_sequence_missing",
8927 None,
8928 Some("recorded transition sequence".to_string()),
8929 Some("missing sequence".to_string()),
8930 "WorkflowContinuedAsNew history is missing its recorded sequence",
8931 ))
8932 })
8933 })
8934 .transpose()?;
8935 let mut message_stream_cursors = HashMap::new();
8936 for event in &history {
8937 if !matches!(
8938 event.event_type.as_str(),
8939 "SignalReceived" | "SignalApplied"
8940 ) || event.payload.get("signal_name").and_then(Value::as_str)
8941 != Some(MESSAGE_STREAM_SIGNAL)
8942 {
8943 continue;
8944 }
8945 let arguments = decode_signal_event_arguments(event, &payload_codec)?;
8946 if arguments.len() != 1 {
8947 continue;
8948 }
8949 let envelope = arguments[0].clone().into_json()?;
8950 let Some(envelope) = envelope.as_object() else {
8951 continue;
8952 };
8953 if envelope.get("schema").and_then(Value::as_str) != Some(MESSAGE_STREAM_CURSOR_SCHEMA)
8954 {
8955 continue;
8956 }
8957 let Some(stream_name) = envelope.get("stream_name").and_then(Value::as_str) else {
8958 continue;
8959 };
8960 let Some(through_position) = envelope.get("through_position").and_then(Value::as_u64)
8961 else {
8962 continue;
8963 };
8964 let cursor = message_stream_cursors
8965 .entry(stream_name.to_string())
8966 .or_insert(0);
8967 *cursor = (*cursor).max(through_position);
8968 }
8969 let event_count = u64::try_from(history.len()).unwrap_or(u64::MAX);
8970 let cancel_requested = history.iter().any(|event| {
8971 matches!(
8972 event.event_type.as_str(),
8973 "WorkflowCancellationRequested" | "WorkflowCancelRequested"
8974 )
8975 });
8976 Ok(Self {
8977 workflow_command_identity: String::new(),
8978 workflow_stream_command_counter: 0,
8979 workflow_id,
8980 run_id,
8981 task_queue,
8982 payload_codec,
8983 history_events: Arc::new(history),
8984 history_budget: WorkflowHistoryBudget {
8985 event_count,
8986 ..WorkflowHistoryBudget::default()
8987 },
8988 cancel_requested,
8989 resume_signal,
8990 recorded_commands,
8991 selection_markers,
8992 selection_marker_cursor: 0,
8993 cancelled_selection_members,
8994 recorded_continue_as_new_sequence,
8995 continue_as_new_consumed: false,
8996 command_cursor: 0,
8997 condition_wait_occurrence_counter: 0,
8998 matched_recorded_pending: false,
8999 version_markers: HashMap::new(),
9000 commands: Vec::new(),
9001 message_stream_messages: HashMap::new(),
9002 message_stream_cursors,
9003 message_stream_waits: HashMap::new(),
9004 })
9005 }
9006}
9007
9008enum MessageStreamDelivery {
9009 Message(MessageStreamMessage),
9010 Cursor {
9011 stream_name: String,
9012 through_position: u64,
9013 },
9014}
9015
9016fn decode_message_stream_delivery(arguments: Vec<Value>) -> Result<Option<MessageStreamDelivery>> {
9017 if arguments.len() != 1 {
9018 return Ok(None);
9019 }
9020 let envelope = arguments
9021 .into_iter()
9022 .next()
9023 .expect("one argument was checked");
9024 let Some(envelope) = envelope.as_object() else {
9025 return Ok(None);
9026 };
9027 let Some(stream_name) = envelope.get("stream_name").and_then(Value::as_str) else {
9028 return Ok(None);
9029 };
9030 if envelope.get("schema").and_then(Value::as_str) == Some(MESSAGE_STREAM_CURSOR_SCHEMA) {
9031 let Some(through_position) = envelope.get("through_position").and_then(Value::as_u64)
9032 else {
9033 return Ok(None);
9034 };
9035 return Ok(Some(MessageStreamDelivery::Cursor {
9036 stream_name: stream_name.to_string(),
9037 through_position,
9038 }));
9039 }
9040 if envelope.get("schema").and_then(Value::as_str) != Some(MESSAGE_STREAM_SCHEMA) {
9041 return Ok(None);
9042 }
9043 let Some(message_id) = envelope.get("message_id").and_then(Value::as_str) else {
9044 return Ok(None);
9045 };
9046 let Some(position) = envelope
9047 .get("position")
9048 .and_then(Value::as_u64)
9049 .filter(|value| *value > 0)
9050 else {
9051 return Ok(None);
9052 };
9053 let Some(payload_envelope) = envelope.get("payload_envelope") else {
9054 return Ok(None);
9055 };
9056 let Ok(payload_envelope) = serde_json::from_value::<PayloadEnvelope>(payload_envelope.clone())
9057 else {
9058 return Ok(None);
9059 };
9060 let decoded = decode_avro_value(&payload_envelope)?;
9061 let AvroValue::Array(values) = decoded else {
9062 return Ok(None);
9063 };
9064 Ok(Some(MessageStreamDelivery::Message(MessageStreamMessage {
9065 stream_name: stream_name.to_string(),
9066 message_id: message_id.to_string(),
9067 position,
9068 arguments: values,
9069 })))
9070}
9071
9072#[derive(Clone, Debug)]
9073enum RecordedCommand {
9074 Activity {
9075 sequence: u64,
9076 activity_type: Option<String>,
9077 options: Option<RecordedActivityOptions>,
9078 outcome: Option<ActivityOutcome>,
9079 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9080 },
9081 Timer {
9082 sequence: u64,
9083 delay_seconds: u64,
9084 fired: bool,
9085 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9086 },
9087 ChildWorkflow {
9088 sequence: u64,
9089 workflow_type: Option<String>,
9090 outcome: Option<ChildWorkflowOutcome>,
9091 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9092 },
9093 SignalWait {
9094 sequence: u64,
9095 signal_name: String,
9096 value: Option<Vec<AvroValue>>,
9097 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9098 },
9099 ConditionWait {
9100 sequence: u64,
9101 occurrence_id: String,
9102 condition_key: Option<String>,
9103 predicate_identity: String,
9104 timeout_seconds: Option<u64>,
9105 result: Option<ConditionWaitResult>,
9106 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9107 },
9108 SearchAttributes {
9109 sequence: u64,
9110 attributes: Value,
9111 attribute_types: RecordedSnapshotValue<BTreeMap<String, String>>,
9112 },
9113 SideEffect {
9114 sequence: u64,
9115 value: AvroValue,
9116 },
9117 VersionMarker {
9118 sequence: u64,
9119 change_id: String,
9120 version: i32,
9121 },
9122 Memo {
9123 sequence: u64,
9124 entries: AvroValue,
9125 },
9126}
9127
9128#[derive(Clone, Debug, PartialEq, Eq)]
9129struct SelectionMarker {
9130 selection_group_id: String,
9131 selection_group_base_sequence: u64,
9132 selection_group_size: usize,
9133 member_key: SelectionKey,
9134 member_index: usize,
9135 member_base_sequence: u64,
9136 member_size: usize,
9137 operation_kind: String,
9138 operation_identity: String,
9139 outcome: String,
9140 resolution_event_id: String,
9141 resolution_event_type: String,
9142}
9143
9144#[derive(Clone, Debug, PartialEq, Eq)]
9145struct SelectionCancellation {
9146 selection_group_id: String,
9147 member_key: SelectionKey,
9148 member_index: usize,
9149 member_base_sequence: u64,
9150 member_size: usize,
9151 operation_kind: String,
9152 operation_identity: String,
9153}
9154
9155fn recorded_selection_markers(events: &[HistoryEvent]) -> Result<Vec<SelectionMarker>> {
9156 let mut markers: Vec<SelectionMarker> = Vec::new();
9157 for event in events
9158 .iter()
9159 .filter(|event| event.event_type == "SelectionResolved")
9160 {
9161 let payload = &event.payload;
9162 let base_sequence = required_selection_u64(payload, "selection_group_base_sequence")?;
9163 let group_size = required_selection_usize(payload, "selection_group_size")?;
9164 let member_base_sequence = required_selection_u64(payload, "member_base_sequence")?;
9165 let member_size = required_selection_usize(payload, "member_size")?;
9166 let member_index = required_selection_usize_allow_zero(payload, "member_index")?;
9167 let group_id = payload_string(payload, "selection_group_id").ok_or_else(|| {
9168 invalid_recorded_history(
9169 "selection_marker_invalid",
9170 base_sequence,
9171 "non-empty selection_group_id",
9172 &payload.to_string(),
9173 "selection winner history is missing its durable group identity",
9174 )
9175 })?;
9176 let expected_group_id = format!("select-calls:{base_sequence}:{group_size}");
9177 if group_id != expected_group_id {
9178 return Err(invalid_recorded_history(
9179 "selection_marker_invalid",
9180 base_sequence,
9181 &expected_group_id,
9182 &group_id,
9183 "selection winner history contains an incompatible group identity",
9184 ));
9185 }
9186 let group_end = base_sequence
9187 .checked_add(u64::try_from(group_size).unwrap_or(u64::MAX))
9188 .unwrap_or(u64::MAX);
9189 let member_end = member_base_sequence
9190 .checked_add(u64::try_from(member_size).unwrap_or(u64::MAX))
9191 .unwrap_or(u64::MAX);
9192 if member_index >= group_size
9193 || member_base_sequence < base_sequence
9194 || member_end > group_end
9195 {
9196 return Err(invalid_recorded_history(
9197 "selection_marker_invalid",
9198 base_sequence,
9199 "winner member within selection group bounds",
9200 &payload.to_string(),
9201 "selection winner history contains an invalid member range",
9202 ));
9203 }
9204 let operation_kind = payload_string(payload, "operation_kind").ok_or_else(|| {
9205 invalid_recorded_history(
9206 "selection_marker_invalid",
9207 base_sequence,
9208 "selection operation kind",
9209 &payload.to_string(),
9210 "selection winner history is missing its operation kind",
9211 )
9212 })?;
9213 if !matches!(
9214 operation_kind.as_str(),
9215 "activity" | "child" | "timer" | "signal" | "condition" | "group"
9216 ) {
9217 return Err(invalid_recorded_history(
9218 "selection_marker_invalid",
9219 base_sequence,
9220 "activity, child, timer, signal, condition, or group",
9221 &operation_kind,
9222 "selection winner history contains an unsupported operation kind",
9223 ));
9224 }
9225 let operation_identity =
9226 payload_string(payload, "operation_identity").ok_or_else(|| {
9227 invalid_recorded_history(
9228 "selection_marker_invalid",
9229 base_sequence,
9230 "non-empty operation identity",
9231 &payload.to_string(),
9232 "selection winner history is missing its durable operation identity",
9233 )
9234 })?;
9235 let outcome = payload_string(payload, "outcome").ok_or_else(|| {
9236 invalid_recorded_history(
9237 "selection_marker_invalid",
9238 base_sequence,
9239 "completed or failed selection outcome",
9240 &payload.to_string(),
9241 "selection winner history is missing its outcome",
9242 )
9243 })?;
9244 if !matches!(outcome.as_str(), "completed" | "failed") {
9245 return Err(invalid_recorded_history(
9246 "selection_marker_invalid",
9247 base_sequence,
9248 "completed or failed selection outcome",
9249 &outcome,
9250 "selection winner history contains an unsupported outcome",
9251 ));
9252 }
9253 let marker = SelectionMarker {
9254 selection_group_id: group_id,
9255 selection_group_base_sequence: base_sequence,
9256 selection_group_size: group_size,
9257 member_key: selection_key_from_value(payload.get("member_key"), base_sequence)?,
9258 member_index,
9259 member_base_sequence,
9260 member_size,
9261 operation_kind,
9262 operation_identity,
9263 outcome,
9264 resolution_event_id: payload_string(payload, "resolution_event_id").ok_or_else(
9265 || {
9266 invalid_recorded_history(
9267 "selection_marker_invalid",
9268 base_sequence,
9269 "durable resolution_event_id",
9270 &payload.to_string(),
9271 "selection winner history is missing its terminal event identity",
9272 )
9273 },
9274 )?,
9275 resolution_event_type: payload_string(payload, "resolution_event_type").ok_or_else(
9276 || {
9277 invalid_recorded_history(
9278 "selection_marker_invalid",
9279 base_sequence,
9280 "durable resolution_event_type",
9281 &payload.to_string(),
9282 "selection winner history is missing its terminal event type",
9283 )
9284 },
9285 )?,
9286 };
9287 if let Some(existing) = markers
9288 .iter()
9289 .find(|existing| existing.selection_group_id == marker.selection_group_id)
9290 {
9291 if existing != &marker {
9292 return Err(invalid_recorded_history(
9293 "selection_marker_conflict",
9294 base_sequence,
9295 &format!("one winner for {}", marker.selection_group_id),
9296 &payload.to_string(),
9297 "selection history records conflicting winners for one durable group",
9298 ));
9299 }
9300 continue;
9301 }
9302 markers.push(marker);
9303 }
9304 Ok(markers)
9305}
9306
9307fn recorded_selection_cancellations(events: &[HistoryEvent]) -> Result<Vec<SelectionCancellation>> {
9308 let mut cancelled: Vec<SelectionCancellation> = Vec::new();
9309 for event in events
9310 .iter()
9311 .filter(|event| event.event_type == "SelectionOperationCancelled")
9312 {
9313 let group_id = payload_string(&event.payload, "selection_group_id").ok_or_else(|| {
9314 invalid_recorded_history(
9315 "selection_cancellation_invalid",
9316 0,
9317 "non-empty selection_group_id",
9318 &event.payload.to_string(),
9319 "selection cancellation history is missing its group identity",
9320 )
9321 })?;
9322 let member_base_sequence = required_selection_u64(&event.payload, "member_base_sequence")?;
9323 let marker = SelectionCancellation {
9324 selection_group_id: group_id,
9325 member_key: selection_key_from_value(
9326 event.payload.get("member_key"),
9327 member_base_sequence,
9328 )?,
9329 member_index: required_selection_usize_allow_zero(&event.payload, "member_index")?,
9330 member_base_sequence,
9331 member_size: required_selection_usize(&event.payload, "member_size")?,
9332 operation_kind: payload_string(&event.payload, "operation_kind").ok_or_else(|| {
9333 invalid_recorded_history(
9334 "selection_cancellation_invalid",
9335 member_base_sequence,
9336 "selection operation kind",
9337 &event.payload.to_string(),
9338 "selection cancellation is missing its operation kind",
9339 )
9340 })?,
9341 operation_identity: payload_string(&event.payload, "operation_identity").ok_or_else(
9342 || {
9343 invalid_recorded_history(
9344 "selection_cancellation_invalid",
9345 member_base_sequence,
9346 "selection operation identity",
9347 &event.payload.to_string(),
9348 "selection cancellation is missing its operation identity",
9349 )
9350 },
9351 )?,
9352 };
9353 if let Some(existing) = cancelled.iter().find(|recorded| {
9354 recorded.selection_group_id == marker.selection_group_id
9355 && recorded.member_base_sequence == marker.member_base_sequence
9356 }) {
9357 if existing != &marker {
9358 return Err(invalid_recorded_history(
9359 "selection_cancellation_conflict",
9360 member_base_sequence,
9361 "one stable SelectionOperationCancelled marker",
9362 &event.payload.to_string(),
9363 "selection cancellation history contains conflicting member metadata",
9364 ));
9365 }
9366 } else {
9367 cancelled.push(marker);
9368 }
9369 }
9370 Ok(cancelled)
9371}
9372
9373fn required_selection_u64(payload: &Value, field: &str) -> Result<u64> {
9374 payload
9375 .get(field)
9376 .and_then(value_as_u64)
9377 .filter(|value| *value > 0)
9378 .ok_or_else(|| {
9379 invalid_recorded_history(
9380 "selection_marker_invalid",
9381 0,
9382 &format!("positive integer {field}"),
9383 &payload.to_string(),
9384 "selection history contains invalid durable identity metadata",
9385 )
9386 })
9387}
9388
9389fn required_selection_usize(payload: &Value, field: &str) -> Result<usize> {
9390 required_selection_usize_allow_zero(payload, field).and_then(|value| {
9391 if value > 0 {
9392 Ok(value)
9393 } else {
9394 Err(invalid_recorded_history(
9395 "selection_marker_invalid",
9396 0,
9397 &format!("positive integer {field}"),
9398 &payload.to_string(),
9399 "selection history contains invalid durable identity metadata",
9400 ))
9401 }
9402 })
9403}
9404
9405fn required_selection_usize_allow_zero(payload: &Value, field: &str) -> Result<usize> {
9406 payload
9407 .get(field)
9408 .and_then(value_as_u64)
9409 .and_then(|value| usize::try_from(value).ok())
9410 .ok_or_else(|| {
9411 invalid_recorded_history(
9412 "selection_marker_invalid",
9413 0,
9414 &format!("non-negative integer {field}"),
9415 &payload.to_string(),
9416 "selection history contains invalid durable identity metadata",
9417 )
9418 })
9419}
9420
9421#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9422struct RecordedActivityOptions {
9423 task_queue: RecordedSnapshotValue<Option<String>>,
9424 execution_mode: RecordedSnapshotValue<Option<String>>,
9425 retry_policy: ActivityRetrySnapshot,
9426}
9427
9428#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9429enum RecordedSnapshotValue<T> {
9430 Unknown,
9432 Known(T),
9433}
9434
9435impl<T: PartialEq> RecordedSnapshotValue<T> {
9436 fn matches_current(&self, current: &Self) -> bool {
9437 match self {
9438 Self::Unknown => true,
9439 Self::Known(recorded) => matches!(current, Self::Known(value) if value == recorded),
9440 }
9441 }
9442}
9443
9444#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9445struct ActivityRetrySnapshot {
9446 snapshot_version: RecordedSnapshotValue<Option<u64>>,
9447 max_attempts: RecordedSnapshotValue<Option<u64>>,
9448 backoff_seconds: RecordedSnapshotValue<Vec<u64>>,
9449 start_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
9450 schedule_to_start_timeout: RecordedSnapshotValue<Option<u64>>,
9451 schedule_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
9452 heartbeat_timeout: RecordedSnapshotValue<Option<u64>>,
9453 non_retryable_error_types: RecordedSnapshotValue<Vec<String>>,
9454}
9455
9456impl ActivityRetrySnapshot {
9457 fn matches_current(&self, current: &Self) -> bool {
9458 self.snapshot_version
9459 .matches_current(¤t.snapshot_version)
9460 && self.max_attempts.matches_current(¤t.max_attempts)
9461 && self
9462 .backoff_seconds
9463 .matches_current(¤t.backoff_seconds)
9464 && self
9465 .start_to_close_timeout
9466 .matches_current(¤t.start_to_close_timeout)
9467 && self
9468 .schedule_to_start_timeout
9469 .matches_current(¤t.schedule_to_start_timeout)
9470 && self
9471 .schedule_to_close_timeout
9472 .matches_current(¤t.schedule_to_close_timeout)
9473 && self
9474 .heartbeat_timeout
9475 .matches_current(¤t.heartbeat_timeout)
9476 && self
9477 .non_retryable_error_types
9478 .matches_current(¤t.non_retryable_error_types)
9479 }
9480}
9481
9482fn recorded_optional_u64(
9483 object: Option<&serde_json::Map<String, Value>>,
9484 field: &str,
9485) -> RecordedSnapshotValue<Option<u64>> {
9486 match object.and_then(|object| object.get(field)) {
9487 None => RecordedSnapshotValue::Unknown,
9488 Some(Value::Null) => RecordedSnapshotValue::Known(None),
9489 Some(value) => RecordedSnapshotValue::Known(value_as_u64(value)),
9490 }
9491}
9492
9493fn recorded_optional_string(
9494 object: &serde_json::Map<String, Value>,
9495 field: &str,
9496) -> RecordedSnapshotValue<Option<String>> {
9497 match object.get(field) {
9498 None => RecordedSnapshotValue::Unknown,
9499 Some(Value::Null) => RecordedSnapshotValue::Known(None),
9500 Some(value) => RecordedSnapshotValue::Known(value.as_str().map(str::to_string)),
9501 }
9502}
9503
9504fn recorded_activity_retry_snapshot(policy: Option<&Value>) -> ActivityRetrySnapshot {
9505 let policy = policy.and_then(Value::as_object);
9506 let backoff_seconds = policy
9507 .and_then(|policy| policy.get("backoff_seconds"))
9508 .and_then(Value::as_array)
9509 .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
9510 .map_or(RecordedSnapshotValue::Unknown, RecordedSnapshotValue::Known);
9511 let mut non_retryable_error_types = Vec::new();
9512 for error_type in policy
9513 .and_then(|policy| policy.get("non_retryable_error_types"))
9514 .and_then(Value::as_array)
9515 .into_iter()
9516 .flatten()
9517 .filter_map(Value::as_str)
9518 .map(str::trim)
9519 .filter(|error_type| !error_type.is_empty())
9520 {
9521 if !non_retryable_error_types
9522 .iter()
9523 .any(|recorded| recorded == error_type)
9524 {
9525 non_retryable_error_types.push(error_type.to_string());
9526 }
9527 }
9528
9529 ActivityRetrySnapshot {
9530 snapshot_version: recorded_optional_u64(policy, "snapshot_version"),
9531 max_attempts: recorded_optional_u64(policy, "max_attempts"),
9532 backoff_seconds,
9533 start_to_close_timeout: recorded_optional_u64(policy, "start_to_close_timeout"),
9534 schedule_to_start_timeout: recorded_optional_u64(policy, "schedule_to_start_timeout"),
9535 schedule_to_close_timeout: recorded_optional_u64(policy, "schedule_to_close_timeout"),
9536 heartbeat_timeout: recorded_optional_u64(policy, "heartbeat_timeout"),
9537 non_retryable_error_types: if policy
9538 .is_some_and(|policy| policy.contains_key("non_retryable_error_types"))
9539 {
9540 RecordedSnapshotValue::Known(non_retryable_error_types)
9541 } else {
9542 RecordedSnapshotValue::Unknown
9543 },
9544 }
9545}
9546
9547fn current_activity_retry_snapshot(options: &ValidatedActivityOptions) -> ActivityRetrySnapshot {
9548 let policy = options.retry_policy.as_ref();
9549 let max_attempts = match policy.and_then(|policy| policy.get("max_attempts")) {
9550 Some(Value::Null) => None,
9551 Some(value) => value_as_u64(value),
9552 None => Some(1),
9553 };
9554 let backoff_seconds = policy
9555 .and_then(|policy| policy.get("backoff_seconds"))
9556 .and_then(Value::as_array)
9557 .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
9558 .unwrap_or_default();
9559 let non_retryable_error_types = policy
9560 .and_then(|policy| policy.get("non_retryable_error_types"))
9561 .and_then(Value::as_array)
9562 .into_iter()
9563 .flatten()
9564 .filter_map(Value::as_str)
9565 .map(str::to_string)
9566 .collect();
9567
9568 ActivityRetrySnapshot {
9569 snapshot_version: RecordedSnapshotValue::Known(Some(1)),
9570 max_attempts: RecordedSnapshotValue::Known(max_attempts),
9571 backoff_seconds: RecordedSnapshotValue::Known(backoff_seconds),
9572 start_to_close_timeout: RecordedSnapshotValue::Known(options.start_to_close_timeout),
9573 schedule_to_start_timeout: RecordedSnapshotValue::Known(options.schedule_to_start_timeout),
9574 schedule_to_close_timeout: RecordedSnapshotValue::Known(options.schedule_to_close_timeout),
9575 heartbeat_timeout: RecordedSnapshotValue::Known(options.heartbeat_timeout),
9576 non_retryable_error_types: RecordedSnapshotValue::Known(non_retryable_error_types),
9577 }
9578}
9579
9580fn activity_options_description(options: &RecordedActivityOptions) -> String {
9581 serde_json::to_string(options).unwrap_or_else(|_| format!("{options:?}"))
9582}
9583
9584impl RecordedCommand {
9585 fn sequence(&self) -> u64 {
9586 match self {
9587 Self::Activity { sequence, .. }
9588 | Self::Timer { sequence, .. }
9589 | Self::ChildWorkflow { sequence, .. }
9590 | Self::SignalWait { sequence, .. }
9591 | Self::ConditionWait { sequence, .. }
9592 | Self::SearchAttributes { sequence, .. }
9593 | Self::SideEffect { sequence, .. }
9594 | Self::VersionMarker { sequence, .. }
9595 | Self::Memo { sequence, .. } => *sequence,
9596 }
9597 }
9598
9599 fn shape(&self) -> &'static str {
9600 match self {
9601 Self::Activity { .. } => "activity",
9602 Self::Timer { .. } => "timer",
9603 Self::ChildWorkflow { .. } => "child workflow",
9604 Self::SignalWait { .. } => "signal wait",
9605 Self::ConditionWait { .. } => "condition wait",
9606 Self::SearchAttributes { .. } => "search-attribute update",
9607 Self::SideEffect { .. } => "side effect",
9608 Self::VersionMarker { .. } => "version marker",
9609 Self::Memo { .. } => "memo upsert",
9610 }
9611 }
9612}
9613
9614fn ensure_version_supported(
9615 change_id: &str,
9616 version: i32,
9617 min_supported: i32,
9618 max_supported: i32,
9619 sequence: u64,
9620) -> Result<()> {
9621 if (min_supported..=max_supported).contains(&version) {
9622 return Ok(());
9623 }
9624 Err(Error::NonDeterministicReplay(ReplayFailure::new(
9625 "version_marker_incompatible_range",
9626 (sequence != 0).then_some(sequence),
9627 Some(format!("{min_supported}..={max_supported}")),
9628 Some(format!("{change_id}:{version}")),
9629 "recorded workflow version is outside the range supported by current code",
9630 )))
9631}
9632
9633#[derive(Clone, Debug)]
9634struct ResumeSignal {
9635 signal_name: String,
9636 arguments: Vec<AvroValue>,
9637}
9638
9639const MAX_PARALLEL_OPERATIONS: usize = 1000;
9640
9641fn parallel_group_prefix(kind: &str) -> &'static str {
9642 match kind {
9643 "activity" => "parallel-activities",
9644 "child" => "parallel-children",
9645 "timer" => "parallel-timers",
9646 _ => "parallel-calls",
9647 }
9648}
9649
9650fn parallel_group_entry(
9651 base_sequence: u64,
9652 size: usize,
9653 index: usize,
9654 kind: &str,
9655) -> ParallelGroupMetadata {
9656 ParallelGroupMetadata {
9657 parallel_group_id: format!("{}:{base_sequence}:{size}", parallel_group_prefix(kind)),
9658 parallel_group_kind: kind.to_string(),
9659 parallel_group_base_sequence: base_sequence,
9660 parallel_group_size: size,
9661 parallel_group_index: index,
9662 parallel_group_mode: None,
9663 selection_member_key: None,
9664 selection_member_index: None,
9665 selection_member_base_sequence: None,
9666 selection_member_size: None,
9667 selection_member_kind: None,
9668 }
9669}
9670
9671struct SelectionMemberMetadata {
9672 key: SelectionKey,
9673 index: usize,
9674 base_sequence: u64,
9675 size: usize,
9676 kind: String,
9677}
9678
9679fn selection_group_entry(
9680 base_sequence: u64,
9681 size: usize,
9682 index: usize,
9683 kind: &str,
9684 member: &SelectionMemberMetadata,
9685) -> ParallelGroupMetadata {
9686 ParallelGroupMetadata {
9687 parallel_group_id: format!("select-calls:{base_sequence}:{size}"),
9688 parallel_group_kind: kind.to_string(),
9689 parallel_group_base_sequence: base_sequence,
9690 parallel_group_size: size,
9691 parallel_group_index: index,
9692 parallel_group_mode: Some("select".to_string()),
9693 selection_member_key: Some(member.key.clone()),
9694 selection_member_index: Some(member.index),
9695 selection_member_base_sequence: Some(member.base_sequence),
9696 selection_member_size: Some(member.size),
9697 selection_member_kind: Some(member.kind.clone()),
9698 }
9699}
9700
9701fn apply_parallel_group_path(
9702 command: &mut serde_json::Map<String, Value>,
9703 path: &[ParallelGroupMetadata],
9704) {
9705 let Some(inner) = path.last() else {
9706 return;
9707 };
9708 command.insert(
9709 "parallel_group_id".to_string(),
9710 json!(inner.parallel_group_id),
9711 );
9712 command.insert(
9713 "parallel_group_kind".to_string(),
9714 json!(inner.parallel_group_kind),
9715 );
9716 command.insert(
9717 "parallel_group_base_sequence".to_string(),
9718 json!(inner.parallel_group_base_sequence),
9719 );
9720 command.insert(
9721 "parallel_group_size".to_string(),
9722 json!(inner.parallel_group_size),
9723 );
9724 command.insert(
9725 "parallel_group_index".to_string(),
9726 json!(inner.parallel_group_index),
9727 );
9728 if let Some(mode) = &inner.parallel_group_mode {
9729 command.insert("parallel_group_mode".to_string(), json!(mode));
9730 }
9731 if let Some(key) = &inner.selection_member_key {
9732 command.insert("selection_member_key".to_string(), json!(key));
9733 }
9734 if let Some(index) = inner.selection_member_index {
9735 command.insert("selection_member_index".to_string(), json!(index));
9736 }
9737 if let Some(base_sequence) = inner.selection_member_base_sequence {
9738 command.insert(
9739 "selection_member_base_sequence".to_string(),
9740 json!(base_sequence),
9741 );
9742 }
9743 if let Some(size) = inner.selection_member_size {
9744 command.insert("selection_member_size".to_string(), json!(size));
9745 }
9746 if let Some(kind) = &inner.selection_member_kind {
9747 command.insert("selection_member_kind".to_string(), json!(kind));
9748 }
9749 command.insert("parallel_group_path".to_string(), json!(path));
9750}
9751
9752fn ensure_parallel_path_matches(
9753 sequence: u64,
9754 recorded: Option<&[ParallelGroupMetadata]>,
9755 expected: &[ParallelGroupMetadata],
9756) -> Result<()> {
9757 match (recorded, expected.is_empty()) {
9758 (None, true) => Ok(()),
9759 (Some(recorded), false) if recorded == expected => Ok(()),
9760 (None, false) => Err(invalid_recorded_history(
9761 "parallel_group_metadata_missing",
9762 sequence,
9763 &serde_json::to_string(expected).unwrap_or_default(),
9764 "<missing>",
9765 "recorded parallel member is missing its durable group path",
9766 )),
9767 (Some(recorded), true) => Err(invalid_recorded_history(
9768 "parallel_group_shape_mismatch",
9769 sequence,
9770 "sequential command",
9771 &serde_json::to_string(recorded).unwrap_or_default(),
9772 "recorded command belonged to a parallel group but current code schedules it sequentially",
9773 )),
9774 (Some(recorded), false) => Err(invalid_recorded_history(
9775 "parallel_group_shape_mismatch",
9776 sequence,
9777 &serde_json::to_string(recorded).unwrap_or_default(),
9778 &serde_json::to_string(expected).unwrap_or_default(),
9779 "recorded parallel-group identity or path changed during replay",
9780 )),
9781 }
9782}
9783
9784#[derive(Clone, Debug)]
9785enum ParallelShape {
9786 Leaf,
9787 Group(Vec<ParallelShape>),
9788}
9789
9790struct ParallelDescriptor {
9791 operation: ParallelOperation,
9792 offset: usize,
9793 member_path: Vec<usize>,
9794 group_path: Vec<ParallelGroupMetadata>,
9795}
9796
9797fn parallel_leaf_count(operations: &[ParallelOperation]) -> usize {
9798 operations
9799 .iter()
9800 .map(|operation| match operation {
9801 ParallelOperation::Group(children) => parallel_leaf_count(children),
9802 _ => 1,
9803 })
9804 .sum()
9805}
9806
9807fn parallel_operation_kind(operation: &ParallelOperation) -> Option<&'static str> {
9808 match operation {
9809 ParallelOperation::Activity { .. } => Some("activity"),
9810 ParallelOperation::ChildWorkflow { .. } => Some("child"),
9811 ParallelOperation::Timer(_) => Some("timer"),
9812 ParallelOperation::Signal(_) => Some("signal"),
9813 ParallelOperation::Condition { .. } => Some("condition"),
9814 ParallelOperation::Group(children) => parallel_group_kind(children),
9815 }
9816}
9817
9818fn parallel_group_kind(operations: &[ParallelOperation]) -> Option<&'static str> {
9819 let mut kind = None;
9820 for operation in operations {
9821 let Some(operation_kind) = parallel_operation_kind(operation) else {
9822 continue;
9823 };
9824 match kind {
9825 None => kind = Some(operation_kind),
9826 Some(current) if current == operation_kind => {}
9827 Some(_) => return Some("mixed"),
9828 }
9829 }
9830 kind
9831}
9832
9833fn validate_parallel_operations(
9834 operations: &[ParallelOperation],
9835 member_path: &mut Vec<usize>,
9836 root: bool,
9837) -> Result<()> {
9838 let leaves = parallel_leaf_count(operations);
9839 if leaves > MAX_PARALLEL_OPERATIONS {
9840 return Err(Error::InvalidParallelGroup(ParallelGroupError {
9841 reason: "fan_out_limit_exceeded",
9842 member_path: member_path.clone(),
9843 message: format!(
9844 "group contains {leaves} durable leaves; the limit is {MAX_PARALLEL_OPERATIONS}"
9845 ),
9846 }));
9847 }
9848 if !root && operations.is_empty() {
9849 return Err(Error::InvalidParallelGroup(ParallelGroupError {
9850 reason: "nested_group_empty",
9851 member_path: member_path.clone(),
9852 message: "a nested group must contain at least one durable leaf".to_string(),
9853 }));
9854 }
9855
9856 for (index, operation) in operations.iter().enumerate() {
9857 member_path.push(index);
9858 match operation {
9859 ParallelOperation::Activity {
9860 options, arguments, ..
9861 } => {
9862 options
9863 .validate()
9864 .map_err(|error| Error::InvalidActivityOptions(error))?;
9865 if let Err(error) = arguments {
9866 return Err(Error::InvalidParallelGroup(ParallelGroupError {
9867 reason: "arguments_invalid",
9868 member_path: member_path.clone(),
9869 message: error.to_string(),
9870 }));
9871 }
9872 }
9873 ParallelOperation::ChildWorkflow {
9874 options, arguments, ..
9875 } => {
9876 validate_parallel_child_options(options)?;
9877 if let Err(error) = arguments {
9878 return Err(Error::InvalidParallelGroup(ParallelGroupError {
9879 reason: "arguments_invalid",
9880 member_path: member_path.clone(),
9881 message: error.to_string(),
9882 }));
9883 }
9884 }
9885 ParallelOperation::Timer(duration)
9886 if duration.as_secs() == u64::MAX && duration.subsec_nanos() > 0 =>
9887 {
9888 return Err(Error::TimerDurationOverflow);
9889 }
9890 ParallelOperation::Timer(_) => {}
9891 ParallelOperation::Signal(signal_name) => {
9892 validate_user_signal_name(signal_name)?;
9893 if signal_name.trim().is_empty() {
9894 return Err(Error::InvalidParallelGroup(ParallelGroupError {
9895 reason: "signal_name_empty",
9896 member_path: member_path.clone(),
9897 message: "signal wait name must not be empty".to_string(),
9898 }));
9899 }
9900 }
9901 ParallelOperation::Condition { options, .. } => {
9902 options.validate()?;
9903 }
9904 ParallelOperation::Group(children) => {
9905 validate_parallel_operations(children, member_path, false)?;
9906 }
9907 }
9908 member_path.pop();
9909 }
9910 Ok(())
9911}
9912
9913fn validate_parallel_child_options(options: &ChildWorkflowOptions) -> Result<()> {
9914 if options.task_queue.trim().is_empty() {
9915 return Err(Error::InvalidChildWorkflowOptions(
9916 "task_queue must not be empty".to_string(),
9917 ));
9918 }
9919 for (name, value) in [
9920 (
9921 "execution_timeout_seconds",
9922 options.execution_timeout_seconds,
9923 ),
9924 ("run_timeout_seconds", options.run_timeout_seconds),
9925 ] {
9926 if value == Some(0) {
9927 return Err(Error::InvalidChildWorkflowOptions(format!(
9928 "{name} must be at least 1"
9929 )));
9930 }
9931 }
9932 if options
9933 .retry_policy
9934 .as_ref()
9935 .is_some_and(|policy| policy.max_attempts == Some(0))
9936 {
9937 return Err(Error::InvalidChildWorkflowOptions(
9938 "retry_policy.max_attempts must be at least 1".to_string(),
9939 ));
9940 }
9941 Ok(())
9942}
9943
9944fn parallel_shape(operations: &[ParallelOperation]) -> ParallelShape {
9945 ParallelShape::Group(
9946 operations
9947 .iter()
9948 .map(|operation| match operation {
9949 ParallelOperation::Group(children) => parallel_shape(children),
9950 _ => ParallelShape::Leaf,
9951 })
9952 .collect(),
9953 )
9954}
9955
9956fn parallel_descriptors(
9957 operations: Vec<ParallelOperation>,
9958 base_sequence: u64,
9959) -> Result<Vec<ParallelDescriptor>> {
9960 let size = parallel_leaf_count(&operations);
9961 let kind = parallel_group_kind(&operations).unwrap_or("activity");
9962 let mut descriptors = Vec::with_capacity(size);
9963 let mut cursor = 0;
9964
9965 for (index, operation) in operations.into_iter().enumerate() {
9966 match operation {
9967 ParallelOperation::Group(children) => {
9968 let child_base = base_sequence
9969 .checked_add(u64::try_from(cursor).unwrap_or(u64::MAX))
9970 .ok_or(Error::TimerDurationOverflow)?;
9971 for mut descriptor in parallel_descriptors(children, child_base)? {
9972 let outer_index = cursor + descriptor.offset;
9973 descriptor.group_path.insert(
9974 0,
9975 parallel_group_entry(base_sequence, size, outer_index, kind),
9976 );
9977 descriptor.member_path.insert(0, index);
9978 descriptor.offset = outer_index;
9979 descriptors.push(descriptor);
9980 }
9981 cursor = descriptors.len();
9982 }
9983 operation => {
9984 descriptors.push(ParallelDescriptor {
9985 operation,
9986 offset: cursor,
9987 member_path: vec![index],
9988 group_path: vec![parallel_group_entry(base_sequence, size, cursor, kind)],
9989 });
9990 cursor += 1;
9991 }
9992 }
9993 }
9994 Ok(descriptors)
9995}
9996
9997enum ParallelLeafCall {
9998 Activity(ActivityCall),
9999 ChildWorkflow(ChildWorkflowCall),
10000 Timer(TimerCall),
10001 Signal(SignalCall),
10002 Condition(ConditionWaitCall),
10003}
10004
10005fn parallel_leaf_call(
10006 ctx: &WorkflowContext,
10007 operation: ParallelOperation,
10008 parallel_group_path: Vec<ParallelGroupMetadata>,
10009) -> ParallelLeafCall {
10010 match operation {
10011 ParallelOperation::Activity {
10012 activity_type,
10013 options,
10014 arguments,
10015 } => ParallelLeafCall::Activity(ActivityCall {
10016 ctx: ctx.clone(),
10017 activity_type,
10018 options,
10019 args: Some(arguments),
10020 scheduled: false,
10021 parallel_group_path,
10022 }),
10023 ParallelOperation::ChildWorkflow {
10024 workflow_type,
10025 options,
10026 arguments,
10027 } => ParallelLeafCall::ChildWorkflow(ChildWorkflowCall {
10028 ctx: ctx.clone(),
10029 workflow_type,
10030 options,
10031 args: Some(arguments),
10032 scheduled: false,
10033 matched_pending: false,
10034 parallel_group_path,
10035 }),
10036 ParallelOperation::Timer(duration) => {
10037 let delay_seconds = duration
10038 .as_secs()
10039 .checked_add(u64::from(duration.subsec_nanos() > 0));
10040 ParallelLeafCall::Timer(TimerCall {
10041 ctx: ctx.clone(),
10042 delay_seconds,
10043 scheduled: false,
10044 matched_pending: false,
10045 parallel_group_path,
10046 })
10047 }
10048 ParallelOperation::Signal(signal_name) => ParallelLeafCall::Signal(SignalCall {
10049 ctx: ctx.clone(),
10050 signal_name,
10051 runtime_reserved_allowed: false,
10052 opened_wait: false,
10053 matched_pending: false,
10054 parallel_group_path,
10055 }),
10056 ParallelOperation::Condition { options, predicate } => {
10057 ParallelLeafCall::Condition(ConditionWaitCall {
10058 ctx: ctx.clone(),
10059 options,
10060 predicate,
10061 occurrence_id: None,
10062 opened_wait: false,
10063 parallel_group_path,
10064 })
10065 }
10066 ParallelOperation::Group(_) => {
10067 unreachable!("parallel descriptors contain only durable leaves")
10068 }
10069 }
10070}
10071
10072impl ParallelLeafCall {
10073 fn poll_avro_value(&mut self, cx: &mut TaskContext<'_>) -> Poll<Result<ParallelAvroResult>> {
10074 match self {
10075 Self::Activity(call) => Pin::new(call)
10076 .poll_avro_value(cx)
10077 .map_ok(ParallelAvroResult::Activity),
10078 Self::ChildWorkflow(call) => Pin::new(call)
10079 .poll_avro_value(cx)
10080 .map_ok(ParallelAvroResult::ChildWorkflow),
10081 Self::Timer(call) => Pin::new(call)
10082 .poll(cx)
10083 .map_ok(|()| ParallelAvroResult::Timer),
10084 Self::Signal(call) => Pin::new(call)
10085 .poll_avro_value(cx)
10086 .map_ok(ParallelAvroResult::Signal),
10087 Self::Condition(call) => Pin::new(call)
10088 .poll(cx)
10089 .map_ok(ParallelAvroResult::Condition),
10090 }
10091 }
10092}
10093
10094struct ParallelLeaf {
10095 call: ParallelLeafCall,
10096 member_path: Vec<usize>,
10097 group_path: Vec<ParallelGroupMetadata>,
10098 result: Option<ParallelAvroResult>,
10099}
10100
10101pub struct ParallelCall {
10103 ctx: WorkflowContext,
10104 operations: Option<Vec<ParallelOperation>>,
10105 shape: Option<ParallelShape>,
10106 leaves: Vec<ParallelLeaf>,
10107}
10108
10109impl ParallelCall {
10110 fn new(ctx: WorkflowContext, operations: Vec<ParallelOperation>) -> Self {
10111 Self {
10112 ctx,
10113 operations: Some(operations),
10114 shape: None,
10115 leaves: Vec::new(),
10116 }
10117 }
10118
10119 fn initialize(&mut self) -> Result<()> {
10120 let operations = self.operations.take().unwrap_or_default();
10121 validate_parallel_operations(&operations, &mut Vec::new(), true)?;
10122 self.shape = Some(parallel_shape(&operations));
10123 if operations.is_empty() {
10124 return Ok(());
10125 }
10126
10127 let base_sequence = {
10128 let state = self
10129 .ctx
10130 .state
10131 .lock()
10132 .map_err(|_| Error::WorkflowStatePoisoned)?;
10133 if let Some(recorded) = state.recorded_commands.get(state.command_cursor) {
10134 recorded.sequence()
10135 } else {
10136 let last = state
10137 .recorded_commands
10138 .last()
10139 .map(RecordedCommand::sequence)
10140 .unwrap_or(0);
10141 last.checked_add(u64::try_from(state.commands.len()).unwrap_or(u64::MAX))
10142 .and_then(|sequence| sequence.checked_add(1))
10143 .ok_or_else(|| {
10144 Error::InvalidParallelGroup(ParallelGroupError {
10145 reason: "sequence_overflow",
10146 member_path: Vec::new(),
10147 message: "parallel group sequence identity overflowed u64".to_string(),
10148 })
10149 })?
10150 }
10151 };
10152
10153 self.leaves = parallel_descriptors(operations, base_sequence)?
10154 .into_iter()
10155 .map(|descriptor| {
10156 let call = parallel_leaf_call(
10157 &self.ctx,
10158 descriptor.operation,
10159 descriptor.group_path.clone(),
10160 );
10161 ParallelLeaf {
10162 call,
10163 member_path: descriptor.member_path,
10164 group_path: descriptor.group_path,
10165 result: None,
10166 }
10167 })
10168 .collect();
10169 Ok(())
10170 }
10171
10172 fn poll_avro_value(
10173 mut self: Pin<&mut Self>,
10174 cx: &mut TaskContext<'_>,
10175 ) -> Poll<Result<Vec<ParallelAvroResult>>> {
10176 if self.operations.is_some() {
10177 if let Err(error) = self.initialize() {
10178 return Poll::Ready(Err(error));
10179 }
10180 }
10181 if self.leaves.is_empty() {
10182 return Poll::Ready(Ok(Vec::new()));
10183 }
10184
10185 let mut failures = Vec::new();
10186 let mut pending = false;
10187 for (index, leaf) in self.leaves.iter_mut().enumerate() {
10188 if leaf.result.is_some() {
10189 continue;
10190 }
10191 match leaf.call.poll_avro_value(cx) {
10192 Poll::Ready(Ok(result)) => leaf.result = Some(result),
10193 Poll::Ready(Err(error)) => failures.push((index, error)),
10194 Poll::Pending => pending = true,
10195 }
10196 }
10197
10198 if !failures.is_empty() {
10199 if let Some(position) = failures
10200 .iter()
10201 .position(|(_, error)| workflow_task_integrity_error(error))
10202 {
10203 return Poll::Ready(Err(failures.remove(position).1));
10204 }
10205 failures.sort_by_key(|(index, _)| *index);
10206 let (failed_index, cause) = failures.remove(0);
10207 let failed = &self.leaves[failed_index];
10208 let completed = self
10209 .leaves
10210 .iter()
10211 .filter_map(|leaf| {
10212 leaf.result
10213 .clone()
10214 .and_then(|result| result.into_json_result().ok())
10215 .map(|result| ParallelCompletion {
10216 member_path: leaf.member_path.clone(),
10217 result,
10218 })
10219 })
10220 .collect();
10221 let group_id = failed
10222 .group_path
10223 .first()
10224 .map(|entry| entry.parallel_group_id.clone())
10225 .unwrap_or_default();
10226 return Poll::Ready(Err(Error::ParallelFailed(ParallelFailure {
10227 group_id,
10228 member_path: failed.member_path.clone(),
10229 group_path: failed.group_path.clone(),
10230 completed,
10231 cause: Box::new(cause),
10232 })));
10233 }
10234 if pending {
10235 return Poll::Pending;
10236 }
10237
10238 let mut flat_results = self
10239 .leaves
10240 .iter_mut()
10241 .map(|leaf| leaf.result.take().expect("completed parallel leaf"))
10242 .collect::<Vec<_>>()
10243 .into_iter();
10244 let results = parallel_results_for_shape(
10245 self.shape.as_ref().expect("initialized parallel shape"),
10246 &mut flat_results,
10247 );
10248 Poll::Ready(Ok(match results {
10249 ParallelAvroResult::Group(results) => results,
10250 ParallelAvroResult::Activity(_)
10251 | ParallelAvroResult::ChildWorkflow(_)
10252 | ParallelAvroResult::Timer
10253 | ParallelAvroResult::Signal(_)
10254 | ParallelAvroResult::Condition(_) => {
10255 unreachable!("root parallel shape is a group")
10256 }
10257 }))
10258 }
10259}
10260
10261fn parallel_results_for_shape(
10262 shape: &ParallelShape,
10263 flat_results: &mut impl Iterator<Item = ParallelAvroResult>,
10264) -> ParallelAvroResult {
10265 match shape {
10266 ParallelShape::Leaf => flat_results.next().expect("one result per parallel leaf"),
10267 ParallelShape::Group(children) => ParallelAvroResult::Group(
10268 children
10269 .iter()
10270 .map(|child| parallel_results_for_shape(child, flat_results))
10271 .collect(),
10272 ),
10273 }
10274}
10275
10276impl Future for ParallelCall {
10277 type Output = Result<Vec<ParallelResult>>;
10278
10279 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
10280 self.poll_avro_value(cx)
10281 .map_ok(|results| {
10282 results
10283 .into_iter()
10284 .map(ParallelAvroResult::into_json_result)
10285 .collect::<Result<Vec<_>>>()
10286 })
10287 .map_ok(|result| result)
10288 .flatten_result()
10289 }
10290}
10291
10292#[derive(Clone, Debug)]
10293struct SelectionMemberPlan {
10294 key: SelectionKey,
10295 index: usize,
10296 base_sequence: u64,
10297 size: usize,
10298 kind: String,
10299 shape: ParallelShape,
10300 leaf_start: usize,
10301}
10302
10303fn selection_operation_kind(operation: &ParallelOperation) -> &'static str {
10304 match operation {
10305 ParallelOperation::Activity { .. } => "activity",
10306 ParallelOperation::ChildWorkflow { .. } => "child",
10307 ParallelOperation::Timer(_) => "timer",
10308 ParallelOperation::Signal(_) => "signal",
10309 ParallelOperation::Condition { .. } => "condition",
10310 ParallelOperation::Group(_) => "group",
10311 }
10312}
10313
10314fn selection_operation_shape(operation: &ParallelOperation) -> ParallelShape {
10315 match operation {
10316 ParallelOperation::Group(children) => parallel_shape(children),
10317 _ => ParallelShape::Leaf,
10318 }
10319}
10320
10321fn selection_descriptors(
10322 operations: Vec<(SelectionKey, ParallelOperation)>,
10323 base_sequence: u64,
10324) -> Result<(Vec<ParallelDescriptor>, Vec<SelectionMemberPlan>)> {
10325 if operations.is_empty() {
10326 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10327 reason: "selection_empty",
10328 member_path: Vec::new(),
10329 message: "durable selection requires at least one operation".to_string(),
10330 }));
10331 }
10332 let operation_refs = operations
10333 .iter()
10334 .map(|(_, operation)| operation)
10335 .collect::<Vec<_>>();
10336 let total_size = operation_refs
10337 .iter()
10338 .map(|operation| match operation {
10339 ParallelOperation::Group(children) => parallel_leaf_count(children),
10340 _ => 1,
10341 })
10342 .sum::<usize>();
10343 if total_size > MAX_PARALLEL_OPERATIONS {
10344 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10345 reason: "fan_out_limit_exceeded",
10346 member_path: Vec::new(),
10347 message: format!(
10348 "selection contains {total_size} durable leaves; the limit is {MAX_PARALLEL_OPERATIONS}"
10349 ),
10350 }));
10351 }
10352 let group_kind = {
10353 let mut kind = None;
10354 for operation in &operation_refs {
10355 let operation_kind = parallel_operation_kind(operation).unwrap_or("mixed");
10356 match kind {
10357 None => kind = Some(operation_kind),
10358 Some(current) if current == operation_kind => {}
10359 Some(_) => {
10360 kind = Some("mixed");
10361 break;
10362 }
10363 }
10364 }
10365 kind.unwrap_or("mixed")
10366 };
10367
10368 let mut descriptors = Vec::with_capacity(total_size);
10369 let mut members = Vec::with_capacity(operations.len());
10370 let mut cursor = 0usize;
10371 let mut seen_keys: Vec<SelectionKey> = Vec::new();
10372 for (member_index, (key, operation)) in operations.into_iter().enumerate() {
10373 if matches!(&key, SelectionKey::Name(value) if value.is_empty()) {
10374 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10375 reason: "selection_key_invalid",
10376 member_path: vec![member_index],
10377 message: "selection member keys must be non-empty strings or non-negative integers"
10378 .to_string(),
10379 }));
10380 }
10381 if seen_keys.contains(&key) {
10382 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10383 reason: "selection_key_duplicate",
10384 member_path: vec![member_index],
10385 message: format!("selection member key {key:?} is duplicated"),
10386 }));
10387 }
10388 seen_keys.push(key.clone());
10389 let member_size = match &operation {
10390 ParallelOperation::Group(children) => parallel_leaf_count(children),
10391 _ => 1,
10392 };
10393 if member_size == 0 {
10394 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10395 reason: "selection_member_empty",
10396 member_path: vec![member_index],
10397 message: "a selection member must contain at least one durable leaf".to_string(),
10398 }));
10399 }
10400 let member_base = base_sequence
10401 .checked_add(u64::try_from(cursor).unwrap_or(u64::MAX))
10402 .ok_or(Error::TimerDurationOverflow)?;
10403 let member_kind = selection_operation_kind(&operation).to_string();
10404 let member_shape = selection_operation_shape(&operation);
10405 let leaf_start = descriptors.len();
10406 match operation {
10407 ParallelOperation::Group(children) => {
10408 validate_parallel_operations(&children, &mut vec![member_index], false)?;
10409 for mut descriptor in parallel_descriptors(children, member_base)? {
10410 let flat_index = cursor + descriptor.offset;
10411 descriptor.group_path.insert(
10412 0,
10413 selection_group_entry(
10414 base_sequence,
10415 total_size,
10416 flat_index,
10417 group_kind,
10418 &SelectionMemberMetadata {
10419 key: key.clone(),
10420 index: member_index,
10421 base_sequence: member_base,
10422 size: member_size,
10423 kind: member_kind.clone(),
10424 },
10425 ),
10426 );
10427 descriptor.member_path.insert(0, member_index);
10428 descriptor.offset = flat_index;
10429 descriptors.push(descriptor);
10430 }
10431 }
10432 operation => {
10433 validate_parallel_operations(
10434 std::slice::from_ref(&operation),
10435 &mut Vec::new(),
10436 true,
10437 )?;
10438 descriptors.push(ParallelDescriptor {
10439 operation,
10440 offset: cursor,
10441 member_path: vec![member_index],
10442 group_path: vec![selection_group_entry(
10443 base_sequence,
10444 total_size,
10445 cursor,
10446 group_kind,
10447 &SelectionMemberMetadata {
10448 key: key.clone(),
10449 index: member_index,
10450 base_sequence: member_base,
10451 size: member_size,
10452 kind: member_kind.clone(),
10453 },
10454 )],
10455 });
10456 }
10457 }
10458 members.push(SelectionMemberPlan {
10459 key,
10460 index: member_index,
10461 base_sequence: member_base,
10462 size: member_size,
10463 kind: member_kind,
10464 shape: member_shape,
10465 leaf_start,
10466 });
10467 cursor += member_size;
10468 }
10469 Ok((descriptors, members))
10470}
10471
10472struct SelectionLeaf {
10473 call: ParallelLeafCall,
10474 outcome: Option<Result<ParallelAvroResult>>,
10475}
10476
10477#[derive(Clone)]
10479pub struct DurableOperationHandle {
10480 ctx: WorkflowContext,
10481 pub key: SelectionKey,
10482 pub index: usize,
10483 pub kind: String,
10484 pub identity: String,
10485 pub base_sequence: u64,
10486 pub size: usize,
10487 pub selection_group_id: String,
10488 shape: ParallelShape,
10489}
10490
10491impl std::fmt::Debug for DurableOperationHandle {
10492 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10493 formatter
10494 .debug_struct("DurableOperationHandle")
10495 .field("key", &self.key)
10496 .field("index", &self.index)
10497 .field("kind", &self.kind)
10498 .field("identity", &self.identity)
10499 .field("base_sequence", &self.base_sequence)
10500 .field("size", &self.size)
10501 .field("selection_group_id", &self.selection_group_id)
10502 .finish()
10503 }
10504}
10505
10506impl DurableOperationHandle {
10507 pub fn await_result(&self) -> DurableOperationAwaitCall {
10509 DurableOperationAwaitCall {
10510 handle: self.clone(),
10511 }
10512 }
10513
10514 pub fn cancel(&self) -> CancelDurableOperationCall {
10518 CancelDurableOperationCall {
10519 handle: self.clone(),
10520 emitted: false,
10521 }
10522 }
10523}
10524
10525#[derive(Debug)]
10527pub struct SelectionResult {
10528 pub key: SelectionKey,
10529 pub index: usize,
10530 pub kind: String,
10531 pub identity: String,
10532 pub value: Option<ParallelResult>,
10533 pub failure: Option<Error>,
10534 pub winner: DurableOperationHandle,
10535 pub handles: Vec<DurableOperationHandle>,
10536}
10537
10538impl SelectionResult {
10539 pub fn succeeded(&self) -> bool {
10540 self.failure.is_none()
10541 }
10542
10543 pub fn handle(&self, key: &SelectionKey) -> Option<&DurableOperationHandle> {
10544 self.handles.iter().find(|handle| &handle.key == key)
10545 }
10546
10547 pub fn remaining(&self) -> Vec<&DurableOperationHandle> {
10548 self.handles
10549 .iter()
10550 .filter(|handle| handle.index != self.index)
10551 .collect()
10552 }
10553
10554 pub fn into_result(self) -> Result<ParallelResult> {
10555 match (self.value, self.failure) {
10556 (Some(value), None) => Ok(value),
10557 (_, Some(error)) => Err(error),
10558 _ => Err(Error::WorkerLoop(
10559 "selection result contained neither a value nor a failure".to_string(),
10560 )),
10561 }
10562 }
10563}
10564
10565pub struct SelectCall {
10567 ctx: WorkflowContext,
10568 operations: Option<Vec<(SelectionKey, ParallelOperation)>>,
10569 members: Vec<SelectionMemberPlan>,
10570 leaves: Vec<SelectionLeaf>,
10571 group_id: Option<String>,
10572}
10573
10574impl SelectCall {
10575 fn new(ctx: WorkflowContext, operations: Vec<(SelectionKey, ParallelOperation)>) -> Self {
10576 Self {
10577 ctx,
10578 operations: Some(operations),
10579 members: Vec::new(),
10580 leaves: Vec::new(),
10581 group_id: None,
10582 }
10583 }
10584
10585 fn initialize(&mut self) -> Result<()> {
10586 let operations = self.operations.take().unwrap_or_default();
10587 let base_sequence = {
10588 let state = self
10589 .ctx
10590 .state
10591 .lock()
10592 .map_err(|_| Error::WorkflowStatePoisoned)?;
10593 if let Some(marker) = state.selection_markers.get(state.selection_marker_cursor) {
10594 marker.selection_group_base_sequence
10595 } else if let Some(recorded) = state.recorded_commands.get(state.command_cursor) {
10596 recorded.sequence()
10597 } else {
10598 let last = state
10599 .recorded_commands
10600 .last()
10601 .map(RecordedCommand::sequence)
10602 .unwrap_or(0);
10603 last.checked_add(u64::try_from(state.commands.len()).unwrap_or(u64::MAX))
10604 .and_then(|sequence| sequence.checked_add(1))
10605 .ok_or(Error::TimerDurationOverflow)?
10606 }
10607 };
10608 let (descriptors, members) = selection_descriptors(operations, base_sequence)?;
10609 let group_id = format!("select-calls:{base_sequence}:{}", descriptors.len());
10610 self.leaves = descriptors
10611 .into_iter()
10612 .map(|descriptor| SelectionLeaf {
10613 call: parallel_leaf_call(&self.ctx, descriptor.operation, descriptor.group_path),
10614 outcome: None,
10615 })
10616 .collect();
10617 self.members = members;
10618 self.group_id = Some(group_id);
10619 Ok(())
10620 }
10621}
10622
10623impl Future for SelectCall {
10624 type Output = Result<SelectionResult>;
10625
10626 fn poll(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
10627 if self.operations.is_some() {
10628 if let Err(error) = self.initialize() {
10629 return Poll::Ready(Err(error));
10630 }
10631 }
10632
10633 for leaf in &mut self.leaves {
10634 if leaf.outcome.is_some() {
10635 continue;
10636 }
10637 if let Poll::Ready(outcome) = leaf.call.poll_avro_value(cx) {
10638 if outcome
10639 .as_ref()
10640 .err()
10641 .is_some_and(workflow_task_integrity_error)
10642 {
10643 return Poll::Ready(outcome.map(|_| unreachable!()));
10644 }
10645 leaf.outcome = Some(outcome);
10646 }
10647 }
10648
10649 let all_members_terminal = self.leaves.iter().all(|leaf| leaf.outcome.is_some());
10650 let selection_member_range = self
10651 .members
10652 .first()
10653 .map(|member| member.base_sequence)
10654 .zip(self.leaves.len().try_into().ok())
10655 .map(|(base_sequence, size): (u64, u64)| {
10656 base_sequence..base_sequence.saturating_add(size)
10657 });
10658 let marker = {
10659 let mut state = match self.ctx.state.lock() {
10660 Ok(state) => state,
10661 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
10662 };
10663 let marker = state
10664 .selection_markers
10665 .get(state.selection_marker_cursor)
10666 .cloned();
10667 if marker.is_none()
10668 && all_members_terminal
10669 && selection_member_range.as_ref().is_some_and(|member_range| {
10670 state
10671 .recorded_commands
10672 .iter()
10673 .any(|command| member_range.contains(&command.sequence()))
10674 })
10675 {
10676 state.matched_recorded_pending = true;
10682 }
10683 marker
10684 };
10685 let Some(marker) = marker else {
10686 return Poll::Pending;
10687 };
10688 if self.group_id.as_deref() != Some(marker.selection_group_id.as_str())
10689 || marker.selection_group_size != self.leaves.len()
10690 || self.members.first().map(|member| member.base_sequence)
10691 != Some(marker.selection_group_base_sequence)
10692 {
10693 return Poll::Ready(Err(invalid_recorded_history(
10694 "selection_group_shape_mismatch",
10695 marker.selection_group_base_sequence,
10696 self.group_id
10697 .as_deref()
10698 .unwrap_or("initialized selection group"),
10699 &marker.selection_group_id,
10700 "recorded selection group differs from current workflow code",
10701 )));
10702 }
10703 let Some(member_position) = self.members.iter().position(|member| {
10704 member.key == marker.member_key
10705 && member.index == marker.member_index
10706 && member.base_sequence == marker.member_base_sequence
10707 && member.size == marker.member_size
10708 && member.kind == marker.operation_kind
10709 }) else {
10710 return Poll::Ready(Err(invalid_recorded_history(
10711 "selection_member_shape_mismatch",
10712 marker.member_base_sequence,
10713 "winner member matching current workflow code",
10714 &format!("{:?}", marker.member_key),
10715 "recorded selection winner differs from the authored member identity",
10716 )));
10717 };
10718 let member = self.members[member_position].clone();
10719 let (handles, resolution_sequence) = {
10720 let mut state = match self.ctx.state.lock() {
10721 Ok(state) => state,
10722 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
10723 };
10724 let identities = self
10725 .members
10726 .iter()
10727 .map(|candidate| {
10728 selection_operation_identity(
10729 &state,
10730 &candidate.kind,
10731 candidate.base_sequence,
10732 candidate.size,
10733 )
10734 })
10735 .collect::<Vec<_>>();
10736 if let Some((position, missing)) = identities
10737 .iter()
10738 .enumerate()
10739 .find(|(_, identity)| identity.is_empty())
10740 .map(|(position, identity)| (position, identity.clone()))
10741 {
10742 let candidate = &self.members[position];
10743 return Poll::Ready(Err(invalid_recorded_history(
10744 "selection_operation_identity_missing",
10745 candidate.base_sequence,
10746 &format!(
10747 "durable {} resource identity from scheduled/open history",
10748 candidate.kind
10749 ),
10750 &missing,
10751 "selection member history is missing its canonical durable identity",
10752 )));
10753 }
10754 let expected_winner_identity = &identities[member_position];
10755 let resolution_sequence = match validated_selection_resolution_sequence(
10756 &state,
10757 &marker,
10758 &member,
10759 expected_winner_identity,
10760 ) {
10761 Ok(sequence) => sequence,
10762 Err(error) => return Poll::Ready(Err(error)),
10763 };
10764 let handles = self
10765 .members
10766 .iter()
10767 .zip(identities)
10768 .map(|(member, identity)| DurableOperationHandle {
10769 ctx: self.ctx.clone(),
10770 key: member.key.clone(),
10771 index: member.index,
10772 kind: member.kind.clone(),
10773 identity,
10774 base_sequence: member.base_sequence,
10775 size: member.size,
10776 selection_group_id: marker.selection_group_id.clone(),
10777 shape: member.shape.clone(),
10778 })
10779 .collect::<Vec<_>>();
10780 if let Err(error) = validate_selection_cancellations_for_handles(&state, &handles) {
10781 return Poll::Ready(Err(error));
10782 }
10783 state.selection_marker_cursor += 1;
10784 (handles, resolution_sequence)
10785 };
10786
10787 let mut winner_failure = None;
10788 let mut flat_results = Vec::with_capacity(member.size);
10789 if marker.outcome == "failed" {
10790 let resolution_offset = match resolution_sequence
10791 .checked_sub(member.base_sequence)
10792 .and_then(|offset| usize::try_from(offset).ok())
10793 {
10794 Some(offset) if offset < member.size => offset,
10795 _ => {
10796 return Poll::Ready(Err(invalid_recorded_history(
10797 "selection_resolution_event_mismatch",
10798 member.base_sequence,
10799 "failure event within selected member bounds",
10800 &resolution_sequence.to_string(),
10801 "selection failure event is outside the authored member",
10802 )))
10803 }
10804 };
10805 let leaf = &mut self.leaves[member.leaf_start + resolution_offset];
10806 match leaf.outcome.take() {
10807 Some(Err(error)) => winner_failure = Some(error),
10808 _ => {
10809 return Poll::Ready(Err(invalid_recorded_history(
10810 "selection_winner_outcome_mismatch",
10811 member.base_sequence,
10812 "exact failed terminal history referenced by SelectionResolved",
10813 "missing or successful resolution event",
10814 "selection winner marker disagrees with terminal operation history",
10815 )))
10816 }
10817 }
10818 } else {
10819 for leaf in &mut self.leaves[member.leaf_start..member.leaf_start + member.size] {
10820 match leaf.outcome.take() {
10821 Some(Ok(result)) => flat_results.push(result),
10822 Some(Err(_)) => {
10823 return Poll::Ready(Err(invalid_recorded_history(
10824 "selection_winner_outcome_mismatch",
10825 member.base_sequence,
10826 "fully completed nested selection member",
10827 "failed durable leaf",
10828 "completed selection winner contains a failed leaf",
10829 )))
10830 }
10831 None => {
10832 return Poll::Ready(Err(invalid_recorded_history(
10833 "selection_winner_unresolved",
10834 member.base_sequence,
10835 "terminal history for every completed winner leaf",
10836 "pending member history",
10837 "completed SelectionResolved member has an unfinished durable barrier",
10838 )))
10839 }
10840 }
10841 }
10842 }
10843 let value = if winner_failure.is_none() {
10844 let mut flat_results = flat_results.into_iter();
10845 let value = parallel_results_for_shape(&member.shape, &mut flat_results);
10846 match value.into_json_result() {
10847 Ok(value) => Some(value),
10848 Err(error) => return Poll::Ready(Err(error)),
10849 }
10850 } else {
10851 None
10852 };
10853 let winner = handles[member_position].clone();
10854 Poll::Ready(Ok(SelectionResult {
10855 key: winner.key.clone(),
10856 index: winner.index,
10857 kind: winner.kind.clone(),
10858 identity: winner.identity.clone(),
10859 value,
10860 failure: winner_failure,
10861 winner,
10862 handles,
10863 }))
10864 }
10865}
10866
10867fn selection_operation_identity(
10868 state: &WorkflowState,
10869 kind: &str,
10870 base_sequence: u64,
10871 size: usize,
10872) -> String {
10873 if kind == "group" {
10874 return format!("group:{base_sequence}:{size}");
10875 }
10876 let fields: &[&str] = match kind {
10877 "activity" => &["activity_execution_id"],
10878 "child" => &["child_workflow_run_id"],
10879 "timer" => &["timer_id"],
10880 "signal" => &["signal_wait_id"],
10881 "condition" => &["condition_wait_id"],
10882 _ => &[],
10883 };
10884 for sequence in base_sequence..base_sequence.saturating_add(size as u64) {
10885 for event in state
10886 .history_events
10887 .iter()
10888 .filter(|event| durable_event_sequence(event) == Some(sequence))
10889 {
10890 for field in fields {
10891 if let Some(identity) = event.payload.get(*field).and_then(Value::as_str) {
10892 if !identity.is_empty() {
10893 return identity.to_string();
10894 }
10895 }
10896 }
10897 }
10898 }
10899 String::new()
10900}
10901
10902fn validated_selection_resolution_sequence(
10903 state: &WorkflowState,
10904 marker: &SelectionMarker,
10905 member: &SelectionMemberPlan,
10906 expected_identity: &str,
10907) -> Result<u64> {
10908 if expected_identity.is_empty() {
10909 return Err(invalid_recorded_history(
10910 "selection_operation_identity_missing",
10911 member.base_sequence,
10912 &format!(
10913 "durable {} resource identity from scheduled/open history",
10914 member.kind
10915 ),
10916 "missing operation identity",
10917 "selection member history is missing its canonical durable identity",
10918 ));
10919 }
10920 if marker.operation_identity != expected_identity {
10921 return Err(invalid_recorded_history(
10922 "selection_operation_identity_mismatch",
10923 member.base_sequence,
10924 expected_identity,
10925 &marker.operation_identity,
10926 "selection winner identity does not match durable scheduled/open history",
10927 ));
10928 }
10929
10930 let failure_types = [
10931 "ActivityFailed",
10932 "ActivityCancelled",
10933 "ActivityTimedOut",
10934 "ChildRunFailed",
10935 "ChildRunCancelled",
10936 "ChildRunTerminated",
10937 ];
10938 let success_types = [
10939 "ActivityCompleted",
10940 "ChildRunCompleted",
10941 "TimerFired",
10942 "SignalApplied",
10943 "ConditionWaitSatisfied",
10944 "ConditionWaitTimedOut",
10945 ];
10946 let terminal_types: &[&str] = if marker.outcome == "failed" {
10947 &failure_types
10948 } else {
10949 &success_types
10950 };
10951 let mut candidates = Vec::new();
10952 for event in state.history_events.iter() {
10953 let Some(sequence) = durable_event_sequence(event) else {
10954 continue;
10955 };
10956 if sequence < member.base_sequence
10957 || sequence >= member.base_sequence.saturating_add(member.size as u64)
10958 || !terminal_types.contains(&event.event_type.as_str())
10959 {
10960 continue;
10961 }
10962 let event_id = event
10963 .raw
10964 .get("id")
10965 .or_else(|| event.raw.get("event_id"))
10966 .and_then(Value::as_str)
10967 .filter(|value| !value.is_empty())
10968 .ok_or_else(|| {
10969 invalid_recorded_history(
10970 "selection_resolution_event_id_missing",
10971 member.base_sequence,
10972 "terminal selection history with a durable event id",
10973 &event.payload.to_string(),
10974 "selection terminal history cannot be bound to its winner marker",
10975 )
10976 })?;
10977 candidates.push((event_id.to_string(), event.event_type.clone(), sequence));
10978 }
10979 let resolution = if marker.outcome == "failed" {
10980 candidates.first()
10981 } else {
10982 candidates.last()
10983 };
10984 let Some((event_id, event_type, sequence)) = resolution else {
10985 return Err(invalid_recorded_history(
10986 "selection_resolution_event_missing",
10987 member.base_sequence,
10988 "terminal history for the selected member",
10989 &format!("{:?}", marker.member_key),
10990 "selection winner marker has no matching durable terminal event",
10991 ));
10992 };
10993 if event_id != &marker.resolution_event_id || event_type != &marker.resolution_event_type {
10994 return Err(invalid_recorded_history(
10995 "selection_resolution_event_mismatch",
10996 member.base_sequence,
10997 &format!("{event_type}:{event_id}"),
10998 &format!(
10999 "{}:{}",
11000 marker.resolution_event_type, marker.resolution_event_id
11001 ),
11002 "selection winner marker does not reference the event that made its member terminal",
11003 ));
11004 }
11005 Ok(*sequence)
11006}
11007
11008fn recorded_selection_member_outcome(
11009 state: &WorkflowState,
11010 handle: &DurableOperationHandle,
11011) -> Result<Option<ParallelResult>> {
11012 for event in state.history_events.iter() {
11013 let Some(sequence) = durable_event_sequence(event) else {
11014 continue;
11015 };
11016 if sequence < handle.base_sequence
11017 || sequence >= handle.base_sequence.saturating_add(handle.size as u64)
11018 || !matches!(
11019 event.event_type.as_str(),
11020 "ActivityFailed"
11021 | "ActivityCancelled"
11022 | "ActivityTimedOut"
11023 | "ChildRunFailed"
11024 | "ChildRunCancelled"
11025 | "ChildRunTerminated"
11026 )
11027 {
11028 continue;
11029 }
11030 let Some(command) = state
11031 .recorded_commands
11032 .iter()
11033 .find(|command| command.sequence() == sequence)
11034 else {
11035 continue;
11036 };
11037 match command {
11038 RecordedCommand::Activity {
11039 outcome: Some(Err(failure)),
11040 ..
11041 } => return Err(Error::ActivityFailed(failure.clone())),
11042 RecordedCommand::ChildWorkflow {
11043 outcome: Some(Err(failure)),
11044 ..
11045 } => return Err(Error::ChildWorkflowFailed(failure.clone())),
11046 _ => {}
11047 }
11048 }
11049
11050 let mut results = Vec::with_capacity(handle.size);
11051 for sequence in handle.base_sequence..handle.base_sequence.saturating_add(handle.size as u64) {
11052 let Some(command) = state
11053 .recorded_commands
11054 .iter()
11055 .find(|command| command.sequence() == sequence)
11056 else {
11057 return Ok(None);
11058 };
11059 let result = match command {
11060 RecordedCommand::Activity { outcome, .. } => match outcome {
11061 Some(Ok(value)) => ParallelAvroResult::Activity(value.clone()),
11062 Some(Err(failure)) => return Err(Error::ActivityFailed(failure.clone())),
11063 None => return Ok(None),
11064 },
11065 RecordedCommand::Timer { fired, .. } => {
11066 if !fired {
11067 return Ok(None);
11068 }
11069 ParallelAvroResult::Timer
11070 }
11071 RecordedCommand::ChildWorkflow { outcome, .. } => match outcome {
11072 Some(Ok(value)) => ParallelAvroResult::ChildWorkflow(value.clone()),
11073 Some(Err(failure)) => return Err(Error::ChildWorkflowFailed(failure.clone())),
11074 None => return Ok(None),
11075 },
11076 RecordedCommand::SignalWait { value, .. } => match value {
11077 Some(value) => ParallelAvroResult::Signal(value.clone()),
11078 None => return Ok(None),
11079 },
11080 RecordedCommand::ConditionWait { result, .. } => match result {
11081 Some(result) => ParallelAvroResult::Condition(*result),
11082 None => return Ok(None),
11083 },
11084 other => {
11085 return Err(command_mismatch(
11086 other,
11087 format!("selected {} member", handle.kind),
11088 ))
11089 }
11090 };
11091 results.push(result);
11092 }
11093 let mut results = results.into_iter();
11094 parallel_results_for_shape(&handle.shape, &mut results)
11095 .into_json_result()
11096 .map(Some)
11097}
11098
11099fn recorded_selection_member_is_terminal(
11100 state: &WorkflowState,
11101 handle: &DurableOperationHandle,
11102) -> bool {
11103 let mut completed = 0usize;
11104 let mut all_completed = true;
11105 for sequence in handle.base_sequence..handle.base_sequence.saturating_add(handle.size as u64) {
11106 let Some(command) = state
11107 .recorded_commands
11108 .iter()
11109 .find(|command| command.sequence() == sequence)
11110 else {
11111 all_completed = false;
11112 continue;
11113 };
11114 let terminal = match command {
11115 RecordedCommand::Activity {
11116 outcome: Some(Err(_)),
11117 ..
11118 }
11119 | RecordedCommand::ChildWorkflow {
11120 outcome: Some(Err(_)),
11121 ..
11122 } => return true,
11123 RecordedCommand::Activity { outcome, .. } => outcome.is_some(),
11124 RecordedCommand::ChildWorkflow { outcome, .. } => outcome.is_some(),
11125 RecordedCommand::Timer { fired, .. } => *fired,
11126 RecordedCommand::SignalWait { value, .. } => value.is_some(),
11127 RecordedCommand::ConditionWait { result, .. } => result.is_some(),
11128 RecordedCommand::SearchAttributes { .. }
11129 | RecordedCommand::SideEffect { .. }
11130 | RecordedCommand::VersionMarker { .. }
11131 | RecordedCommand::Memo { .. } => false,
11132 };
11133 if !terminal {
11134 all_completed = false;
11135 continue;
11136 }
11137 completed += 1;
11138 }
11139 all_completed && completed == handle.size
11140}
11141
11142fn selection_cancellation_for_handle(
11143 state: &WorkflowState,
11144 handle: &DurableOperationHandle,
11145) -> Result<bool> {
11146 let Some(marker) = state.cancelled_selection_members.iter().find(|recorded| {
11147 recorded.selection_group_id == handle.selection_group_id
11148 && recorded.member_base_sequence == handle.base_sequence
11149 }) else {
11150 return Ok(false);
11151 };
11152 validate_selection_cancellation_marker(marker, handle)?;
11153 Ok(true)
11154}
11155
11156fn validate_selection_cancellations_for_handles(
11157 state: &WorkflowState,
11158 handles: &[DurableOperationHandle],
11159) -> Result<()> {
11160 let Some(group_id) = handles
11161 .first()
11162 .map(|handle| handle.selection_group_id.as_str())
11163 else {
11164 return Ok(());
11165 };
11166 for marker in state
11167 .cancelled_selection_members
11168 .iter()
11169 .filter(|marker| marker.selection_group_id == group_id)
11170 {
11171 let Some(handle) = handles
11172 .iter()
11173 .find(|handle| handle.base_sequence == marker.member_base_sequence)
11174 else {
11175 return Err(invalid_recorded_history(
11176 "selection_cancellation_member_mismatch",
11177 marker.member_base_sequence,
11178 "SelectionOperationCancelled matching an authored selection handle",
11179 &format!("{marker:?}"),
11180 "selection cancellation member base does not name an authored member",
11181 ));
11182 };
11183 validate_selection_cancellation_marker(marker, handle)?;
11184 }
11185 Ok(())
11186}
11187
11188fn validate_selection_cancellation_marker(
11189 marker: &SelectionCancellation,
11190 handle: &DurableOperationHandle,
11191) -> Result<()> {
11192 if marker.selection_group_id != handle.selection_group_id
11193 || marker.member_key != handle.key
11194 || marker.member_index != handle.index
11195 || marker.member_base_sequence != handle.base_sequence
11196 || marker.member_size != handle.size
11197 || marker.operation_kind != handle.kind
11198 || marker.operation_identity != handle.identity
11199 {
11200 return Err(invalid_recorded_history(
11201 "selection_cancellation_member_mismatch",
11202 handle.base_sequence,
11203 "SelectionOperationCancelled matching the authored selection handle",
11204 &format!("{marker:?}"),
11205 "selection cancellation history targets different authored member metadata",
11206 ));
11207 }
11208 Ok(())
11209}
11210
11211pub struct DurableOperationAwaitCall {
11213 handle: DurableOperationHandle,
11214}
11215
11216impl Future for DurableOperationAwaitCall {
11217 type Output = Result<ParallelResult>;
11218
11219 fn poll(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11220 let state = match self.handle.ctx.state.lock() {
11221 Ok(state) => state,
11222 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11223 };
11224 match selection_cancellation_for_handle(&state, &self.handle) {
11225 Err(error) => return Poll::Ready(Err(error)),
11226 Ok(false) => {}
11227 Ok(true) => {
11228 return Poll::Ready(Err(Error::DurableOperationCancelled(
11229 DurableOperationCancelled {
11230 selection_group_id: self.handle.selection_group_id.clone(),
11231 member_key: self.handle.key.clone(),
11232 member_index: self.handle.index,
11233 operation_kind: self.handle.kind.clone(),
11234 operation_identity: self.handle.identity.clone(),
11235 },
11236 )));
11237 }
11238 }
11239 match recorded_selection_member_outcome(&state, &self.handle) {
11240 Ok(Some(result)) => Poll::Ready(Ok(result)),
11241 Ok(None) => Poll::Pending,
11242 Err(error) => Poll::Ready(Err(error)),
11243 }
11244 }
11245}
11246
11247pub struct CancelDurableOperationCall {
11249 handle: DurableOperationHandle,
11250 emitted: bool,
11251}
11252
11253impl Future for CancelDurableOperationCall {
11254 type Output = Result<()>;
11255
11256 fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11257 let ctx = self.handle.ctx.clone();
11258 let mut state = match ctx.state.lock() {
11259 Ok(state) => state,
11260 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11261 };
11262 match selection_cancellation_for_handle(&state, &self.handle) {
11263 Err(error) => return Poll::Ready(Err(error)),
11264 Ok(true) => return Poll::Ready(Ok(())),
11265 Ok(false) => {}
11266 }
11267 if recorded_selection_member_is_terminal(&state, &self.handle) {
11268 return Poll::Ready(Ok(()));
11269 }
11270 if !self.emitted {
11271 state.commands.push(json!({
11272 "type": "cancel_selection_operation",
11273 "selection_group_id": self.handle.selection_group_id,
11274 "member_key": self.handle.key,
11275 "member_index": self.handle.index,
11276 "member_base_sequence": self.handle.base_sequence,
11277 "member_size": self.handle.size,
11278 "operation_kind": self.handle.kind,
11279 "operation_identity": self.handle.identity,
11280 }));
11281 self.emitted = true;
11282 }
11283 Poll::Pending
11287 }
11288}
11289
11290trait PollNestedResultExt<T> {
11291 fn flatten_result(self) -> Poll<Result<T>>;
11292}
11293
11294impl<T> PollNestedResultExt<T> for Poll<Result<Result<T>>> {
11295 fn flatten_result(self) -> Poll<Result<T>> {
11296 match self {
11297 Poll::Ready(Ok(result)) => Poll::Ready(result),
11298 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
11299 Poll::Pending => Poll::Pending,
11300 }
11301 }
11302}
11303
11304struct SagaCompensation {
11305 activity_type: String,
11306 options: ActivityOptions,
11307 arguments: AvroValue,
11308 registration_order: usize,
11309}
11310
11311pub struct Saga {
11319 ctx: WorkflowContext,
11320 compensations: Vec<SagaCompensation>,
11321}
11322
11323impl Saga {
11324 fn new(ctx: WorkflowContext) -> Self {
11325 Self {
11326 ctx,
11327 compensations: Vec::new(),
11328 }
11329 }
11330
11331 pub fn add_compensation<T: Serialize>(
11332 &mut self,
11333 activity_type: impl Into<String>,
11334 args: T,
11335 ) -> Result<&mut Self> {
11336 self.add_compensation_with_options(activity_type, ActivityOptions::new(), args)
11337 }
11338
11339 pub fn add_compensation_with_options<T: Serialize>(
11340 &mut self,
11341 activity_type: impl Into<String>,
11342 options: ActivityOptions,
11343 args: T,
11344 ) -> Result<&mut Self> {
11345 let activity_type = activity_type.into();
11346 if activity_type.trim().is_empty() || activity_type.trim() != activity_type {
11347 return Err(Error::Codec(
11348 "saga compensation activity type must be non-empty without surrounding whitespace"
11349 .to_string(),
11350 ));
11351 }
11352 options.validate().map_err(Error::InvalidActivityOptions)?;
11353 let arguments = AvroValue::from_serialize(&args)?;
11354 let registration_order = self.compensations.len() + 1;
11355 self.compensations.push(SagaCompensation {
11356 activity_type,
11357 options,
11358 arguments,
11359 registration_order,
11360 });
11361 Ok(self)
11362 }
11363
11364 pub async fn compensate(mut self, initiating_failure: Error) -> Error {
11366 while let Some(compensation) = self.compensations.pop() {
11367 if let Err(compensation_failure) = self
11368 .ctx
11369 .activity_with_options(
11370 compensation.activity_type.clone(),
11371 compensation.options,
11372 compensation.arguments,
11373 )
11374 .await
11375 {
11376 if workflow_task_integrity_error(&compensation_failure) {
11377 return compensation_failure;
11378 }
11379 return Error::SagaCompensationFailed(SagaCompensationFailure {
11380 initiating_failure: Box::new(initiating_failure),
11381 compensation_failure: Box::new(compensation_failure),
11382 compensation_activity_type: compensation.activity_type,
11383 compensation_registration_order: compensation.registration_order,
11384 });
11385 }
11386 }
11387 initiating_failure
11388 }
11389
11390 pub async fn finish<T>(self, outcome: Result<T>) -> Result<T> {
11392 match outcome {
11393 Ok(value) => Ok(value),
11394 Err(error) => Err(self.compensate(error).await),
11395 }
11396 }
11397}
11398
11399pub struct ActivityCall {
11400 ctx: WorkflowContext,
11401 activity_type: String,
11402 options: ActivityOptions,
11403 args: Option<Result<AvroValue>>,
11404 scheduled: bool,
11405 parallel_group_path: Vec<ParallelGroupMetadata>,
11406}
11407
11408impl ActivityCall {
11409 fn poll_avro_value(
11410 mut self: Pin<&mut Self>,
11411 _cx: &mut TaskContext<'_>,
11412 ) -> Poll<Result<AvroValue>> {
11413 let ctx = self.ctx.clone();
11414 let mut state = match ctx.state.lock() {
11415 Ok(state) => state,
11416 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11417 };
11418
11419 if self.scheduled {
11420 return Poll::Pending;
11421 }
11422
11423 let options = match self.options.validate() {
11424 Ok(options) => options,
11425 Err(error) => {
11426 return Poll::Ready(Err(Error::InvalidActivityOptions(error)));
11427 }
11428 };
11429 let task_queue = options
11430 .task_queue
11431 .clone()
11432 .unwrap_or_else(|| state.task_queue.clone());
11433 let current_recorded_options = RecordedActivityOptions {
11434 task_queue: RecordedSnapshotValue::Known(Some(task_queue.clone())),
11435 execution_mode: RecordedSnapshotValue::Known(None),
11438 retry_policy: current_activity_retry_snapshot(&options),
11439 };
11440
11441 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
11442 let sequence = recorded.sequence();
11443 match recorded {
11444 RecordedCommand::Activity {
11445 activity_type,
11446 options: recorded_options,
11447 outcome,
11448 parallel_group_path,
11449 ..
11450 } => {
11451 if let Err(error) = ensure_parallel_path_matches(
11452 sequence,
11453 parallel_group_path.as_deref(),
11454 &self.parallel_group_path,
11455 ) {
11456 return Poll::Ready(Err(error));
11457 }
11458 if let Some(recorded_type) = activity_type {
11459 if recorded_type != self.activity_type {
11460 return Poll::Ready(Err(Error::NonDeterministicReplay(
11461 ReplayFailure::new(
11462 "recorded_command_detail_mismatch",
11463 Some(sequence),
11464 Some(format!("activity:{recorded_type}")),
11465 Some(format!("activity:{}", self.activity_type)),
11466 "recorded activity type differs from the current workflow command",
11467 ),
11468 )));
11469 }
11470 }
11471 if let Some(recorded_options) = recorded_options {
11472 if !recorded_options
11473 .task_queue
11474 .matches_current(¤t_recorded_options.task_queue)
11475 {
11476 return Poll::Ready(Err(Error::NonDeterministicReplay(
11477 ReplayFailure::new(
11478 "activity_task_queue_mismatch",
11479 Some(sequence),
11480 Some(activity_options_description(&recorded_options)),
11481 Some(activity_options_description(¤t_recorded_options)),
11482 "recorded activity task queue differs from the current workflow command",
11483 ),
11484 )));
11485 }
11486 if !recorded_options
11487 .execution_mode
11488 .matches_current(¤t_recorded_options.execution_mode)
11489 {
11490 return Poll::Ready(Err(Error::NonDeterministicReplay(
11491 ReplayFailure::new(
11492 "activity_execution_mode_mismatch",
11493 Some(sequence),
11494 Some(activity_options_description(&recorded_options)),
11495 Some(activity_options_description(¤t_recorded_options)),
11496 "recorded activity execution mode differs from the current workflow command",
11497 ),
11498 )));
11499 }
11500 if !recorded_options
11501 .retry_policy
11502 .matches_current(¤t_recorded_options.retry_policy)
11503 {
11504 return Poll::Ready(Err(Error::NonDeterministicReplay(
11505 ReplayFailure::new(
11506 "activity_retry_policy_mismatch",
11507 Some(sequence),
11508 Some(activity_options_description(&recorded_options)),
11509 Some(activity_options_description(¤t_recorded_options)),
11510 "recorded activity retry policy differs from the current workflow command",
11511 ),
11512 )));
11513 }
11514 }
11515 state.command_cursor += 1;
11516 if let Some(outcome) = outcome {
11517 return Poll::Ready(outcome.map_err(Error::ActivityFailed));
11518 }
11519 state.matched_recorded_pending = true;
11520 self.scheduled = true;
11521 return Poll::Pending;
11522 }
11523 other => {
11524 return Poll::Ready(Err(command_mismatch(
11525 &other,
11526 format!("activity:{}", self.activity_type),
11527 )));
11528 }
11529 }
11530 }
11531
11532 if !self.scheduled {
11533 let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
11534 Ok(args) => args,
11535 Err(error) => return Poll::Ready(Err(error)),
11536 };
11537 let arguments = normalize_avro_arguments(args);
11538 let envelope = match encode_typed_envelope(&arguments, &state.payload_codec) {
11539 Ok(envelope) => envelope,
11540 Err(error) => return Poll::Ready(Err(error)),
11541 };
11542
11543 let mut command = serde_json::Map::from_iter([
11544 ("type".to_string(), json!("schedule_activity")),
11545 (
11546 "activity_type".to_string(),
11547 json!(self.activity_type.clone()),
11548 ),
11549 ("queue".to_string(), json!(task_queue)),
11550 ("arguments".to_string(), envelope),
11551 ]);
11552 for (field, value) in [
11553 ("start_to_close_timeout", options.start_to_close_timeout),
11554 (
11555 "schedule_to_start_timeout",
11556 options.schedule_to_start_timeout,
11557 ),
11558 (
11559 "schedule_to_close_timeout",
11560 options.schedule_to_close_timeout,
11561 ),
11562 ("heartbeat_timeout", options.heartbeat_timeout),
11563 ] {
11564 if let Some(value) = value {
11565 command.insert(field.to_string(), json!(value));
11566 }
11567 }
11568 if let Some(retry_policy) = options.retry_policy {
11569 command.insert("retry_policy".to_string(), retry_policy);
11570 }
11571 apply_parallel_group_path(&mut command, &self.parallel_group_path);
11572 state.commands.push(Value::Object(command));
11573 self.scheduled = true;
11574 }
11575
11576 Poll::Pending
11577 }
11578}
11579
11580impl Future for ActivityCall {
11581 type Output = Result<Value>;
11582
11583 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11584 match self.poll_avro_value(cx) {
11585 Poll::Ready(Ok(value)) => Poll::Ready(value.into_json()),
11586 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
11587 Poll::Pending => Poll::Pending,
11588 }
11589 }
11590}
11591
11592pub struct TimerCall {
11594 ctx: WorkflowContext,
11595 delay_seconds: Option<u64>,
11596 scheduled: bool,
11597 matched_pending: bool,
11598 parallel_group_path: Vec<ParallelGroupMetadata>,
11599}
11600
11601impl Future for TimerCall {
11602 type Output = Result<()>;
11603
11604 fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11605 if self.matched_pending {
11606 return Poll::Pending;
11607 }
11608
11609 let ctx = self.ctx.clone();
11610 let Some(requested_delay) = self.delay_seconds else {
11611 return Poll::Ready(Err(Error::TimerDurationOverflow));
11612 };
11613 let mut state = match ctx.state.lock() {
11614 Ok(state) => state,
11615 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11616 };
11617
11618 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
11619 match recorded {
11620 RecordedCommand::Timer {
11621 sequence,
11622 delay_seconds,
11623 fired,
11624 parallel_group_path,
11625 ..
11626 } => {
11627 if let Err(error) = ensure_parallel_path_matches(
11628 sequence,
11629 parallel_group_path.as_deref(),
11630 &self.parallel_group_path,
11631 ) {
11632 return Poll::Ready(Err(error));
11633 }
11634 if delay_seconds != requested_delay {
11635 return Poll::Ready(Err(Error::NonDeterministicReplay(
11636 ReplayFailure::new(
11637 "timer_delay_mismatch",
11638 Some(sequence),
11639 Some(format!("timer:{delay_seconds}s")),
11640 Some(format!("timer:{requested_delay}s")),
11641 "recorded timer delay differs from the current workflow command",
11642 ),
11643 )));
11644 }
11645 state.command_cursor += 1;
11646 if fired {
11647 return Poll::Ready(Ok(()));
11648 }
11649 state.matched_recorded_pending = true;
11650 self.scheduled = true;
11651 self.matched_pending = true;
11652 return Poll::Pending;
11653 }
11654 other => return Poll::Ready(Err(command_mismatch(&other, "timer"))),
11655 }
11656 }
11657
11658 if !self.scheduled {
11659 let mut command = serde_json::Map::from_iter([
11660 ("type".to_string(), json!("start_timer")),
11661 ("delay_seconds".to_string(), json!(requested_delay)),
11662 ]);
11663 apply_parallel_group_path(&mut command, &self.parallel_group_path);
11664 state.commands.push(Value::Object(command));
11665 self.scheduled = true;
11666 }
11667
11668 Poll::Pending
11669 }
11670}
11671
11672pub struct ConditionWaitCall {
11674 ctx: WorkflowContext,
11675 options: ConditionWaitOptions,
11676 predicate: Box<dyn Fn() -> Result<bool> + Send + 'static>,
11677 occurrence_id: Option<String>,
11678 opened_wait: bool,
11679 parallel_group_path: Vec<ParallelGroupMetadata>,
11680}
11681
11682impl Future for ConditionWaitCall {
11683 type Output = Result<ConditionWaitResult>;
11684
11685 fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11686 if self.opened_wait {
11687 return Poll::Pending;
11688 }
11689
11690 let options = match self.options.validate() {
11691 Ok(options) => options,
11692 Err(error) => return Poll::Ready(Err(Error::InvalidConditionWaitOptions(error))),
11693 };
11694 let ctx = self.ctx.clone();
11695 let occurrence_id = match self.occurrence_id.as_ref() {
11696 Some(occurrence_id) => occurrence_id.clone(),
11697 None => {
11698 let mut state = match ctx.state.lock() {
11699 Ok(state) => state,
11700 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11701 };
11702 let ordinal = state.condition_wait_occurrence_counter;
11703 state.condition_wait_occurrence_counter = match ordinal.checked_add(1) {
11704 Some(next) => next,
11705 None => {
11706 return Poll::Ready(Err(Error::WorkerLoop(
11707 "condition wait occurrence counter overflowed".to_string(),
11708 )))
11709 }
11710 };
11711 let occurrence_id = format!("{CONDITION_WAIT_OCCURRENCE_PREFIX}{ordinal}");
11712 drop(state);
11713 self.occurrence_id = Some(occurrence_id.clone());
11714 occurrence_id
11715 }
11716 };
11717
11718 let recorded_result = {
11719 let mut state = match ctx.state.lock() {
11720 Ok(state) => state,
11721 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11722 };
11723 let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() else {
11724 drop(state);
11725 return self.poll_new_condition(options);
11726 };
11727 if !matches!(recorded, RecordedCommand::ConditionWait { .. }) {
11728 return Poll::Ready(Err(command_mismatch(&recorded, "condition wait")));
11729 }
11730
11731 let mut cursor = state.command_cursor;
11732 let mut result = None;
11733 loop {
11734 let Some(RecordedCommand::ConditionWait {
11735 sequence,
11736 occurrence_id: recorded_occurrence_id,
11737 condition_key,
11738 predicate_identity,
11739 timeout_seconds,
11740 result: recorded_result,
11741 parallel_group_path,
11742 ..
11743 }) = state.recorded_commands.get(cursor)
11744 else {
11745 break;
11746 };
11747
11748 if cursor > state.command_cursor && recorded_occurrence_id != &occurrence_id {
11749 break;
11750 }
11751 if let Err(error) = ensure_parallel_path_matches(
11752 *sequence,
11753 parallel_group_path.as_deref(),
11754 &self.parallel_group_path,
11755 ) {
11756 return Poll::Ready(Err(error));
11757 }
11758 if let Err(error) = validate_recorded_condition_wait(
11759 *sequence,
11760 recorded_occurrence_id,
11761 condition_key.as_deref(),
11762 predicate_identity,
11763 *timeout_seconds,
11764 &occurrence_id,
11765 &options,
11766 ) {
11767 return Poll::Ready(Err(error));
11768 }
11769 if result == Some(ConditionWaitResult::TimedOut) {
11770 return Poll::Ready(Err(Error::NonDeterministicReplay(ReplayFailure::new(
11771 "condition_wait_reopened_after_timeout",
11772 Some(*sequence),
11773 Some("timed-out condition is terminal".to_string()),
11774 Some("another physical wait-open".to_string()),
11775 "condition history reopened one logical wait after its durable timeout",
11776 ))));
11777 }
11778 result = *recorded_result;
11779 cursor += 1;
11780 }
11781 state.command_cursor = cursor;
11782 result
11783 };
11784
11785 if let Some(result) = recorded_result {
11786 return Poll::Ready(Ok(result));
11787 }
11788
11789 self.poll_open_condition(options)
11790 }
11791}
11792
11793impl ConditionWaitCall {
11794 fn poll_new_condition(
11795 self: Pin<&mut Self>,
11796 options: ValidatedConditionWaitOptions,
11797 ) -> Poll<Result<ConditionWaitResult>> {
11798 self.poll_open_condition(options)
11799 }
11800
11801 fn poll_open_condition(
11802 mut self: Pin<&mut Self>,
11803 options: ValidatedConditionWaitOptions,
11804 ) -> Poll<Result<ConditionWaitResult>> {
11805 let selection_member = self
11806 .parallel_group_path
11807 .first()
11808 .is_some_and(|entry| entry.parallel_group_mode.as_deref() == Some("select"));
11809 match (self.predicate)() {
11810 Ok(true) if !selection_member => {
11811 return Poll::Ready(Ok(ConditionWaitResult::Satisfied))
11812 }
11813 Ok(_) => {}
11814 Err(error) => return Poll::Ready(Err(error)),
11815 }
11816 if options.timeout_seconds == Some(0) && !selection_member {
11817 return Poll::Ready(Ok(ConditionWaitResult::TimedOut));
11818 }
11819
11820 let ctx = self.ctx.clone();
11821 let mut state = match ctx.state.lock() {
11822 Ok(state) => state,
11823 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11824 };
11825 let mut command = serde_json::Map::from_iter([
11826 ("type".to_string(), json!("open_condition_wait")),
11827 (
11828 "condition_wait_occurrence_id".to_string(),
11829 json!(self.occurrence_id.as_deref().unwrap_or_default()),
11830 ),
11831 ("condition_key".to_string(), json!(options.condition_key)),
11832 (
11833 "condition_definition_fingerprint".to_string(),
11834 json!(options.predicate_identity),
11835 ),
11836 ]);
11837 if let Some(timeout_seconds) = options.timeout_seconds {
11838 command.insert("timeout_seconds".to_string(), json!(timeout_seconds));
11839 }
11840 apply_parallel_group_path(&mut command, &self.parallel_group_path);
11841 state.commands.push(Value::Object(command));
11842 drop(state);
11843 self.opened_wait = true;
11844 Poll::Pending
11845 }
11846}
11847
11848fn validate_recorded_condition_wait(
11849 sequence: u64,
11850 recorded_occurrence_id: &str,
11851 recorded_key: Option<&str>,
11852 recorded_predicate_identity: &str,
11853 recorded_timeout_seconds: Option<u64>,
11854 current_occurrence_id: &str,
11855 current: &ValidatedConditionWaitOptions,
11856) -> Result<()> {
11857 if recorded_occurrence_id != current_occurrence_id {
11858 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
11859 "condition_wait_occurrence_mismatch",
11860 Some(sequence),
11861 Some(recorded_occurrence_id.to_string()),
11862 Some(current_occurrence_id.to_string()),
11863 "recorded condition occurrence differs from the current authored wait position",
11864 )));
11865 }
11866 if recorded_key != Some(current.condition_key.as_str()) {
11867 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
11868 "condition_wait_key_mismatch",
11869 Some(sequence),
11870 recorded_key.map(str::to_string),
11871 Some(current.condition_key.clone()),
11872 "recorded condition identity differs from the current workflow wait",
11873 )));
11874 }
11875 if recorded_predicate_identity != current.predicate_identity {
11876 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
11877 "condition_wait_predicate_mismatch",
11878 Some(sequence),
11879 Some(recorded_predicate_identity.to_string()),
11880 Some(current.predicate_identity.clone()),
11881 "recorded condition predicate behavior differs from current workflow code",
11882 )));
11883 }
11884 if recorded_timeout_seconds != current.timeout_seconds {
11885 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
11886 "condition_wait_timeout_mismatch",
11887 Some(sequence),
11888 recorded_timeout_seconds.map(|seconds| format!("{seconds}s")),
11889 current.timeout_seconds.map(|seconds| format!("{seconds}s")),
11890 "recorded condition timeout differs from the current workflow wait",
11891 )));
11892 }
11893 Ok(())
11894}
11895
11896pub struct ChildWorkflowCall {
11898 ctx: WorkflowContext,
11899 workflow_type: String,
11900 options: ChildWorkflowOptions,
11901 args: Option<Result<AvroValue>>,
11902 scheduled: bool,
11903 matched_pending: bool,
11904 parallel_group_path: Vec<ParallelGroupMetadata>,
11905}
11906
11907impl ChildWorkflowCall {
11908 fn poll_avro_value(
11909 mut self: Pin<&mut Self>,
11910 _cx: &mut TaskContext<'_>,
11911 ) -> Poll<Result<ChildWorkflowAvroResult>> {
11912 if self.matched_pending {
11913 return Poll::Pending;
11914 }
11915
11916 let ctx = self.ctx.clone();
11917 let mut state = match ctx.state.lock() {
11918 Ok(state) => state,
11919 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11920 };
11921
11922 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
11923 let sequence = recorded.sequence();
11924 match recorded {
11925 RecordedCommand::ChildWorkflow {
11926 workflow_type,
11927 outcome,
11928 parallel_group_path,
11929 ..
11930 } => {
11931 if let Err(error) = ensure_parallel_path_matches(
11932 sequence,
11933 parallel_group_path.as_deref(),
11934 &self.parallel_group_path,
11935 ) {
11936 return Poll::Ready(Err(error));
11937 }
11938 if let Some(recorded_type) = workflow_type {
11939 if recorded_type != self.workflow_type {
11940 return Poll::Ready(Err(Error::NonDeterministicReplay(
11941 ReplayFailure::new(
11942 "recorded_command_detail_mismatch",
11943 Some(sequence),
11944 Some(format!("child workflow:{recorded_type}")),
11945 Some(format!("child workflow:{}", self.workflow_type)),
11946 "recorded child workflow type differs from the current workflow command",
11947 ),
11948 )));
11949 }
11950 }
11951 state.command_cursor += 1;
11952 if let Some(outcome) = outcome {
11953 return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed));
11954 }
11955 state.matched_recorded_pending = true;
11956 self.scheduled = true;
11957 self.matched_pending = true;
11958 return Poll::Pending;
11959 }
11960 other => {
11961 return Poll::Ready(Err(command_mismatch(
11962 &other,
11963 format!("child workflow:{}", self.workflow_type),
11964 )));
11965 }
11966 }
11967 }
11968
11969 if !self.scheduled {
11970 if self.options.task_queue.trim().is_empty() {
11971 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
11972 "task_queue must not be empty".to_string(),
11973 )));
11974 }
11975 for (name, value) in [
11976 (
11977 "execution_timeout_seconds",
11978 self.options.execution_timeout_seconds,
11979 ),
11980 ("run_timeout_seconds", self.options.run_timeout_seconds),
11981 ] {
11982 if value == Some(0) {
11983 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(format!(
11984 "{name} must be at least 1"
11985 ))));
11986 }
11987 }
11988
11989 let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
11990 Ok(args) => args,
11991 Err(error) => return Poll::Ready(Err(error)),
11992 };
11993 let arguments = match encode_typed_envelope(
11994 &normalize_avro_arguments(args),
11995 &state.payload_codec,
11996 ) {
11997 Ok(arguments) => arguments,
11998 Err(error) => return Poll::Ready(Err(error)),
11999 };
12000 let mut command = json!({
12001 "type": "start_child_workflow",
12002 "workflow_type": self.workflow_type,
12003 "queue": self.options.task_queue,
12004 "parent_close_policy": self.options.parent_close_policy.as_str(),
12005 "arguments": arguments,
12006 });
12007 let object = command
12008 .as_object_mut()
12009 .expect("child workflow command is always an object");
12010 if let Some(policy) = &self.options.retry_policy {
12011 let mut retry_policy = serde_json::Map::new();
12012 if let Some(max_attempts) = policy.max_attempts {
12013 if max_attempts == 0 {
12014 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
12015 "retry_policy.max_attempts must be at least 1".to_string(),
12016 )));
12017 }
12018 retry_policy.insert("max_attempts".to_string(), json!(max_attempts));
12019 }
12020 if !policy.backoff_seconds.is_empty() {
12021 retry_policy
12022 .insert("backoff_seconds".to_string(), json!(policy.backoff_seconds));
12023 }
12024 if !policy.non_retryable_error_types.is_empty() {
12025 retry_policy.insert(
12026 "non_retryable_error_types".to_string(),
12027 json!(policy.non_retryable_error_types),
12028 );
12029 }
12030 if retry_policy.is_empty() {
12031 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
12032 "retry_policy must configure at least one field".to_string(),
12033 )));
12034 }
12035 object.insert("retry_policy".to_string(), Value::Object(retry_policy));
12036 }
12037 if let Some(seconds) = self.options.execution_timeout_seconds {
12038 object.insert("execution_timeout_seconds".to_string(), json!(seconds));
12039 }
12040 if let Some(seconds) = self.options.run_timeout_seconds {
12041 object.insert("run_timeout_seconds".to_string(), json!(seconds));
12042 }
12043 apply_parallel_group_path(object, &self.parallel_group_path);
12044 state.commands.push(command);
12045 self.scheduled = true;
12046 }
12047
12048 Poll::Pending
12049 }
12050}
12051
12052impl Future for ChildWorkflowCall {
12053 type Output = Result<ChildWorkflowResult>;
12054
12055 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12056 match self.poll_avro_value(cx) {
12057 Poll::Ready(Ok(result)) => match result.result.into_json() {
12058 Ok(projected) => Poll::Ready(Ok(ChildWorkflowResult {
12059 parent: result.parent,
12060 child: result.child,
12061 child_workflow_type: result.child_workflow_type,
12062 result: projected,
12063 })),
12064 Err(error) => Poll::Ready(Err(error)),
12065 },
12066 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
12067 Poll::Pending => Poll::Pending,
12068 }
12069 }
12070}
12071
12072fn command_mismatch(recorded: &RecordedCommand, actual: impl Into<String>) -> Error {
12073 Error::NonDeterministicReplay(ReplayFailure::new(
12074 "recorded_command_mismatch",
12075 Some(recorded.sequence()),
12076 Some(recorded.shape().to_string()),
12077 Some(actual.into()),
12078 "current workflow command does not match the recorded durable command sequence",
12079 ))
12080}
12081
12082pub struct SignalCall {
12083 ctx: WorkflowContext,
12084 signal_name: String,
12085 runtime_reserved_allowed: bool,
12086 opened_wait: bool,
12087 matched_pending: bool,
12088 parallel_group_path: Vec<ParallelGroupMetadata>,
12089}
12090
12091impl SignalCall {
12092 fn poll_avro_value(
12093 mut self: Pin<&mut Self>,
12094 _cx: &mut TaskContext<'_>,
12095 ) -> Poll<Result<Vec<AvroValue>>> {
12096 if self.matched_pending {
12097 return Poll::Pending;
12098 }
12099 if !self.runtime_reserved_allowed {
12100 if let Err(error) = validate_user_signal_name(&self.signal_name) {
12101 return Poll::Ready(Err(error));
12102 }
12103 }
12104
12105 let ctx = self.ctx.clone();
12106 let mut state = match ctx.state.lock() {
12107 Ok(state) => state,
12108 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12109 };
12110
12111 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
12112 match recorded {
12113 RecordedCommand::SignalWait {
12114 sequence,
12115 signal_name,
12116 value,
12117 parallel_group_path,
12118 } => {
12119 if let Err(error) = ensure_parallel_path_matches(
12120 sequence,
12121 parallel_group_path.as_deref(),
12122 &self.parallel_group_path,
12123 ) {
12124 return Poll::Ready(Err(error));
12125 }
12126 if signal_name != self.signal_name {
12127 return Poll::Ready(Err(Error::NonDeterministicReplay(
12128 ReplayFailure::new(
12129 "recorded_command_detail_mismatch",
12130 Some(sequence),
12131 Some(format!("signal wait:{signal_name}")),
12132 Some(format!("signal wait:{}", self.signal_name)),
12133 "recorded signal name differs from the current workflow command",
12134 ),
12135 )));
12136 }
12137
12138 state.command_cursor += 1;
12139 if let Some(value) = value {
12140 return Poll::Ready(Ok(value));
12141 }
12142 if state
12143 .resume_signal
12144 .as_ref()
12145 .is_some_and(|signal| signal.signal_name == self.signal_name)
12146 {
12147 let signal = state
12148 .resume_signal
12149 .take()
12150 .expect("matching resume signal is present");
12151 return Poll::Ready(Ok(signal.arguments));
12152 }
12153
12154 state.matched_recorded_pending = true;
12155 self.opened_wait = true;
12156 self.matched_pending = true;
12157 return Poll::Pending;
12158 }
12159 other => {
12160 return Poll::Ready(Err(command_mismatch(
12161 &other,
12162 format!("signal wait:{}", self.signal_name),
12163 )));
12164 }
12165 }
12166 }
12167
12168 if state
12169 .resume_signal
12170 .as_ref()
12171 .is_some_and(|signal| signal.signal_name == self.signal_name)
12172 {
12173 let signal = state
12174 .resume_signal
12175 .take()
12176 .expect("matching resume signal is present");
12177 return Poll::Ready(Ok(signal.arguments));
12178 }
12179
12180 if !self.opened_wait {
12181 let mut command = serde_json::Map::from_iter([
12182 ("type".to_string(), json!("open_signal_wait")),
12183 ("signal_name".to_string(), json!(self.signal_name)),
12184 ]);
12185 apply_parallel_group_path(&mut command, &self.parallel_group_path);
12186 state.commands.push(Value::Object(command));
12187 self.opened_wait = true;
12188 }
12189
12190 Poll::Pending
12191 }
12192}
12193
12194impl Future for SignalCall {
12195 type Output = Result<Vec<Value>>;
12196
12197 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12198 match self.poll_avro_value(cx) {
12199 Poll::Ready(Ok(values)) => Poll::Ready(
12200 values
12201 .into_iter()
12202 .map(AvroValue::into_json)
12203 .collect::<Result<Vec<_>>>(),
12204 ),
12205 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
12206 Poll::Pending => Poll::Pending,
12207 }
12208 }
12209}
12210
12211#[derive(Clone, Debug)]
12212pub struct ActivityContext {
12213 client: Client,
12214 pub task_id: String,
12215 pub activity_attempt_id: String,
12216 pub lease_owner: String,
12217 pub activity_type: String,
12218 pub attempt_number: u64,
12219 pub task_queue: String,
12220 pub worker_id: String,
12221}
12222
12223impl ActivityContext {
12224 pub async fn heartbeat<T: Serialize>(&self, details: T) -> Result<ActivityHeartbeatResponse> {
12225 self.client
12226 .heartbeat_activity_task(
12227 &self.task_id,
12228 &self.activity_attempt_id,
12229 &self.lease_owner,
12230 details,
12231 )
12232 .await
12233 }
12234}
12235
12236fn decode_task_avro_arguments(value: Option<&Value>, codec: &str) -> Result<AvroValue> {
12237 validate_payload_codec(codec)?;
12238 match value {
12239 Some(value) => Ok(normalize_avro_arguments(decode_wire_avro_value(
12240 value, codec,
12241 )?)),
12242 None => Ok(AvroValue::Array(Vec::new())),
12243 }
12244}
12245
12246fn decode_resume_signal(task: &WorkflowTask) -> Result<Option<ResumeSignal>> {
12247 let Some(signal_name) = task
12248 .signal_name
12249 .as_deref()
12250 .filter(|value| !value.is_empty())
12251 else {
12252 return Ok(None);
12253 };
12254 let decoded = decode_task_avro_arguments(task.signal_arguments.as_ref(), &task.payload_codec)?;
12255 let AvroValue::Array(arguments) = decoded else {
12256 unreachable!("normalize_avro_arguments always returns an array");
12257 };
12258
12259 Ok(Some(ResumeSignal {
12260 signal_name: signal_name.to_string(),
12261 arguments,
12262 }))
12263}
12264
12265fn validate_workflow_task_payloads(task: &WorkflowTask) -> Result<()> {
12266 validate_payload_codec(&task.payload_codec)?;
12267 validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)?;
12268 validate_optional_inbound_payload(task.signal_arguments.as_ref(), &task.payload_codec)?;
12269 for event in &task.history_events {
12270 validate_history_event_payloads(event, &task.payload_codec)?;
12271 }
12272 Ok(())
12273}
12274
12275fn validate_activity_task_payloads(task: &ActivityTask) -> Result<()> {
12276 validate_payload_codec(&task.payload_codec)?;
12277 validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)
12278}
12279
12280fn validate_query_task_payloads(task: &QueryTask) -> Result<()> {
12281 validate_payload_codec(&task.payload_codec)?;
12282 validate_optional_inbound_payload(task.workflow_arguments.as_ref(), &task.payload_codec)?;
12283 validate_optional_inbound_payload(task.query_arguments.as_ref(), &task.payload_codec)?;
12284 for event in &task.history_events {
12285 validate_history_event_payloads(event, &task.payload_codec)?;
12286 }
12287
12288 let Some(export) = task.history_export.as_ref() else {
12289 return Ok(());
12290 };
12291 let export_codec = match export.get("payloads") {
12292 Some(payloads) => declared_payload_codec(payloads, "codec")?,
12293 None => None,
12294 }
12295 .unwrap_or(&task.payload_codec);
12296 validate_payload_codec(export_codec)?;
12297
12298 if let Some(events) = export.get("history_events").and_then(Value::as_array) {
12299 for event in events {
12300 let event_type = event
12301 .get("event_type")
12302 .or_else(|| event.get("type"))
12303 .and_then(Value::as_str)
12304 .unwrap_or_default();
12305 if let Some(payload) = event.get("payload") {
12306 validate_history_payloads(event_type, payload, export_codec)?;
12307 }
12308 }
12309 }
12310 for signal in export
12311 .get("signals")
12312 .and_then(Value::as_array)
12313 .into_iter()
12314 .flatten()
12315 {
12316 let codec = declared_payload_codec(signal, "payload_codec")?.unwrap_or(export_codec);
12317 validate_payload_codec(codec)?;
12318 validate_optional_inbound_payload(signal.get("arguments"), codec)?;
12319 }
12320 for activity in export
12321 .get("activities")
12322 .and_then(Value::as_array)
12323 .into_iter()
12324 .flatten()
12325 {
12326 let codec = declared_payload_codec(activity, "payload_codec")?.unwrap_or(export_codec);
12327 validate_payload_codec(codec)?;
12328 validate_optional_inbound_payload(activity.get("arguments"), codec)?;
12329 validate_optional_inbound_payload(activity.get("result"), codec)?;
12330 }
12331 Ok(())
12332}
12333
12334fn validate_history_event_payloads(event: &HistoryEvent, fallback_codec: &str) -> Result<()> {
12335 validate_history_payloads(&event.event_type, &event.payload, fallback_codec)
12336}
12337
12338fn validate_history_payloads(
12339 event_type: &str,
12340 payload: &Value,
12341 fallback_codec: &str,
12342) -> Result<()> {
12343 let codec = declared_payload_codec(payload, "payload_codec")?.unwrap_or(fallback_codec);
12344 validate_payload_codec(codec)?;
12345 for field in history_payload_fields(event_type) {
12346 validate_optional_inbound_payload(payload.get(*field), codec)?;
12347 }
12348 Ok(())
12349}
12350
12351const SIGNAL_HISTORY_PAYLOAD_FIELDS: &[&str] = &["value", "input", "arguments"];
12352
12353fn history_payload_fields(event_type: &str) -> &'static [&'static str] {
12354 match event_type {
12355 "ActivityCompleted" => &["result"],
12356 "SignalReceived" | "SignalApplied" => SIGNAL_HISTORY_PAYLOAD_FIELDS,
12357 "UpdateAccepted" | "UpdateRejected" | "UpdateApplied" => &["arguments"],
12358 "UpdateCompleted" | "SideEffectRecorded" => &["result"],
12359 "ChildRunCompleted" => &["result", "output"],
12360 "WorkflowCompleted" => &["output"],
12361 "ServiceCallStarted"
12362 | "ServiceCallCompleted"
12363 | "ServiceCallFailed"
12364 | "ServiceCallCancelled" => &["request_payload", "response_payload"],
12365 _ => &[],
12366 }
12367}
12368
12369fn signal_history_payload(payload: &Value) -> Option<&Value> {
12370 SIGNAL_HISTORY_PAYLOAD_FIELDS
12371 .iter()
12372 .find_map(|field| payload.get(*field))
12373}
12374
12375fn declared_payload_codec<'a>(value: &'a Value, field: &str) -> Result<Option<&'a str>> {
12376 match value.get(field) {
12377 None => Ok(None),
12378 Some(Value::String(codec)) => Ok(Some(codec)),
12379 Some(_) => Err(invalid_payload_envelope()),
12380 }
12381}
12382
12383fn validate_optional_inbound_payload(value: Option<&Value>, codec: &str) -> Result<()> {
12384 validate_payload_codec(codec)?;
12385 if let Some(value) = value.filter(|value| !value.is_null()) {
12386 decode_wire_avro_value(value, codec)?;
12387 }
12388 Ok(())
12389}
12390
12391fn recorded_parallel_group_entry(payload: &Value, sequence: u64) -> Result<ParallelGroupMetadata> {
12392 let group_id = payload_string(payload, "parallel_group_id").ok_or_else(|| {
12393 invalid_recorded_history(
12394 "parallel_group_metadata_invalid",
12395 sequence,
12396 "non-empty parallel_group_id",
12397 &payload.to_string(),
12398 "parallel-group history is missing its stable identity",
12399 )
12400 })?;
12401 let kind = payload_string(payload, "parallel_group_kind").ok_or_else(|| {
12402 invalid_recorded_history(
12403 "parallel_group_metadata_invalid",
12404 sequence,
12405 "activity, child, timer, signal, condition, or mixed group kind",
12406 &payload.to_string(),
12407 "parallel-group history is missing its group kind",
12408 )
12409 })?;
12410 if !matches!(
12411 kind.as_str(),
12412 "activity" | "child" | "timer" | "signal" | "condition" | "mixed"
12413 ) {
12414 return Err(invalid_recorded_history(
12415 "parallel_group_metadata_invalid",
12416 sequence,
12417 "activity, child, timer, signal, condition, or mixed group kind",
12418 &kind,
12419 "parallel-group history contains an unsupported group kind",
12420 ));
12421 }
12422 let base_sequence = payload
12423 .get("parallel_group_base_sequence")
12424 .and_then(value_as_u64)
12425 .filter(|value| *value > 0)
12426 .ok_or_else(|| {
12427 invalid_recorded_history(
12428 "parallel_group_metadata_invalid",
12429 sequence,
12430 "positive parallel_group_base_sequence",
12431 &payload.to_string(),
12432 "parallel-group history contains an invalid base sequence",
12433 )
12434 })?;
12435 let size = payload
12436 .get("parallel_group_size")
12437 .and_then(value_as_u64)
12438 .and_then(|value| usize::try_from(value).ok())
12439 .filter(|value| (1..=MAX_PARALLEL_OPERATIONS).contains(value))
12440 .ok_or_else(|| {
12441 invalid_recorded_history(
12442 "parallel_group_metadata_invalid",
12443 sequence,
12444 "bounded positive parallel_group_size",
12445 &payload.to_string(),
12446 "parallel-group history contains an invalid group size",
12447 )
12448 })?;
12449 let index = payload
12450 .get("parallel_group_index")
12451 .and_then(value_as_u64)
12452 .and_then(|value| usize::try_from(value).ok())
12453 .filter(|value| *value < size)
12454 .ok_or_else(|| {
12455 invalid_recorded_history(
12456 "parallel_group_metadata_invalid",
12457 sequence,
12458 "parallel_group_index within group bounds",
12459 &payload.to_string(),
12460 "parallel-group history contains an invalid member index",
12461 )
12462 })?;
12463 if base_sequence.checked_add(u64::try_from(index).unwrap_or(u64::MAX)) != Some(sequence) {
12464 return Err(invalid_recorded_history(
12465 "parallel_group_metadata_invalid",
12466 sequence,
12467 "base sequence plus member index equals workflow sequence",
12468 &payload.to_string(),
12469 "parallel-group path does not preserve durable workflow position",
12470 ));
12471 }
12472 let mode = payload
12473 .get("parallel_group_mode")
12474 .and_then(Value::as_str)
12475 .unwrap_or("all");
12476 if !matches!(mode, "all" | "select") {
12477 return Err(invalid_recorded_history(
12478 "parallel_group_metadata_invalid",
12479 sequence,
12480 "parallel group mode all or select",
12481 mode,
12482 "parallel-group history contains an unsupported group mode",
12483 ));
12484 }
12485 let expected_id = if mode == "select" {
12486 format!("select-calls:{base_sequence}:{size}")
12487 } else {
12488 format!("{}:{base_sequence}:{size}", parallel_group_prefix(&kind))
12489 };
12490 if group_id != expected_id {
12491 return Err(invalid_recorded_history(
12492 "parallel_group_metadata_invalid",
12493 sequence,
12494 &expected_id,
12495 &group_id,
12496 "parallel-group history contains an incompatible stable group ID",
12497 ));
12498 }
12499 let selection_member_key = if mode == "select" {
12500 Some(selection_key_from_value(
12501 payload.get("selection_member_key"),
12502 sequence,
12503 )?)
12504 } else {
12505 None
12506 };
12507 let selection_member_index = if mode == "select" {
12508 Some(required_parallel_usize(
12509 payload,
12510 "selection_member_index",
12511 sequence,
12512 )?)
12513 } else {
12514 None
12515 };
12516 let selection_member_base_sequence = if mode == "select" {
12517 Some(
12518 payload
12519 .get("selection_member_base_sequence")
12520 .and_then(value_as_u64)
12521 .filter(|value| *value >= base_sequence)
12522 .ok_or_else(|| {
12523 invalid_recorded_history(
12524 "parallel_group_metadata_invalid",
12525 sequence,
12526 "selection member base within its group",
12527 &payload.to_string(),
12528 "selection history contains an invalid member base sequence",
12529 )
12530 })?,
12531 )
12532 } else {
12533 None
12534 };
12535 let selection_member_size = if mode == "select" {
12536 let member_size = required_parallel_usize(payload, "selection_member_size", sequence)?;
12537 if member_size == 0 {
12538 return Err(invalid_recorded_history(
12539 "parallel_group_metadata_invalid",
12540 sequence,
12541 "positive selection member size",
12542 &payload.to_string(),
12543 "selection history contains an invalid member size",
12544 ));
12545 }
12546 Some(member_size)
12547 } else {
12548 None
12549 };
12550 let selection_member_kind = if mode == "select" {
12551 let kind = payload_string(payload, "selection_member_kind").ok_or_else(|| {
12552 invalid_recorded_history(
12553 "parallel_group_metadata_invalid",
12554 sequence,
12555 "selection member operation kind",
12556 &payload.to_string(),
12557 "selection history is missing its authored member kind",
12558 )
12559 })?;
12560 if !matches!(
12561 kind.as_str(),
12562 "activity" | "child" | "timer" | "signal" | "condition" | "group"
12563 ) {
12564 return Err(invalid_recorded_history(
12565 "parallel_group_metadata_invalid",
12566 sequence,
12567 "activity, child, timer, signal, condition, or group selection member kind",
12568 &kind,
12569 "selection history contains an unsupported member kind",
12570 ));
12571 }
12572 Some(kind)
12573 } else {
12574 None
12575 };
12576 if let (Some(member_base), Some(member_size)) =
12577 (selection_member_base_sequence, selection_member_size)
12578 {
12579 let member_end = member_base
12580 .checked_add(u64::try_from(member_size).unwrap_or(u64::MAX))
12581 .ok_or_else(|| {
12582 invalid_recorded_history(
12583 "parallel_group_metadata_invalid",
12584 sequence,
12585 "bounded selection member range",
12586 &payload.to_string(),
12587 "selection member range overflowed",
12588 )
12589 })?;
12590 let group_end = base_sequence
12591 .checked_add(u64::try_from(size).unwrap_or(u64::MAX))
12592 .unwrap_or(u64::MAX);
12593 if sequence < member_base || sequence >= member_end || member_end > group_end {
12594 return Err(invalid_recorded_history(
12595 "parallel_group_metadata_invalid",
12596 sequence,
12597 "workflow sequence within one bounded selection member",
12598 &payload.to_string(),
12599 "selection member range does not contain its durable leaf",
12600 ));
12601 }
12602 }
12603 Ok(ParallelGroupMetadata {
12604 parallel_group_id: group_id,
12605 parallel_group_kind: kind,
12606 parallel_group_base_sequence: base_sequence,
12607 parallel_group_size: size,
12608 parallel_group_index: index,
12609 parallel_group_mode: (mode == "select").then(|| "select".to_string()),
12610 selection_member_key,
12611 selection_member_index,
12612 selection_member_base_sequence,
12613 selection_member_size,
12614 selection_member_kind,
12615 })
12616}
12617
12618fn required_parallel_usize(payload: &Value, field: &str, sequence: u64) -> Result<usize> {
12619 payload
12620 .get(field)
12621 .and_then(value_as_u64)
12622 .and_then(|value| usize::try_from(value).ok())
12623 .ok_or_else(|| {
12624 invalid_recorded_history(
12625 "parallel_group_metadata_invalid",
12626 sequence,
12627 &format!("non-negative integer {field}"),
12628 &payload.to_string(),
12629 "selection history contains invalid member metadata",
12630 )
12631 })
12632}
12633
12634fn selection_key_from_value(value: Option<&Value>, sequence: u64) -> Result<SelectionKey> {
12635 match value {
12636 Some(Value::String(value)) if !value.is_empty() => Ok(SelectionKey::Name(value.clone())),
12637 Some(value) => value_as_u64(value)
12638 .and_then(|value| usize::try_from(value).ok())
12639 .map(SelectionKey::Index)
12640 .ok_or_else(|| {
12641 invalid_recorded_history(
12642 "selection_member_key_invalid",
12643 sequence,
12644 "non-empty string or non-negative integer member key",
12645 &value.to_string(),
12646 "selection history contains an invalid member key",
12647 )
12648 }),
12649 None => Err(invalid_recorded_history(
12650 "selection_member_key_missing",
12651 sequence,
12652 "selection_member_key",
12653 "<missing>",
12654 "selection history is missing its stable member key",
12655 )),
12656 }
12657}
12658
12659fn recorded_parallel_group_path(
12660 events: &[&HistoryEvent],
12661 sequence: u64,
12662) -> Result<Option<Vec<ParallelGroupMetadata>>> {
12663 let mut recorded: Option<Vec<ParallelGroupMetadata>> = None;
12664 for event in events {
12665 let payload = &event.payload;
12666 let has_metadata = payload.get("parallel_group_path").is_some()
12667 || payload.get("parallel_group_id").is_some()
12668 || payload.get("parallel_group_kind").is_some()
12669 || payload.get("parallel_group_base_sequence").is_some()
12670 || payload.get("parallel_group_size").is_some()
12671 || payload.get("parallel_group_index").is_some()
12672 || payload.get("parallel_group_mode").is_some()
12673 || payload.get("selection_member_key").is_some();
12674 if !has_metadata {
12675 continue;
12676 }
12677
12678 let top_level = recorded_parallel_group_entry(payload, sequence)?;
12679 let path = match payload.get("parallel_group_path") {
12680 None => vec![top_level.clone()],
12681 Some(Value::Array(entries)) if !entries.is_empty() => entries
12682 .iter()
12683 .map(|entry| recorded_parallel_group_entry(entry, sequence))
12684 .collect::<Result<Vec<_>>>()?,
12685 Some(value) => {
12686 return Err(invalid_recorded_history(
12687 "parallel_group_metadata_invalid",
12688 sequence,
12689 "non-empty parallel_group_path list",
12690 &value.to_string(),
12691 "parallel-group history contains an invalid group path",
12692 ));
12693 }
12694 };
12695 if path.last() != Some(&top_level) {
12696 return Err(invalid_recorded_history(
12697 "parallel_group_metadata_invalid",
12698 sequence,
12699 &serde_json::to_string(&path.last()).unwrap_or_default(),
12700 &serde_json::to_string(&top_level).unwrap_or_default(),
12701 "parallel-group top-level fields do not match the innermost path entry",
12702 ));
12703 }
12704 if recorded.as_ref().is_some_and(|existing| existing != &path) {
12705 return Err(invalid_recorded_history(
12706 "parallel_group_history_conflict",
12707 sequence,
12708 &serde_json::to_string(&recorded.as_ref()).unwrap_or_default(),
12709 &serde_json::to_string(&path).unwrap_or_default(),
12710 "parallel-group metadata changed between scheduling and resolution history",
12711 ));
12712 }
12713 recorded = Some(path);
12714 }
12715 Ok(recorded)
12716}
12717
12718fn recorded_commands(
12719 events: &[HistoryEvent],
12720 fallback_codec: &str,
12721 parent: WorkflowIdentity,
12722) -> Result<Vec<RecordedCommand>> {
12723 let mut events_by_sequence: BTreeMap<u64, Vec<&HistoryEvent>> = BTreeMap::new();
12724 let mut last_new_sequence = None;
12725
12726 for event in events {
12727 let is_activity = matches!(
12728 event.event_type.as_str(),
12729 "ActivityScheduled"
12730 | "ActivityStarted"
12731 | "ActivityHeartbeatRecorded"
12732 | "ActivityRetryScheduled"
12733 | "ActivityCompleted"
12734 | "ActivityFailed"
12735 | "ActivityCancelled"
12736 | "ActivityTimedOut"
12737 );
12738 let is_workflow_timer = matches!(
12739 event.event_type.as_str(),
12740 "TimerScheduled" | "TimerCancelled" | "TimerFired"
12741 ) && !is_internal_timer_event(event);
12742 let is_child_workflow = matches!(
12743 event.event_type.as_str(),
12744 "ChildWorkflowScheduled"
12745 | "ChildRunCompleted"
12746 | "ChildRunFailed"
12747 | "ChildRunCancelled"
12748 | "ChildRunTerminated"
12749 );
12750 let is_signal_wait = is_recorded_signal_wait_event(event);
12751 let is_condition_wait = is_recorded_condition_wait_event(event);
12752 let is_search_attributes = event.event_type == "SearchAttributesUpserted";
12753 let is_side_effect = event.event_type == "SideEffectRecorded";
12754 let is_version_marker = event.event_type == "VersionMarkerRecorded";
12755 let is_memo = event.event_type == "MemoUpserted";
12756 if !is_activity
12757 && !is_workflow_timer
12758 && !is_child_workflow
12759 && !is_signal_wait
12760 && !is_condition_wait
12761 && !is_search_attributes
12762 && !is_side_effect
12763 && !is_version_marker
12764 && !is_memo
12765 {
12766 continue;
12767 }
12768
12769 let sequence = durable_event_sequence(event).ok_or_else(|| {
12770 Error::NonDeterministicReplay(ReplayFailure::new(
12771 "durable_command_sequence_missing",
12772 None,
12773 Some("positive workflow sequence".to_string()),
12774 Some(event.event_type.clone()),
12775 "durable command history event has no workflow sequence",
12776 ))
12777 })?;
12778 if sequence == 0 {
12779 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12780 "durable_command_sequence_invalid",
12781 Some(sequence),
12782 Some("positive workflow sequence".to_string()),
12783 Some(sequence.to_string()),
12784 "durable command history uses an invalid workflow sequence",
12785 )));
12786 }
12787 if !events_by_sequence.contains_key(&sequence) {
12788 if let Some(previous) = last_new_sequence {
12789 if sequence < previous {
12790 return Err(invalid_recorded_history(
12791 "durable_command_sequence_mismatch",
12792 sequence,
12793 &format!("workflow sequence greater than {previous}"),
12794 &sequence.to_string(),
12795 "durable commands are not strictly ordered by their recorded workflow sequence",
12796 ));
12797 }
12798 }
12799 last_new_sequence = Some(sequence);
12800 }
12801 events_by_sequence.entry(sequence).or_default().push(event);
12802 }
12803
12804 let commands: Vec<RecordedCommand> = events_by_sequence
12805 .into_iter()
12806 .map(|(sequence, sequence_events)| {
12807 let activity_events: Vec<_> = sequence_events
12808 .iter()
12809 .copied()
12810 .filter(|event| event.event_type.starts_with("Activity"))
12811 .collect();
12812 let timer_events: Vec<_> = sequence_events
12813 .iter()
12814 .copied()
12815 .filter(|event| event.event_type.starts_with("Timer"))
12816 .collect();
12817 let child_events: Vec<_> = sequence_events
12818 .iter()
12819 .copied()
12820 .filter(|event| {
12821 event.event_type == "ChildWorkflowScheduled"
12822 || event.event_type.starts_with("ChildRun")
12823 })
12824 .collect();
12825 let signal_wait_events: Vec<_> = sequence_events
12826 .iter()
12827 .copied()
12828 .filter(|event| is_recorded_signal_wait_event(event))
12829 .collect();
12830 let condition_wait_events: Vec<_> = sequence_events
12831 .iter()
12832 .copied()
12833 .filter(|event| is_recorded_condition_wait_event(event))
12834 .collect();
12835 let search_attribute_events: Vec<_> = sequence_events
12836 .iter()
12837 .copied()
12838 .filter(|event| event.event_type == "SearchAttributesUpserted")
12839 .collect();
12840 let side_effect_events: Vec<_> = sequence_events
12841 .iter()
12842 .copied()
12843 .filter(|event| event.event_type == "SideEffectRecorded")
12844 .collect();
12845 let version_marker_events: Vec<_> = sequence_events
12846 .iter()
12847 .copied()
12848 .filter(|event| event.event_type == "VersionMarkerRecorded")
12849 .collect();
12850 let memo_events: Vec<_> = sequence_events
12851 .iter()
12852 .copied()
12853 .filter(|event| event.event_type == "MemoUpserted")
12854 .collect();
12855
12856 let command_kind_count = usize::from(!activity_events.is_empty())
12857 + usize::from(!timer_events.is_empty())
12858 + usize::from(!child_events.is_empty())
12859 + usize::from(!signal_wait_events.is_empty())
12860 + usize::from(!condition_wait_events.is_empty())
12861 + usize::from(!search_attribute_events.is_empty())
12862 + usize::from(!side_effect_events.is_empty())
12863 + usize::from(!version_marker_events.is_empty())
12864 + usize::from(!memo_events.is_empty());
12865 if command_kind_count > 1 {
12866 let actual = [
12867 (!activity_events.is_empty()).then_some("activity"),
12868 (!timer_events.is_empty()).then_some("timer"),
12869 (!child_events.is_empty()).then_some("child workflow"),
12870 (!signal_wait_events.is_empty()).then_some("signal wait"),
12871 (!condition_wait_events.is_empty()).then_some("condition wait"),
12872 (!search_attribute_events.is_empty()).then_some("search-attribute update"),
12873 (!side_effect_events.is_empty()).then_some("side effect"),
12874 (!version_marker_events.is_empty()).then_some("version marker"),
12875 (!memo_events.is_empty()).then_some("memo upsert"),
12876 ]
12877 .into_iter()
12878 .flatten()
12879 .collect::<Vec<_>>()
12880 .join(" and ");
12881 return Err(invalid_recorded_history(
12882 "durable_command_sequence_collision",
12883 sequence,
12884 "one durable command kind",
12885 &actual,
12886 "one workflow sequence records more than one durable command kind",
12887 ));
12888 }
12889
12890 if !activity_events.is_empty() {
12891 let parallel_group_path =
12892 recorded_parallel_group_path(&activity_events, sequence)?;
12893 let scheduled_count = activity_events
12894 .iter()
12895 .filter(|event| event.event_type == "ActivityScheduled")
12896 .count();
12897 if scheduled_count > 1 {
12898 return Err(invalid_recorded_history(
12899 "duplicate_activity_schedule",
12900 sequence,
12901 "at most one ActivityScheduled event",
12902 "multiple ActivityScheduled events",
12903 "activity history schedules more than one command at one workflow sequence",
12904 ));
12905 }
12906 let activity_type = activity_events.iter().find_map(|event| {
12907 event
12908 .payload
12909 .get("activity_type")
12910 .or_else(|| event.payload.get("activity_name"))
12911 .and_then(Value::as_str)
12912 .map(str::to_string)
12913 });
12914 if activity_events.iter().filter_map(|event| {
12915 event
12916 .payload
12917 .get("activity_type")
12918 .or_else(|| event.payload.get("activity_name"))
12919 .and_then(Value::as_str)
12920 }).any(|candidate| Some(candidate) != activity_type.as_deref()) {
12921 return Err(invalid_recorded_history(
12922 "activity_identity_mismatch",
12923 sequence,
12924 activity_type.as_deref().unwrap_or("one activity identity"),
12925 "conflicting activity identities",
12926 "activity lifecycle events at one workflow sequence disagree on identity",
12927 ));
12928 }
12929 let terminal: Vec<_> = activity_events
12930 .iter()
12931 .copied()
12932 .filter(|event| {
12933 matches!(
12934 event.event_type.as_str(),
12935 "ActivityCompleted"
12936 | "ActivityFailed"
12937 | "ActivityCancelled"
12938 | "ActivityTimedOut"
12939 )
12940 })
12941 .collect();
12942 let duplicate_delivery = terminal.first().is_some_and(|first| {
12943 terminal.iter().all(|event| {
12944 event.event_type == first.event_type && event.payload == first.payload
12945 })
12946 });
12947 if terminal.len() > 1 && !duplicate_delivery {
12948 return Err(invalid_recorded_history(
12949 "duplicate_activity_terminal_event",
12950 sequence,
12951 "at most one terminal activity event",
12952 "multiple terminal activity events",
12953 "activity history settles one command more than once",
12954 ));
12955 }
12956 let outcome = terminal
12957 .first()
12958 .map(|event| activity_outcome(event, fallback_codec, activity_type.clone()))
12959 .transpose()?;
12960 let options = activity_events
12961 .iter()
12962 .find(|event| event.event_type == "ActivityScheduled")
12963 .and_then(|event| event.payload.get("activity"))
12964 .and_then(Value::as_object)
12965 .map(|activity| RecordedActivityOptions {
12966 task_queue: recorded_optional_string(activity, "queue"),
12967 execution_mode: recorded_optional_string(activity, "execution_mode"),
12968 retry_policy: recorded_activity_retry_snapshot(
12969 activity.get("retry_policy"),
12970 ),
12971 });
12972 return Ok(RecordedCommand::Activity {
12973 sequence,
12974 activity_type,
12975 options,
12976 outcome,
12977 parallel_group_path,
12978 });
12979 }
12980
12981 if !child_events.is_empty() {
12982 let parallel_group_path = recorded_parallel_group_path(&child_events, sequence)?;
12983 let scheduled: Vec<_> = child_events
12984 .iter()
12985 .copied()
12986 .filter(|event| event.event_type == "ChildWorkflowScheduled")
12987 .collect();
12988 if scheduled.len() != 1 {
12989 return Err(invalid_recorded_history(
12990 "child_workflow_schedule_missing_or_duplicate",
12991 sequence,
12992 "one ChildWorkflowScheduled event",
12993 &format!("{} ChildWorkflowScheduled events", scheduled.len()),
12994 "child workflow replay requires exactly one recorded schedule event",
12995 ));
12996 }
12997 let workflow_type = child_events.iter().find_map(|event| {
12998 event
12999 .payload
13000 .get("child_workflow_type")
13001 .or_else(|| event.payload.get("workflow_type"))
13002 .and_then(Value::as_str)
13003 .filter(|value| !value.is_empty())
13004 .map(str::to_string)
13005 });
13006 if child_events
13007 .iter()
13008 .filter_map(|event| {
13009 event
13010 .payload
13011 .get("child_workflow_type")
13012 .or_else(|| event.payload.get("workflow_type"))
13013 .and_then(Value::as_str)
13014 })
13015 .any(|candidate| Some(candidate) != workflow_type.as_deref())
13016 {
13017 return Err(invalid_recorded_history(
13018 "child_workflow_identity_mismatch",
13019 sequence,
13020 workflow_type
13021 .as_deref()
13022 .unwrap_or("one child workflow type"),
13023 "conflicting child workflow types",
13024 "child workflow lifecycle events at one sequence disagree on type",
13025 ));
13026 }
13027 let mut outcomes = child_workflow_outcomes(
13028 &child_events.iter().map(|event| (*event).clone()).collect::<Vec<_>>(),
13029 fallback_codec,
13030 parent.clone(),
13031 )?;
13032 let terminal_events = child_events
13033 .iter()
13034 .copied()
13035 .filter(|event| event.event_type.starts_with("ChildRun"))
13036 .collect::<Vec<_>>();
13037 let duplicate_delivery = terminal_events.first().is_some_and(|first| {
13038 terminal_events.iter().all(|event| {
13039 event.event_type == first.event_type && event.payload == first.payload
13040 })
13041 });
13042 if outcomes.len() > 1 && !duplicate_delivery {
13043 return Err(invalid_recorded_history(
13044 "duplicate_child_workflow_terminal_event",
13045 sequence,
13046 "at most one terminal child event",
13047 "multiple terminal child events",
13048 "child workflow history settles one command more than once",
13049 ));
13050 }
13051 return Ok(RecordedCommand::ChildWorkflow {
13052 sequence,
13053 workflow_type,
13054 outcome: outcomes.pop(),
13055 parallel_group_path,
13056 });
13057 }
13058
13059 if !signal_wait_events.is_empty() {
13060 let opened: Vec<_> = signal_wait_events
13061 .iter()
13062 .copied()
13063 .filter(|event| event.event_type == "SignalWaitOpened")
13064 .collect();
13065 if opened.len() != 1 {
13066 return Err(invalid_recorded_history(
13067 "signal_wait_open_missing_or_duplicate",
13068 sequence,
13069 "one SignalWaitOpened event",
13070 &format!("{} SignalWaitOpened events", opened.len()),
13071 "signal replay requires exactly one canonical wait-open event",
13072 ));
13073 }
13074
13075 let applied: Vec<_> = signal_wait_events
13076 .iter()
13077 .copied()
13078 .filter(|event| event.event_type == "SignalApplied")
13079 .collect();
13080 if applied.len() > 1 {
13081 return Err(invalid_recorded_history(
13082 "duplicate_signal_wait_apply",
13083 sequence,
13084 "at most one SignalApplied event",
13085 "multiple SignalApplied events",
13086 "signal history applies one durable wait more than once",
13087 ));
13088 }
13089
13090 let signal_names = signal_wait_events
13091 .iter()
13092 .map(|event| required_signal_wait_name(event, sequence))
13093 .collect::<Result<Vec<_>>>()?;
13094 let signal_name = signal_names
13095 .first()
13096 .expect("signal wait events are not empty")
13097 .clone();
13098 if signal_names.iter().any(|candidate| candidate != &signal_name) {
13099 return Err(invalid_recorded_history(
13100 "signal_wait_identity_mismatch",
13101 sequence,
13102 &signal_name,
13103 "conflicting signal names",
13104 "signal wait lifecycle events at one workflow sequence disagree on identity",
13105 ));
13106 }
13107 let value = applied
13108 .first()
13109 .map(|event| decode_signal_event_arguments(event, fallback_codec))
13110 .transpose()?;
13111 return Ok(RecordedCommand::SignalWait {
13112 sequence,
13113 signal_name,
13114 value,
13115 parallel_group_path: recorded_parallel_group_path(
13116 &signal_wait_events,
13117 sequence,
13118 )?,
13119 });
13120 }
13121
13122 if !condition_wait_events.is_empty() {
13123 return recorded_condition_wait(
13124 sequence,
13125 &condition_wait_events,
13126 events,
13127 );
13128 }
13129
13130 if !search_attribute_events.is_empty() {
13131 if search_attribute_events.len() != 1 {
13132 return Err(invalid_recorded_history(
13133 "duplicate_search_attribute_update",
13134 sequence,
13135 "one SearchAttributesUpserted event",
13136 &format!(
13137 "{} SearchAttributesUpserted events",
13138 search_attribute_events.len()
13139 ),
13140 "search-attribute history records one workflow command more than once",
13141 ));
13142 }
13143 let payload = &search_attribute_events[0].payload;
13144 let attributes = payload
13145 .get("attributes")
13146 .filter(|value| value.as_object().is_some_and(|values| !values.is_empty()))
13147 .cloned()
13148 .ok_or_else(|| {
13149 invalid_recorded_history(
13150 "search_attribute_update_missing",
13151 sequence,
13152 "non-empty attributes object",
13153 "missing or invalid attributes",
13154 "search-attribute history is missing its recorded mutation",
13155 )
13156 })?;
13157 let attribute_types =
13158 recorded_search_attribute_types(payload, &attributes, sequence)?;
13159 return Ok(RecordedCommand::SearchAttributes {
13160 sequence,
13161 attributes,
13162 attribute_types,
13163 });
13164 }
13165
13166 if !side_effect_events.is_empty() {
13167 if side_effect_events.len() != 1 {
13168 return Err(invalid_recorded_history(
13169 "duplicate_side_effect_record",
13170 sequence,
13171 "one SideEffectRecorded event",
13172 &format!("{} SideEffectRecorded events", side_effect_events.len()),
13173 "side-effect history records one workflow command more than once",
13174 ));
13175 }
13176 let event = side_effect_events[0];
13177 let result = event.payload.get("result").ok_or_else(|| {
13178 invalid_recorded_history(
13179 "side_effect_result_missing",
13180 sequence,
13181 "recorded result payload",
13182 "missing result",
13183 "side-effect history is missing its recorded value",
13184 )
13185 })?;
13186 let has_published_envelope = result.as_str().is_some()
13187 || result.as_object().is_some_and(|envelope| {
13188 envelope.get("codec").and_then(Value::as_str).is_some()
13189 && envelope.get("blob").and_then(Value::as_str).is_some()
13190 });
13191 if !has_published_envelope {
13192 return Err(invalid_recorded_history(
13193 "side_effect_payload_malformed",
13194 sequence,
13195 "payload blob or {codec, blob} envelope",
13196 &result.to_string(),
13197 "side-effect history result does not use a published payload envelope",
13198 ));
13199 }
13200 let codec = event
13201 .payload
13202 .get("payload_codec")
13203 .and_then(Value::as_str)
13204 .unwrap_or(fallback_codec);
13205 let value = decode_wire_avro_value(result, codec).map_err(|error| {
13206 if error.to_string().contains("unsupported_payload_codec") {
13207 return error;
13208 }
13209
13210 invalid_recorded_history(
13211 "side_effect_payload_incompatible",
13212 sequence,
13213 &format!("valid {codec} payload envelope"),
13214 &error.to_string(),
13215 "side-effect history payload cannot be decoded with its recorded codec",
13216 )
13217 })?;
13218 return Ok(RecordedCommand::SideEffect { sequence, value });
13219 }
13220
13221 if !version_marker_events.is_empty() {
13222 if version_marker_events.len() != 1 {
13223 return Err(invalid_recorded_history(
13224 "duplicate_version_marker_record",
13225 sequence,
13226 "one VersionMarkerRecorded event",
13227 &format!("{} VersionMarkerRecorded events", version_marker_events.len()),
13228 "version-marker history records one workflow command more than once",
13229 ));
13230 }
13231 let payload = &version_marker_events[0].payload;
13232 let change_id = payload
13233 .get("change_id")
13234 .and_then(Value::as_str)
13235 .filter(|value| !value.is_empty())
13236 .map(str::to_string)
13237 .ok_or_else(|| {
13238 invalid_recorded_history(
13239 "version_marker_field_missing",
13240 sequence,
13241 "non-empty change_id",
13242 "missing or invalid change_id",
13243 "version-marker history is missing its stable change ID",
13244 )
13245 })?;
13246 let version = required_version_i32(payload, "version", sequence)?;
13247 let min_supported = required_version_i32(payload, "min_supported", sequence)?;
13248 let max_supported = required_version_i32(payload, "max_supported", sequence)?;
13249 if min_supported > max_supported || version < min_supported || version > max_supported {
13250 return Err(invalid_recorded_history(
13251 "version_marker_history_range_invalid",
13252 sequence,
13253 "min_supported <= version <= max_supported",
13254 &format!("{min_supported} <= {version} <= {max_supported}"),
13255 "recorded version marker contains an internally incompatible range",
13256 ));
13257 }
13258 return Ok(RecordedCommand::VersionMarker {
13259 sequence,
13260 change_id,
13261 version,
13262 });
13263 }
13264
13265 if !memo_events.is_empty() {
13266 if memo_events.len() != 1 {
13267 return Err(invalid_recorded_history(
13268 "duplicate_memo_upsert_record",
13269 sequence,
13270 "one MemoUpserted event",
13271 &format!("{} MemoUpserted events", memo_events.len()),
13272 "memo history records one workflow update more than once",
13273 ));
13274 }
13275 let payload = &memo_events[0].payload;
13276 let entries = payload.get("entries").cloned().ok_or_else(|| {
13277 invalid_recorded_history(
13278 "memo_entries_missing",
13279 sequence,
13280 "memo entries object",
13281 "missing entries",
13282 "MemoUpserted history is missing replay identity entries",
13283 )
13284 })?;
13285 let entries = decode_memo_history_map(&entries, true).map_err(|error| {
13286 invalid_recorded_history(
13287 "memo_entries_invalid",
13288 sequence,
13289 "valid canonical memo entries",
13290 &error.to_string(),
13291 "MemoUpserted history contains invalid replay identity entries",
13292 )
13293 })?;
13294 let merged = payload.get("merged").cloned().ok_or_else(|| {
13295 invalid_recorded_history(
13296 "memo_merged_projection_missing",
13297 sequence,
13298 "merged memo projection",
13299 "missing merged",
13300 "MemoUpserted history is missing its merged projection",
13301 )
13302 })?;
13303 decode_memo_history_map(&merged, false).map_err(|error| {
13304 invalid_recorded_history(
13305 "memo_merged_projection_invalid",
13306 sequence,
13307 "valid merged memo projection",
13308 &error.to_string(),
13309 "MemoUpserted history contains an invalid merged projection",
13310 )
13311 })?;
13312
13313 return Ok(RecordedCommand::Memo { sequence, entries });
13314 }
13315 let scheduled: Vec<_> = timer_events
13316 .iter()
13317 .copied()
13318 .filter(|event| event.event_type == "TimerScheduled")
13319 .collect();
13320 let fired: Vec<_> = timer_events
13321 .iter()
13322 .copied()
13323 .filter(|event| event.event_type == "TimerFired")
13324 .collect();
13325 if scheduled.len() != 1 {
13326 return Err(invalid_recorded_history(
13327 "timer_schedule_missing_or_duplicate",
13328 sequence,
13329 "one TimerScheduled event",
13330 &format!("{} TimerScheduled events", scheduled.len()),
13331 "timer replay requires exactly one recorded schedule event",
13332 ));
13333 }
13334 if fired.len() > 1 {
13335 return Err(invalid_recorded_history(
13336 "duplicate_timer_fire",
13337 sequence,
13338 "at most one TimerFired event",
13339 "multiple TimerFired events",
13340 "timer history contains more than one fire event for a workflow sequence",
13341 ));
13342 }
13343
13344 let scheduled = scheduled[0];
13345 let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
13346 let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
13347 if let Some(fired) = fired.first() {
13348 let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
13349 if fired_timer_id != timer_id {
13350 return Err(invalid_recorded_history(
13351 "timer_identity_mismatch",
13352 sequence,
13353 &timer_id,
13354 &fired_timer_id,
13355 "TimerFired does not correspond to the recorded TimerScheduled event",
13356 ));
13357 }
13358 let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
13359 if fired_delay != delay_seconds {
13360 return Err(invalid_recorded_history(
13361 "timer_history_delay_mismatch",
13362 sequence,
13363 &delay_seconds.to_string(),
13364 &fired_delay.to_string(),
13365 "TimerScheduled and TimerFired record different delays",
13366 ));
13367 }
13368 }
13369
13370 Ok(RecordedCommand::Timer {
13371 sequence,
13372 delay_seconds,
13373 fired: !fired.is_empty(),
13374 parallel_group_path: recorded_parallel_group_path(&timer_events, sequence)?,
13375 })
13376 })
13377 .collect::<Result<_>>()?;
13378
13379 let mut marker_sequences = HashMap::new();
13380 for command in &commands {
13381 if let RecordedCommand::VersionMarker {
13382 sequence,
13383 change_id,
13384 ..
13385 } = command
13386 {
13387 if let Some(first_sequence) = marker_sequences.insert(change_id.clone(), *sequence) {
13388 return Err(invalid_recorded_history(
13389 "duplicate_version_marker",
13390 *sequence,
13391 &format!("one marker for change ID {change_id:?}"),
13392 &format!("markers at sequences {first_sequence} and {sequence}"),
13393 "workflow history contains duplicate markers for one stable change ID",
13394 ));
13395 }
13396 }
13397 }
13398
13399 Ok(commands)
13400}
13401
13402fn required_version_i32(payload: &Value, field: &str, sequence: u64) -> Result<i32> {
13403 payload
13404 .get(field)
13405 .and_then(Value::as_i64)
13406 .and_then(|value| i32::try_from(value).ok())
13407 .ok_or_else(|| {
13408 invalid_recorded_history(
13409 "version_marker_field_missing",
13410 sequence,
13411 &format!("integer {field}"),
13412 "missing or out-of-range integer",
13413 "version-marker history is missing a required integer field",
13414 )
13415 })
13416}
13417
13418fn durable_event_sequence(event: &HistoryEvent) -> Option<u64> {
13419 event
13420 .payload
13421 .get("sequence")
13422 .or_else(|| event.payload.get("workflow_sequence"))
13423 .or_else(|| event.raw.get("sequence"))
13424 .or_else(|| event.raw.get("workflow_sequence"))
13425 .and_then(value_as_u64)
13426}
13427
13428fn is_internal_timer_event(event: &HistoryEvent) -> bool {
13429 matches!(
13430 event
13431 .payload
13432 .get("timer_kind")
13433 .or_else(|| event.raw.get("timer_kind"))
13434 .and_then(Value::as_str),
13435 Some("condition_timeout" | "signal_timeout")
13436 )
13437}
13438
13439fn is_recorded_condition_wait_event(event: &HistoryEvent) -> bool {
13440 matches!(
13441 event.event_type.as_str(),
13442 "ConditionWaitOpened" | "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
13443 )
13444}
13445
13446fn recorded_condition_wait(
13447 sequence: u64,
13448 condition_events: &[&HistoryEvent],
13449 all_events: &[HistoryEvent],
13450) -> Result<RecordedCommand> {
13451 let opened = condition_events
13452 .iter()
13453 .copied()
13454 .filter(|event| event.event_type == "ConditionWaitOpened")
13455 .collect::<Vec<_>>();
13456 if opened.len() != 1 {
13457 return Err(invalid_recorded_history(
13458 "condition_wait_open_missing_or_duplicate",
13459 sequence,
13460 "one ConditionWaitOpened event",
13461 &format!("{} ConditionWaitOpened events", opened.len()),
13462 "condition replay requires exactly one canonical wait-open event",
13463 ));
13464 }
13465 let terminal = condition_events
13466 .iter()
13467 .copied()
13468 .filter(|event| {
13469 matches!(
13470 event.event_type.as_str(),
13471 "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
13472 )
13473 })
13474 .collect::<Vec<_>>();
13475 if terminal.len() > 1 {
13476 return Err(invalid_recorded_history(
13477 "duplicate_condition_wait_terminal_event",
13478 sequence,
13479 "at most one condition terminal event",
13480 "multiple condition terminal events",
13481 "condition history settles one durable wait more than once",
13482 ));
13483 }
13484
13485 let opened = opened[0];
13486 let condition_wait_id = required_condition_wait_id(opened, sequence)?;
13487 let occurrence_id = required_condition_wait_occurrence_id(opened, sequence)?;
13488 for event in condition_events
13489 .iter()
13490 .copied()
13491 .filter(|event| !std::ptr::eq(*event, opened))
13492 {
13493 let event_wait_id = required_condition_wait_id(event, sequence)?;
13494 if event_wait_id != condition_wait_id {
13495 return Err(invalid_recorded_history(
13496 "condition_wait_id_mismatch",
13497 sequence,
13498 &condition_wait_id,
13499 &event_wait_id,
13500 "condition lifecycle events at one sequence disagree on wait identity",
13501 ));
13502 }
13503 let event_occurrence_id = required_condition_wait_occurrence_id(event, sequence)?;
13504 if event_occurrence_id != occurrence_id {
13505 return Err(invalid_recorded_history(
13506 "condition_wait_occurrence_history_mismatch",
13507 sequence,
13508 &occurrence_id,
13509 &event_occurrence_id,
13510 "condition lifecycle events at one sequence disagree on authored occurrence identity",
13511 ));
13512 }
13513 }
13514
13515 let condition_key = optional_non_empty_history_string(opened, "condition_key");
13516 let predicate_identity = opened
13517 .payload
13518 .get("condition_definition_fingerprint")
13519 .and_then(Value::as_str)
13520 .filter(|value| !value.is_empty())
13521 .map(str::to_string)
13522 .ok_or_else(|| {
13523 invalid_recorded_history(
13524 "condition_wait_predicate_fingerprint_missing",
13525 sequence,
13526 "non-empty condition_definition_fingerprint",
13527 &opened.event_type,
13528 "canonical condition history is missing its predicate identity",
13529 )
13530 })?;
13531 let timeout_seconds = optional_history_u64(opened, "timeout_seconds", sequence)?;
13532 for event in condition_events
13533 .iter()
13534 .copied()
13535 .filter(|event| !std::ptr::eq(*event, opened))
13536 {
13537 for (field, opened_value) in [
13538 ("condition_key", condition_key.as_deref()),
13539 (
13540 "condition_definition_fingerprint",
13541 Some(predicate_identity.as_str()),
13542 ),
13543 ] {
13544 if let Some(value) = optional_non_empty_history_string(event, field) {
13545 if opened_value.is_some_and(|opened_value| opened_value != value) {
13546 return Err(invalid_recorded_history(
13547 "condition_wait_definition_history_mismatch",
13548 sequence,
13549 opened_value.unwrap_or_default(),
13550 &value,
13551 "condition lifecycle events disagree on the recorded definition",
13552 ));
13553 }
13554 }
13555 }
13556 if let Some(event_timeout) = optional_history_u64(event, "timeout_seconds", sequence)? {
13557 if timeout_seconds.is_some_and(|opened_timeout| opened_timeout != event_timeout) {
13558 return Err(invalid_recorded_history(
13559 "condition_wait_definition_history_mismatch",
13560 sequence,
13561 &format!("{}s", timeout_seconds.unwrap_or_default()),
13562 &format!("{event_timeout}s"),
13563 "condition lifecycle events disagree on the recorded timeout",
13564 ));
13565 }
13566 }
13567 }
13568
13569 let timeout_timer_events = all_events
13570 .iter()
13571 .filter(|event| {
13572 matches!(
13573 event.event_type.as_str(),
13574 "TimerScheduled" | "TimerCancelled" | "TimerFired"
13575 ) && event.payload.get("timer_kind").and_then(Value::as_str)
13576 == Some("condition_timeout")
13577 && event
13578 .payload
13579 .get("condition_wait_id")
13580 .and_then(Value::as_str)
13581 == Some(condition_wait_id.as_str())
13582 })
13583 .collect::<Vec<_>>();
13584 let scheduled = timeout_timer_events
13585 .iter()
13586 .copied()
13587 .filter(|event| event.event_type == "TimerScheduled")
13588 .collect::<Vec<_>>();
13589 let fired = timeout_timer_events
13590 .iter()
13591 .copied()
13592 .filter(|event| event.event_type == "TimerFired")
13593 .collect::<Vec<_>>();
13594 if scheduled.len() > 1 || fired.len() > 1 || (!fired.is_empty() && scheduled.len() != 1) {
13595 return Err(invalid_recorded_history(
13596 "condition_wait_timeout_history_invalid",
13597 sequence,
13598 "one timeout schedule and at most one fire",
13599 &format!("{} schedules and {} fires", scheduled.len(), fired.len()),
13600 "condition timeout history has a missing or duplicate lifecycle event",
13601 ));
13602 }
13603 if let Some(scheduled) = scheduled.first() {
13604 let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
13605 let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
13606 if timeout_seconds.is_some_and(|timeout| timeout != delay_seconds) {
13607 return Err(invalid_recorded_history(
13608 "condition_wait_timeout_delay_mismatch",
13609 sequence,
13610 &format!("{}s", timeout_seconds.unwrap_or_default()),
13611 &format!("{delay_seconds}s"),
13612 "condition timeout timer differs from the wait definition",
13613 ));
13614 }
13615 if let Some(fired) = fired.first() {
13616 let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
13617 let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
13618 if fired_timer_id != timer_id || fired_delay != delay_seconds {
13619 return Err(invalid_recorded_history(
13620 "condition_wait_timeout_identity_mismatch",
13621 sequence,
13622 &format!("{timer_id}:{delay_seconds}s"),
13623 &format!("{fired_timer_id}:{fired_delay}s"),
13624 "condition timeout fire does not match its durable schedule",
13625 ));
13626 }
13627 }
13628 }
13629
13630 let result = terminal.first().map(|event| {
13631 if event.event_type == "ConditionWaitTimedOut" {
13632 ConditionWaitResult::TimedOut
13633 } else {
13634 ConditionWaitResult::Satisfied
13635 }
13636 });
13637 let result = if !fired.is_empty() {
13638 if result == Some(ConditionWaitResult::Satisfied) {
13639 return Err(invalid_recorded_history(
13640 "condition_wait_terminal_conflict",
13641 sequence,
13642 "one satisfied or timed-out outcome",
13643 "satisfied event and fired timeout",
13644 "condition history records conflicting terminal outcomes",
13645 ));
13646 }
13647 Some(ConditionWaitResult::TimedOut)
13648 } else {
13649 result
13650 };
13651
13652 Ok(RecordedCommand::ConditionWait {
13653 sequence,
13654 occurrence_id,
13655 condition_key,
13656 predicate_identity,
13657 timeout_seconds,
13658 result,
13659 parallel_group_path: recorded_parallel_group_path(condition_events, sequence)?,
13660 })
13661}
13662
13663fn required_condition_wait_occurrence_id(event: &HistoryEvent, sequence: u64) -> Result<String> {
13664 event
13665 .payload
13666 .get("condition_wait_occurrence_id")
13667 .and_then(Value::as_str)
13668 .filter(|value| !value.is_empty())
13669 .map(str::to_string)
13670 .ok_or_else(|| {
13671 invalid_recorded_history(
13672 "condition_wait_occurrence_id_missing",
13673 sequence,
13674 "non-empty condition_wait_occurrence_id",
13675 &event.event_type,
13676 "condition history is missing authored occurrence identity",
13677 )
13678 })
13679}
13680
13681fn required_condition_wait_id(event: &HistoryEvent, sequence: u64) -> Result<String> {
13682 event
13683 .payload
13684 .get("condition_wait_id")
13685 .and_then(Value::as_str)
13686 .filter(|value| !value.is_empty())
13687 .map(str::to_string)
13688 .ok_or_else(|| {
13689 invalid_recorded_history(
13690 "condition_wait_id_missing",
13691 sequence,
13692 "non-empty condition_wait_id",
13693 &event.event_type,
13694 "canonical condition history is missing its durable wait identity",
13695 )
13696 })
13697}
13698
13699fn optional_non_empty_history_string(event: &HistoryEvent, field: &str) -> Option<String> {
13700 event
13701 .payload
13702 .get(field)
13703 .and_then(Value::as_str)
13704 .filter(|value| !value.is_empty())
13705 .map(str::to_string)
13706}
13707
13708fn optional_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<Option<u64>> {
13709 match event.payload.get(field) {
13710 None | Some(Value::Null) => Ok(None),
13711 Some(value) => value_as_u64(value).map(Some).ok_or_else(|| {
13712 invalid_recorded_history(
13713 "condition_wait_definition_invalid",
13714 sequence,
13715 &format!("non-negative integer {field}"),
13716 &value.to_string(),
13717 "condition history contains an invalid numeric definition field",
13718 )
13719 }),
13720 }
13721}
13722
13723fn required_signal_wait_name(event: &HistoryEvent, sequence: u64) -> Result<String> {
13724 event
13725 .payload
13726 .get("signal_name")
13727 .or_else(|| event.raw.get("signal_name"))
13728 .and_then(Value::as_str)
13729 .filter(|value| !value.is_empty())
13730 .map(str::to_string)
13731 .ok_or_else(|| {
13732 invalid_recorded_history(
13733 "signal_wait_name_missing",
13734 sequence,
13735 "non-empty signal_name",
13736 &event.event_type,
13737 "canonical signal-wait history is missing its signal identity",
13738 )
13739 })
13740}
13741
13742fn is_recorded_signal_wait_event(event: &HistoryEvent) -> bool {
13743 matches!(
13744 event.event_type.as_str(),
13745 "SignalWaitOpened" | "SignalApplied"
13746 )
13747}
13748
13749fn required_history_string(event: &HistoryEvent, field: &str, sequence: u64) -> Result<String> {
13750 event
13751 .payload
13752 .get(field)
13753 .and_then(Value::as_str)
13754 .filter(|value| !value.is_empty())
13755 .map(str::to_string)
13756 .ok_or_else(|| {
13757 invalid_recorded_history(
13758 "timer_history_field_missing",
13759 sequence,
13760 field,
13761 &event.event_type,
13762 "timer history is missing a required identity field",
13763 )
13764 })
13765}
13766
13767fn required_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<u64> {
13768 event
13769 .payload
13770 .get(field)
13771 .and_then(value_as_u64)
13772 .ok_or_else(|| {
13773 invalid_recorded_history(
13774 "timer_history_field_missing",
13775 sequence,
13776 field,
13777 &event.event_type,
13778 "timer history is missing a required numeric field",
13779 )
13780 })
13781}
13782
13783fn recorded_search_attribute_types(
13784 payload: &Value,
13785 attributes: &Value,
13786 sequence: u64,
13787) -> Result<RecordedSnapshotValue<BTreeMap<String, String>>> {
13788 let Some(raw_types) = payload.get("attribute_types") else {
13789 return Ok(RecordedSnapshotValue::Unknown);
13793 };
13794 let Some(raw_types) = raw_types.as_object() else {
13795 return Err(invalid_recorded_history(
13796 "search_attribute_types_malformed",
13797 sequence,
13798 "canonical attribute type map",
13799 &raw_types.to_string(),
13800 "search-attribute history contains malformed type identity",
13801 ));
13802 };
13803 let attribute_keys = attributes
13804 .as_object()
13805 .expect("recorded search attributes were validated as an object");
13806 let mut types = BTreeMap::new();
13807 for (key, value) in raw_types {
13808 let Some(attribute_type) = value.as_str() else {
13809 return Err(invalid_recorded_history(
13810 "search_attribute_types_malformed",
13811 sequence,
13812 "canonical string type name",
13813 &value.to_string(),
13814 "search-attribute history contains a non-string type identity",
13815 ));
13816 };
13817 if !attribute_keys.contains_key(key)
13818 || !matches!(
13819 attribute_type,
13820 "string" | "keyword" | "keyword_list" | "int" | "float" | "bool" | "datetime"
13821 )
13822 {
13823 return Err(invalid_recorded_history(
13824 "search_attribute_types_malformed",
13825 sequence,
13826 "canonical types for keys present in attributes",
13827 &format!("{key}:{attribute_type}"),
13828 "search-attribute history contains unsupported or orphaned type identity",
13829 ));
13830 }
13831 types.insert(key.clone(), attribute_type.to_string());
13832 }
13833 Ok(RecordedSnapshotValue::Known(types))
13834}
13835
13836fn invalid_recorded_history(
13837 reason: &str,
13838 sequence: u64,
13839 expected: &str,
13840 actual: &str,
13841 message: &str,
13842) -> Error {
13843 Error::NonDeterministicReplay(ReplayFailure::new(
13844 reason,
13845 Some(sequence),
13846 Some(expected.to_string()),
13847 Some(actual.to_string()),
13848 message,
13849 ))
13850}
13851
13852type ActivityOutcome = std::result::Result<AvroValue, ActivityFailure>;
13853
13854fn activity_outcome(
13855 event: &HistoryEvent,
13856 fallback_codec: &str,
13857 recorded_activity_type: Option<String>,
13858) -> Result<ActivityOutcome> {
13859 if event.event_type == "ActivityCompleted" {
13860 let codec = event
13861 .payload
13862 .get("payload_codec")
13863 .and_then(Value::as_str)
13864 .unwrap_or(fallback_codec);
13865 return Ok(Ok(decode_wire_avro_value(
13866 event.payload.get("result").unwrap_or(&Value::Null),
13867 codec,
13868 )?));
13869 }
13870
13871 let payload = &event.payload;
13872 let (kind, fallback_reason, fallback_message) = match event.event_type.as_str() {
13873 "ActivityFailed" => (ActivityFailureKind::Failed, "activity", "activity failed"),
13874 "ActivityCancelled" => (
13875 ActivityFailureKind::Cancelled,
13876 "cancelled",
13877 "activity was cancelled",
13878 ),
13879 "ActivityTimedOut" => (
13880 ActivityFailureKind::TimedOut,
13881 "timeout",
13882 "activity timed out",
13883 ),
13884 _ => unreachable!("activity_outcome is called only for terminal activity events"),
13885 };
13886 let exception = payload
13887 .get("exception")
13888 .filter(|value| !value.is_null())
13889 .cloned();
13890 let failure_category = payload_string(payload, "failure_category");
13891 let timeout_kind = payload_string(payload, "timeout_kind");
13892 let reason = payload_string(payload, "reason").unwrap_or_else(|| match kind {
13893 ActivityFailureKind::Failed => failure_category
13894 .clone()
13895 .unwrap_or_else(|| fallback_reason.to_string()),
13896 ActivityFailureKind::Cancelled => fallback_reason.to_string(),
13897 ActivityFailureKind::TimedOut => timeout_kind
13898 .clone()
13899 .unwrap_or_else(|| fallback_reason.to_string()),
13900 });
13901 let message = payload_string(payload, "message")
13902 .or_else(|| {
13903 exception
13904 .as_ref()
13905 .and_then(|value| payload_string(value, "message"))
13906 })
13907 .unwrap_or_else(|| fallback_message.to_string());
13908
13909 Ok(Err(ActivityFailure {
13910 kind,
13911 reason,
13912 message,
13913 activity_execution_id: payload_string(payload, "activity_execution_id"),
13914 activity_attempt_id: payload_string(payload, "activity_attempt_id"),
13915 activity_type: payload_string(payload, "activity_type")
13916 .or_else(|| payload_string(payload, "activity_name"))
13917 .or(recorded_activity_type),
13918 activity_class: payload_string(payload, "activity_class"),
13919 attempt_number: payload.get("attempt_number").and_then(value_as_u64),
13920 failure_id: payload_string(payload, "failure_id"),
13921 failure_category,
13922 timeout_kind,
13923 non_retryable: payload
13924 .get("non_retryable")
13925 .and_then(Value::as_bool)
13926 .unwrap_or(false),
13927 exception_type: payload_string(payload, "exception_type").or_else(|| {
13928 exception
13929 .as_ref()
13930 .and_then(|value| payload_string(value, "type"))
13931 }),
13932 exception_class: payload_string(payload, "exception_class").or_else(|| {
13933 exception
13934 .as_ref()
13935 .and_then(|value| payload_string(value, "class"))
13936 }),
13937 code: payload
13938 .get("code")
13939 .filter(|value| !value.is_null())
13940 .cloned(),
13941 exception,
13942 }))
13943}
13944
13945type ChildWorkflowOutcome = std::result::Result<ChildWorkflowAvroResult, ChildWorkflowFailure>;
13946
13947fn child_workflow_outcomes(
13948 events: &[HistoryEvent],
13949 fallback_codec: &str,
13950 parent: WorkflowIdentity,
13951) -> Result<Vec<ChildWorkflowOutcome>> {
13952 let mut outcomes = Vec::new();
13953
13954 for event in events {
13955 let kind = match event.event_type.as_str() {
13956 "ChildRunCompleted" => None,
13957 "ChildRunFailed" => Some((
13958 ChildWorkflowFailureKind::Failed,
13959 "child_workflow",
13960 "child workflow failed",
13961 )),
13962 "ChildRunCancelled" => Some((
13963 ChildWorkflowFailureKind::Cancelled,
13964 "cancelled",
13965 "child workflow was cancelled",
13966 )),
13967 "ChildRunTerminated" => Some((
13968 ChildWorkflowFailureKind::Terminated,
13969 "terminated",
13970 "child workflow was terminated",
13971 )),
13972 _ => continue,
13973 };
13974 let payload = &event.payload;
13975 let child_workflow_id = payload_string(payload, "child_workflow_instance_id");
13976 let child_workflow_run_id = payload_string(payload, "child_workflow_run_id");
13977 let child_workflow_type = payload_string(payload, "child_workflow_type");
13978
13979 if let Some((kind, reason, fallback_message)) = kind {
13980 let exception = payload
13981 .get("exception")
13982 .filter(|value| !value.is_null())
13983 .cloned();
13984 let message = payload_string(payload, "message")
13985 .or_else(|| {
13986 exception
13987 .as_ref()
13988 .and_then(|value| payload_string(value, "message"))
13989 })
13990 .unwrap_or_else(|| fallback_message.to_string());
13991 let exception_type = payload_string(payload, "exception_type").or_else(|| {
13992 exception
13993 .as_ref()
13994 .and_then(|value| payload_string(value, "type"))
13995 });
13996 let exception_class = payload_string(payload, "exception_class").or_else(|| {
13997 exception
13998 .as_ref()
13999 .and_then(|value| payload_string(value, "class"))
14000 });
14001 outcomes.push(Err(ChildWorkflowFailure {
14002 kind,
14003 reason: reason.to_string(),
14004 message,
14005 parent_workflow_id: parent.workflow_id.clone(),
14006 parent_workflow_run_id: parent.run_id.clone(),
14007 child_workflow_id,
14008 child_workflow_run_id,
14009 child_workflow_type,
14010 failure_id: payload_string(payload, "failure_id"),
14011 failure_category: payload_string(payload, "failure_category"),
14012 exception_type,
14013 exception_class,
14014 non_retryable: payload
14015 .get("non_retryable")
14016 .and_then(Value::as_bool)
14017 .unwrap_or(false),
14018 code: payload
14019 .get("code")
14020 .filter(|value| !value.is_null())
14021 .cloned(),
14022 exception,
14023 }));
14024 continue;
14025 }
14026
14027 let codec = payload
14028 .get("payload_codec")
14029 .and_then(Value::as_str)
14030 .unwrap_or(fallback_codec);
14031 let result = payload
14032 .get("result")
14033 .or_else(|| payload.get("output"))
14034 .unwrap_or(&Value::Null);
14035 outcomes.push(Ok(ChildWorkflowAvroResult {
14036 parent: parent.clone(),
14037 child: WorkflowIdentity {
14038 workflow_id: child_workflow_id,
14039 run_id: child_workflow_run_id,
14040 },
14041 child_workflow_type,
14042 result: decode_wire_avro_value(result, codec)?,
14043 }));
14044 }
14045
14046 Ok(outcomes)
14047}
14048
14049fn payload_string(payload: &Value, key: &str) -> Option<String> {
14050 payload
14051 .get(key)
14052 .and_then(Value::as_str)
14053 .filter(|value| !value.is_empty())
14054 .map(str::to_string)
14055}
14056
14057fn workflow_failure_command(error: &Error) -> Value {
14058 let (exception_type, exception_class, properties) = match error {
14059 Error::ActivityFailed(failure) => (
14060 match failure.kind {
14061 ActivityFailureKind::Failed => "ActivityFailed",
14062 ActivityFailureKind::Cancelled => "ActivityCancelled",
14063 ActivityFailureKind::TimedOut => "ActivityTimedOut",
14064 },
14065 "durable_workflow::ActivityFailure",
14066 json!({
14067 "reason": failure.reason,
14068 "activity_execution_id": failure.activity_execution_id,
14069 "activity_attempt_id": failure.activity_attempt_id,
14070 "activity_type": failure.activity_type,
14071 "activity_class": failure.activity_class,
14072 "attempt_number": failure.attempt_number,
14073 "failure_id": failure.failure_id,
14074 "failure_category": failure.failure_category,
14075 "timeout_kind": failure.timeout_kind,
14076 "activity_non_retryable": failure.non_retryable,
14077 "activity_exception_type": failure.exception_type,
14078 "activity_exception_class": failure.exception_class,
14079 "activity_code": failure.code,
14080 "activity_exception": failure.exception,
14081 }),
14082 ),
14083 Error::ChildWorkflowFailed(failure) => (
14084 match failure.kind {
14085 ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
14086 ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
14087 ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
14088 },
14089 "durable_workflow::ChildWorkflowFailure",
14090 json!({
14091 "reason": failure.reason,
14092 "parent_workflow_id": failure.parent_workflow_id,
14093 "parent_workflow_run_id": failure.parent_workflow_run_id,
14094 "child_workflow_id": failure.child_workflow_id,
14095 "child_workflow_run_id": failure.child_workflow_run_id,
14096 "child_workflow_type": failure.child_workflow_type,
14097 "failure_id": failure.failure_id,
14098 "failure_category": failure.failure_category,
14099 "child_exception_type": failure.exception_type,
14100 "child_exception_class": failure.exception_class,
14101 "child_non_retryable": failure.non_retryable,
14102 "child_code": failure.code,
14103 "child_exception": failure.exception,
14104 }),
14105 ),
14106 Error::ParallelFailed(failure) => (
14107 "ParallelFailed",
14108 "durable_workflow::ParallelFailure",
14109 json!({
14110 "parallel_group_id": failure.group_id,
14111 "parallel_member_path": failure.member_path,
14112 "parallel_group_path": failure.group_path,
14113 "completed_members": failure.completed.iter().map(|completion| &completion.member_path).collect::<Vec<_>>(),
14114 "cause_type": workflow_error_type(&failure.cause),
14115 "cause_message": failure.cause.to_string(),
14116 }),
14117 ),
14118 Error::SagaCompensationFailed(failure) => (
14119 "SagaCompensationFailed",
14120 "durable_workflow::SagaCompensationFailure",
14121 json!({
14122 "initiating_failure_type": workflow_error_type(&failure.initiating_failure),
14123 "initiating_failure_message": failure.initiating_failure.to_string(),
14124 "compensation_activity_type": failure.compensation_activity_type,
14125 "compensation_registration_order": failure.compensation_registration_order,
14126 "compensation_failure_type": workflow_error_type(&failure.compensation_failure),
14127 "compensation_failure_message": failure.compensation_failure.to_string(),
14128 }),
14129 ),
14130 Error::WorkflowCancellationRequested(_) => (
14131 "WorkflowCancellationRequested",
14132 "durable_workflow::WorkflowCancellationRequested",
14133 json!({"reason": "cancelled"}),
14134 ),
14135 Error::NonDeterministicReplay(_) => (
14136 "NonDeterministicReplay",
14137 "durable_workflow::Error",
14138 Value::Null,
14139 ),
14140 _ => ("RustWorkflowError", "durable_workflow::Error", Value::Null),
14141 };
14142 let non_retryable = match error {
14143 Error::ActivityFailed(failure) => failure.non_retryable,
14144 Error::ChildWorkflowFailed(failure) => failure.non_retryable,
14145 Error::ParallelFailed(failure) => workflow_error_non_retryable(&failure.cause),
14146 Error::SagaCompensationFailed(failure) => {
14147 workflow_error_non_retryable(&failure.compensation_failure)
14148 }
14149 Error::WorkflowCancellationRequested(_) => true,
14150 Error::NonDeterministicReplay(_) => true,
14151 _ => false,
14152 };
14153
14154 json!({
14155 "type": "fail_workflow",
14156 "message": error.to_string(),
14157 "exception_type": exception_type,
14158 "exception_class": exception_class,
14159 "non_retryable": non_retryable,
14160 "exception": {
14161 "type": exception_type,
14162 "class": exception_class,
14163 "message": error.to_string(),
14164 "properties": properties,
14165 }
14166 })
14167}
14168
14169fn workflow_error_type(error: &Error) -> &'static str {
14170 match error {
14171 Error::ActivityFailed(failure) => match failure.kind {
14172 ActivityFailureKind::Failed => "ActivityFailed",
14173 ActivityFailureKind::Cancelled => "ActivityCancelled",
14174 ActivityFailureKind::TimedOut => "ActivityTimedOut",
14175 },
14176 Error::ChildWorkflowFailed(failure) => match failure.kind {
14177 ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
14178 ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
14179 ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
14180 },
14181 Error::ParallelFailed(_) => "ParallelFailed",
14182 Error::SagaCompensationFailed(_) => "SagaCompensationFailed",
14183 Error::WorkflowCancellationRequested(_) => "WorkflowCancellationRequested",
14184 Error::NonDeterministicReplay(_) => "NonDeterministicReplay",
14185 _ => "RustWorkflowError",
14186 }
14187}
14188
14189fn workflow_error_non_retryable(error: &Error) -> bool {
14190 match error {
14191 Error::ActivityFailed(failure) => failure.non_retryable,
14192 Error::ChildWorkflowFailed(failure) => failure.non_retryable,
14193 Error::ParallelFailed(failure) => workflow_error_non_retryable(&failure.cause),
14194 Error::SagaCompensationFailed(failure) => {
14195 workflow_error_non_retryable(&failure.compensation_failure)
14196 }
14197 Error::WorkflowCancellationRequested(_) | Error::NonDeterministicReplay(_) => true,
14198 _ => false,
14199 }
14200}
14201
14202fn workflow_task_integrity_error(error: &Error) -> bool {
14203 matches!(
14204 error,
14205 Error::NonDeterministicReplay(_)
14206 | Error::Protocol(_)
14207 | Error::MissingWorkflowCommandIdentity
14208 | Error::WorkflowStatePoisoned
14209 )
14210}
14211
14212fn decode_signal_event_arguments(
14213 event: &HistoryEvent,
14214 fallback_codec: &str,
14215) -> Result<Vec<AvroValue>> {
14216 let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
14217 validate_payload_codec(codec)?;
14218 let raw = signal_history_payload(&event.payload);
14219 let decoded = match raw.filter(|value| !value.is_null()) {
14220 Some(value) => decode_wire_avro_value(value, codec)?,
14221 None => AvroValue::Array(Vec::new()),
14222 };
14223 let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
14224 unreachable!("normalize_avro_arguments always returns an array");
14225 };
14226 Ok(arguments)
14227}
14228
14229fn decode_update_event_arguments(
14230 event: &HistoryEvent,
14231 fallback_codec: &str,
14232) -> Result<Vec<AvroValue>> {
14233 let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
14234 validate_payload_codec(codec)?;
14235 let decoded = match event
14236 .payload
14237 .get("arguments")
14238 .filter(|value| !value.is_null())
14239 {
14240 Some(value) => decode_wire_avro_value(value, codec)?,
14241 None => AvroValue::Array(Vec::new()),
14242 };
14243 let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
14244 unreachable!("normalize_avro_arguments always returns an array");
14245 };
14246 Ok(arguments)
14247}
14248
14249fn hydrate_query_history_from_export(task: &mut QueryTask) -> Result<()> {
14250 let Some(export_events) = task
14251 .history_export
14252 .as_ref()
14253 .and_then(|export| export.get("history_events"))
14254 .and_then(Value::as_array)
14255 else {
14256 return Ok(());
14257 };
14258
14259 if export_events.len() > task.history_events.len() {
14260 task.history_events = serde_json::from_value(Value::Array(export_events.clone()))?;
14261 }
14262
14263 Ok(())
14264}
14265
14266fn enrich_query_history_from_export(task: &mut QueryTask) -> Result<()> {
14267 let Some(export) = task.history_export.as_ref() else {
14268 return Ok(());
14269 };
14270 let signals = export
14271 .get("signals")
14272 .and_then(Value::as_array)
14273 .cloned()
14274 .unwrap_or_default();
14275 let activities = export
14276 .get("activities")
14277 .and_then(Value::as_array)
14278 .cloned()
14279 .unwrap_or_default();
14280 let export_codec = export
14281 .get("payloads")
14282 .and_then(|payloads| payloads.get("codec"))
14283 .and_then(Value::as_str)
14284 .unwrap_or(&task.payload_codec)
14285 .to_string();
14286 let mut signal_name_offsets: HashMap<String, usize> = HashMap::new();
14287
14288 for event in &mut task.history_events {
14289 if event.event_type == "ActivityCompleted" {
14290 let sequence = event
14291 .payload
14292 .get("sequence")
14293 .or_else(|| event.payload.get("workflow_sequence"))
14294 .and_then(value_as_u64);
14295 let Some(activity) = sequence.and_then(|sequence| {
14296 activities.iter().find(|activity| {
14297 activity.get("sequence").and_then(value_as_u64) == Some(sequence)
14298 })
14299 }) else {
14300 continue;
14301 };
14302 let Some(payload) = event.payload.as_object_mut() else {
14303 continue;
14304 };
14305 if missing_payload(payload.get("result")) {
14306 if let Some(result) = activity
14307 .get("result")
14308 .filter(|value| !missing_payload(Some(value)))
14309 {
14310 payload.insert("result".to_string(), result.clone());
14311 }
14312 }
14313 for field in ["payload_codec", "activity_type"] {
14314 if payload
14315 .get(field)
14316 .and_then(Value::as_str)
14317 .unwrap_or_default()
14318 .is_empty()
14319 {
14320 if let Some(value) = activity.get(field) {
14321 payload.insert(field.to_string(), value.clone());
14322 }
14323 }
14324 }
14325 continue;
14326 }
14327
14328 if event.event_type != "SignalReceived" && event.event_type != "SignalApplied" {
14329 continue;
14330 }
14331 let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
14332 let command_id = event
14333 .payload
14334 .get("workflow_command_id")
14335 .or_else(|| event.raw.get("workflow_command_id"))
14336 .and_then(Value::as_str);
14337 let signal_name = event
14338 .payload
14339 .get("signal_name")
14340 .and_then(Value::as_str)
14341 .unwrap_or_default()
14342 .to_string();
14343 let matched = signals
14344 .iter()
14345 .find(|signal| {
14346 signal_id.is_some() && signal.get("id").and_then(Value::as_str) == signal_id
14347 })
14348 .or_else(|| {
14349 signals.iter().find(|signal| {
14350 command_id.is_some()
14351 && signal.get("command_id").and_then(Value::as_str) == command_id
14352 })
14353 })
14354 .or_else(|| {
14355 let offset = signal_name_offsets.entry(signal_name.clone()).or_default();
14356 let signal = signals
14357 .iter()
14358 .filter(|signal| {
14359 signal.get("name").and_then(Value::as_str) == Some(signal_name.as_str())
14360 })
14361 .nth(*offset);
14362 if signal.is_some() {
14363 *offset += 1;
14364 }
14365 signal
14366 });
14367 let Some(signal) = matched else {
14368 continue;
14369 };
14370 let signal_codec = signal
14371 .get("payload_codec")
14372 .and_then(Value::as_str)
14373 .unwrap_or(&export_codec);
14374 let Some(payload) = event.payload.as_object_mut() else {
14375 continue;
14376 };
14377 if missing_payload(payload.get("arguments")) {
14378 if let Some(arguments) = signal
14379 .get("arguments")
14380 .filter(|value| !missing_payload(Some(value)))
14381 {
14382 let envelope = match arguments {
14383 Value::String(blob) => json!({"codec": signal_codec, "blob": blob}),
14384 other => other.clone(),
14385 };
14386 payload.insert("arguments".to_string(), envelope);
14387 }
14388 }
14389 if payload
14390 .get("payload_codec")
14391 .and_then(Value::as_str)
14392 .unwrap_or_default()
14393 .is_empty()
14394 {
14395 payload.insert("payload_codec".to_string(), json!(signal_codec));
14396 }
14397 }
14398
14399 Ok(())
14400}
14401
14402fn missing_payload(value: Option<&Value>) -> bool {
14403 match value {
14404 None | Some(Value::Null) => true,
14405 Some(Value::String(value)) => value.is_empty(),
14406 Some(_) => false,
14407 }
14408}
14409
14410fn query_signal_events(task: &QueryTask) -> Result<Vec<QuerySignal>> {
14411 let export_signals = task
14412 .history_export
14413 .as_ref()
14414 .and_then(|export| export.get("signals"))
14415 .and_then(Value::as_array)
14416 .cloned()
14417 .unwrap_or_default();
14418 let export_codec = task
14419 .history_export
14420 .as_ref()
14421 .and_then(|export| export.get("payloads"))
14422 .and_then(|payloads| payloads.get("codec"))
14423 .and_then(Value::as_str)
14424 .unwrap_or(&task.payload_codec);
14425 let mut name_offsets: HashMap<String, usize> = HashMap::new();
14426 let mut signals = Vec::new();
14427
14428 for event in &task.history_events {
14429 if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
14430 continue;
14431 }
14432
14433 let name = event
14434 .payload
14435 .get("signal_name")
14436 .and_then(Value::as_str)
14437 .unwrap_or_default();
14438 if name.is_empty() {
14439 continue;
14440 }
14441 let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
14442 let command_id = event
14443 .payload
14444 .get("workflow_command_id")
14445 .or_else(|| event.raw.get("workflow_command_id"))
14446 .and_then(Value::as_str);
14447 let matched_export = export_signals
14448 .iter()
14449 .find(|candidate| {
14450 signal_id.is_some() && candidate.get("id").and_then(Value::as_str) == signal_id
14451 })
14452 .or_else(|| {
14453 export_signals.iter().find(|candidate| {
14454 command_id.is_some()
14455 && candidate.get("command_id").and_then(Value::as_str) == command_id
14456 })
14457 })
14458 .or_else(|| {
14459 let offset = name_offsets.entry(name.to_string()).or_default();
14460 let candidate = export_signals
14461 .iter()
14462 .filter(|candidate| candidate.get("name").and_then(Value::as_str) == Some(name))
14463 .nth(*offset);
14464 if candidate.is_some() {
14465 *offset += 1;
14466 }
14467 candidate
14468 });
14469 let codec = event
14470 .payload
14471 .get("payload_codec")
14472 .and_then(Value::as_str)
14473 .or_else(|| {
14474 matched_export
14475 .and_then(|signal| signal.get("payload_codec"))
14476 .and_then(Value::as_str)
14477 })
14478 .unwrap_or(export_codec);
14479 let raw_arguments = signal_history_payload(&event.payload)
14480 .filter(|value| !value.is_null())
14481 .or_else(|| matched_export.and_then(|signal| signal.get("arguments")));
14482 let (arguments, avro_arguments) = decode_query_signal_arguments(raw_arguments, codec)?;
14483 let workflow_sequence = event
14484 .payload
14485 .get("workflow_sequence")
14486 .and_then(value_as_u64)
14487 .or_else(|| {
14488 matched_export
14489 .and_then(|signal| signal.get("workflow_sequence"))
14490 .and_then(value_as_u64)
14491 });
14492
14493 signals.push(QuerySignal {
14494 id: signal_id.map(str::to_string).or_else(|| {
14495 matched_export
14496 .and_then(|signal| signal.get("id"))
14497 .and_then(Value::as_str)
14498 .map(str::to_string)
14499 }),
14500 name: name.to_string(),
14501 arguments,
14502 avro_arguments,
14503 workflow_sequence,
14504 });
14505 }
14506
14507 if signals.is_empty() {
14508 for signal in export_signals {
14509 if signal.get("status").and_then(Value::as_str) == Some("rejected") {
14510 continue;
14511 }
14512 let Some(name) = signal.get("name").and_then(Value::as_str) else {
14513 continue;
14514 };
14515 let codec = signal
14516 .get("payload_codec")
14517 .and_then(Value::as_str)
14518 .unwrap_or(export_codec);
14519 let (arguments, avro_arguments) =
14520 decode_query_signal_arguments(signal.get("arguments"), codec)?;
14521 signals.push(QuerySignal {
14522 id: signal.get("id").and_then(Value::as_str).map(str::to_string),
14523 name: name.to_string(),
14524 arguments,
14525 avro_arguments,
14526 workflow_sequence: signal.get("workflow_sequence").and_then(value_as_u64),
14527 });
14528 }
14529 signals.sort_by_key(|signal| signal.workflow_sequence.unwrap_or(u64::MAX));
14530 }
14531
14532 Ok(signals)
14533}
14534
14535fn decode_query_signal_arguments(
14536 raw: Option<&Value>,
14537 codec: &str,
14538) -> Result<(Vec<Value>, Vec<AvroValue>)> {
14539 validate_payload_codec(codec)?;
14540 let decoded = match raw.filter(|value| !value.is_null()) {
14541 Some(value) => decode_wire_avro_value(value, codec)?,
14542 None => AvroValue::Array(Vec::new()),
14543 };
14544 let AvroValue::Array(avro_arguments) = normalize_avro_arguments(decoded) else {
14545 unreachable!("normalize_avro_arguments always returns an array");
14546 };
14547 let arguments = avro_arguments
14548 .iter()
14549 .cloned()
14550 .map(AvroValue::into_json)
14551 .collect::<Result<Vec<_>>>()?;
14552 Ok((arguments, avro_arguments))
14553}
14554
14555fn value_as_u64(value: &Value) -> Option<u64> {
14556 value
14557 .as_u64()
14558 .or_else(|| value.as_str().and_then(|value| value.parse().ok()))
14559}
14560
14561#[cfg(test)]
14562mod tests {
14563 use super::*;
14564 use std::{
14565 fs,
14566 io::{Read, Write},
14567 net::{SocketAddr, TcpListener, TcpStream},
14568 process::Command as ProcessCommand,
14569 sync::atomic::AtomicUsize,
14570 thread,
14571 };
14572
14573 #[derive(Clone, Copy, Debug)]
14574 enum InvalidTaskPayloadCodec {
14575 Missing,
14576 Null,
14577 NonString,
14578 }
14579
14580 impl InvalidTaskPayloadCodec {
14581 fn label(self) -> &'static str {
14582 match self {
14583 Self::Missing => "missing",
14584 Self::Null => "null",
14585 Self::NonString => "non-string",
14586 }
14587 }
14588
14589 fn apply(self, task: &mut Value) {
14590 let task = task.as_object_mut().expect("task fixture object");
14591 match self {
14592 Self::Missing => {
14593 task.remove("payload_codec");
14594 }
14595 Self::Null => {
14596 task.insert("payload_codec".to_string(), Value::Null);
14597 }
14598 Self::NonString => {
14599 task.insert("payload_codec".to_string(), json!(42));
14600 }
14601 }
14602 }
14603 }
14604
14605 fn fixture_envelope(value: Value) -> Value {
14606 encode_value_envelope(&value, DEFAULT_CODEC).expect("encode Avro test fixture")
14607 }
14608
14609 fn fixture_blob(value: Value) -> String {
14610 encode_payload(&value, DEFAULT_CODEC)
14611 .expect("encode Avro test fixture")
14612 .blob
14613 }
14614
14615 #[test]
14616 fn client_builder_rejects_the_sdk_owned_api_suffix() {
14617 for base_url in [
14618 "http://127.0.0.1:8080/api",
14619 "http://localhost:8080/api/",
14620 "https://runtime.example.test/namespaces/orders/api",
14621 ] {
14622 let error = Client::builder(base_url)
14623 .build()
14624 .expect_err("SDK-owned /api suffix must be rejected during build");
14625
14626 assert!(matches!(error, Error::InvalidBaseUrl), "{base_url}");
14627 assert!(
14628 error.to_string().contains("SDK appends /api automatically"),
14629 "the validation error must explain how to fix the endpoint"
14630 );
14631 }
14632 }
14633
14634 #[test]
14635 fn client_builder_preserves_self_hosted_and_managed_runtime_prefixes() {
14636 for (base_url, expected) in [
14637 ("http://127.0.0.1:8080", "http://127.0.0.1:8080"),
14638 (
14639 "http://localhost:8080/durable-workflow/",
14640 "http://localhost:8080/durable-workflow",
14641 ),
14642 (
14643 "https://runtime.example.test/namespaces/orders",
14644 "https://runtime.example.test/namespaces/orders",
14645 ),
14646 (
14647 "https://runtime.example.test/gateway/api/namespaces/orders",
14648 "https://runtime.example.test/gateway/api/namespaces/orders",
14649 ),
14650 (
14651 "https://api.example.test/runtime/orders/",
14652 "https://api.example.test/runtime/orders",
14653 ),
14654 ] {
14655 let client = Client::builder(base_url)
14656 .build()
14657 .expect("Server and Cloud runtime base URL must remain valid");
14658
14659 assert_eq!(client.base_url, expected);
14660 }
14661 }
14662
14663 #[test]
14664 fn workflow_completion_uses_the_additive_command_protocol_floor() {
14665 assert_eq!(
14666 workflow_completion_protocol_version(&[json!({"type": "complete_workflow"})]),
14667 WORKER_PROTOCOL_VERSION
14668 );
14669 assert_eq!(
14670 workflow_completion_protocol_version(&[json!({
14671 "type": "upsert_search_attributes",
14672 "attributes": {"OrderStatus": "waiting"},
14673 })]),
14674 SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION
14675 );
14676 assert_eq!(
14677 workflow_completion_protocol_version(&[json!({
14678 "type": "upsert_search_attributes",
14679 "attributes": {"OrderStatus": "waiting"},
14680 "attribute_types": {"OrderStatus": "keyword"},
14681 })]),
14682 TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION
14683 );
14684 assert_eq!(
14685 workflow_completion_protocol_version(&[
14686 json!({"type": "upsert_memo", "entries": {"status": "waiting"}}),
14687 json!({"type": "open_condition_wait", "condition_key": "ready"}),
14688 ]),
14689 MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION
14690 );
14691 assert_eq!(
14692 workflow_completion_protocol_version(&[
14693 json!({"type": "upsert_search_attributes", "attributes": {"State": "waiting"}}),
14694 json!({"type": "open_condition_wait", "condition_key": "ready"}),
14695 ]),
14696 CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION
14697 );
14698 assert_eq!(
14699 workflow_completion_protocol_version(&[json!({
14700 "type": "open_condition_wait",
14701 "condition_wait_occurrence_id": "rust:condition-wait:0",
14702 "condition_key": "ready",
14703 })]),
14704 CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
14705 );
14706 assert_eq!(
14707 workflow_completion_protocol_version_with_message_streams(
14708 &[json!({"type": "upsert_memo", "entries": {"status": "waiting"}})],
14709 true,
14710 ),
14711 MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION
14712 );
14713 assert_eq!(
14714 workflow_completion_protocol_version_with_message_streams(
14715 &[json!({
14716 "type": "open_condition_wait",
14717 "condition_wait_occurrence_id": "rust:condition-wait:0",
14718 "condition_key": "ready",
14719 })],
14720 true,
14721 ),
14722 CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
14723 );
14724 }
14725
14726 #[test]
14727 fn portable_worker_affinity_manifest_explicitly_refuses_unimplemented_features() {
14728 let manifest = portable_worker_affinity_capability_manifest();
14729
14730 for capability in ["local_activities", "worker_sessions", "sticky_execution"] {
14731 assert_eq!(manifest[capability]["supported"], json!(false));
14732 assert_eq!(
14733 manifest[capability]["minimum_protocol_version"],
14734 json!(PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION)
14735 );
14736 assert!(manifest[capability]["reason"]
14737 .as_str()
14738 .is_some_and(|reason| !reason.is_empty()));
14739 }
14740 }
14741
14742 fn typed_fidelity_probe() -> AvroValue {
14743 AvroValue::Map(BTreeMap::from([
14744 ("bytes".to_string(), AvroValue::Bytes(vec![0, 0xff])),
14745 ("empty".to_string(), AvroValue::Map(BTreeMap::new())),
14746 (
14747 "numeric".to_string(),
14748 AvroValue::Map(BTreeMap::from([
14749 ("0".to_string(), AvroValue::String("zero".to_string())),
14750 ("1".to_string(), AvroValue::String("one".to_string())),
14751 ])),
14752 ),
14753 (
14754 "nested".to_string(),
14755 AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([(
14756 "enabled".to_string(),
14757 AvroValue::Boolean(true),
14758 )]))]),
14759 ),
14760 (
14761 "projection_collisions".to_string(),
14762 AvroValue::Array(projection_collision_probe()),
14763 ),
14764 ]))
14765 }
14766
14767 fn projection_collision_probe() -> Vec<AvroValue> {
14768 vec![
14769 AvroValue::Map(BTreeMap::from([
14770 ("$type".to_string(), AvroValue::String("bytes".to_string())),
14771 (
14772 "base64".to_string(),
14773 AvroValue::String("ordinary user text".to_string()),
14774 ),
14775 ])),
14776 AvroValue::Map(BTreeMap::from([
14777 ("$type".to_string(), AvroValue::String("map".to_string())),
14778 (
14779 "entries".to_string(),
14780 AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([
14781 ("key".to_string(), AvroValue::String("ordinary".to_string())),
14782 (
14783 "value".to_string(),
14784 AvroValue::String("user map".to_string()),
14785 ),
14786 ]))]),
14787 ),
14788 ])),
14789 ]
14790 }
14791
14792 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
14793 struct TypedContract {
14794 nested: TypedNested,
14795 mode: TypedMode,
14796 optional: Option<String>,
14797 absent: Option<String>,
14798 items: Vec<i64>,
14799 labels: BTreeMap<String, String>,
14800 bytes: serde_bytes::ByteBuf,
14801 signed: i64,
14802 finite: f64,
14803 }
14804
14805 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
14806 struct TypedNested {
14807 enabled: bool,
14808 }
14809
14810 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
14811 enum TypedMode {
14812 Detailed { label: String },
14813 }
14814
14815 fn typed_contract() -> TypedContract {
14816 TypedContract {
14817 nested: TypedNested { enabled: true },
14818 mode: TypedMode::Detailed {
14819 label: "compiler-checked".to_string(),
14820 },
14821 optional: Some("present".to_string()),
14822 absent: None,
14823 items: vec![i64::MIN, 0, i64::MAX],
14824 labels: BTreeMap::from([
14825 ("language".to_string(), "rust".to_string()),
14826 ("wire".to_string(), "avro".to_string()),
14827 ]),
14828 bytes: serde_bytes::ByteBuf::from(vec![0, 0xff, 7]),
14829 signed: -9_223_372_036_854_775_000,
14830 finite: 12.5,
14831 }
14832 }
14833
14834 #[derive(Clone, Debug, Default, PartialEq)]
14835 struct ReplayCounterState {
14836 loaded: Option<String>,
14837 count: i64,
14838 finished: bool,
14839 }
14840
14841 fn replay_counter_worker() -> Worker {
14842 let client = Client::new("http://127.0.0.1:8080").expect("client");
14843 let mut worker = Worker::new(client, "rust-workers");
14844 worker.register_replayed_workflow(
14845 "replay-counter",
14846 ReplayCounterState::default,
14847 |ctx, _input, state| async move {
14848 let loaded = ctx.activity("load-counter", json!([])).await?;
14849 state.update(|current| {
14850 current.loaded = loaded.as_str().map(str::to_string);
14851 })?;
14852 for _ in 0..2 {
14853 let signal = ctx.wait_signal("increment").await?;
14854 let amount = signal.first().and_then(Value::as_i64).unwrap_or_default();
14855 state.update(|current| current.count += amount)?;
14856 }
14857 state.update(|current| current.finished = true)?;
14858 state.read(|current| Ok(json!(current.count)))?
14859 },
14860 );
14861 worker.register_replayed_query::<ReplayCounterState, _, _>(
14862 "replay-counter",
14863 "current",
14864 |_ctx, state, _args| async move {
14865 Ok(json!({
14866 "loaded": state.loaded,
14867 "count": state.count,
14868 "finished": state.finished,
14869 }))
14870 },
14871 );
14872 worker.register_replayed_query::<ReplayCounterState, _, _>(
14873 "replay-counter",
14874 "detached-mutation",
14875 |_ctx, state, _args| async move {
14876 let mut detached = (*state).clone();
14877 detached.count = 999;
14878 Ok(json!(detached.count))
14879 },
14880 );
14881 worker.register_replayed_query::<ReplayCounterState, _, _>(
14882 "replay-counter",
14883 "failed-mutation",
14884 |_ctx, state, _args| async move {
14885 let mut detached = (*state).clone();
14886 detached.count = 999;
14887 Err(Error::WorkerLoop("query refused".to_string()))
14888 },
14889 );
14890 worker
14891 }
14892
14893 fn replay_counter_query(
14894 query_name: &str,
14895 history_events: Value,
14896 run_status: &str,
14897 ) -> QueryTask {
14898 let arguments = fixture_envelope(json!([]));
14899 serde_json::from_value(json!({
14900 "query_task_id": format!("query-{query_name}"),
14901 "workflow_type": "replay-counter",
14902 "query_name": query_name,
14903 "payload_codec": DEFAULT_CODEC,
14904 "workflow_arguments": arguments.clone(),
14905 "query_arguments": arguments,
14906 "history_events": history_events,
14907 "run_status": run_status,
14908 }))
14909 .expect("query task")
14910 }
14911
14912 fn workflow_context(history: Vec<HistoryEvent>) -> WorkflowContext {
14913 workflow_context_with_codec(history, DEFAULT_CODEC)
14914 }
14915
14916 fn workflow_context_with_codec(
14917 history: Vec<HistoryEvent>,
14918 payload_codec: &str,
14919 ) -> WorkflowContext {
14920 WorkflowContext {
14921 state: Arc::new(Mutex::new(
14922 WorkflowState::new_with_identity(
14923 history,
14924 None,
14925 None,
14926 "rust-workers".to_string(),
14927 payload_codec.to_string(),
14928 None,
14929 )
14930 .expect("valid workflow history"),
14931 )),
14932 }
14933 }
14934
14935 fn history_event(event_type: &str, payload: Value) -> HistoryEvent {
14936 HistoryEvent {
14937 event_type: event_type.to_string(),
14938 payload,
14939 raw: HashMap::new(),
14940 }
14941 }
14942
14943 fn parallel_path_entry(
14944 kind: &str,
14945 base: u64,
14946 size: usize,
14947 index: usize,
14948 ) -> ParallelGroupMetadata {
14949 parallel_group_entry(base, size, index, kind)
14950 }
14951
14952 fn parallel_history_event(
14953 event_type: &str,
14954 sequence: u64,
14955 identity_field: &str,
14956 identity: &str,
14957 path: Vec<ParallelGroupMetadata>,
14958 result: Option<Value>,
14959 ) -> HistoryEvent {
14960 let mut payload = serde_json::Map::from_iter([
14961 ("sequence".to_string(), json!(sequence)),
14962 (identity_field.to_string(), json!(identity)),
14963 ]);
14964 let inner = path.last().expect("parallel history path");
14965 apply_parallel_group_path(&mut payload, std::slice::from_ref(inner));
14966 payload.insert("parallel_group_path".to_string(), json!(path));
14967 if let Some(result) = result {
14968 let field = if event_type == "ChildRunCompleted" {
14969 "result"
14970 } else {
14971 "result"
14972 };
14973 payload.insert(field.to_string(), fixture_envelope(result));
14974 payload.insert("payload_codec".to_string(), json!(DEFAULT_CODEC));
14975 }
14976 history_event(event_type, Value::Object(payload))
14977 }
14978
14979 fn nested_parallel_operations() -> Vec<ParallelOperation> {
14980 vec![
14981 ParallelOperation::activity("first", json!([])),
14982 ParallelOperation::group(vec![
14983 ParallelOperation::child_workflow(
14984 "second",
14985 ChildWorkflowOptions::new("child-workers"),
14986 json!([]),
14987 ),
14988 ParallelOperation::activity("third", json!([])),
14989 ]),
14990 ]
14991 }
14992
14993 fn nested_parallel_paths() -> [Vec<ParallelGroupMetadata>; 3] {
14994 let outer = [
14995 parallel_path_entry("mixed", 1, 3, 0),
14996 parallel_path_entry("mixed", 1, 3, 1),
14997 parallel_path_entry("mixed", 1, 3, 2),
14998 ];
14999 [
15000 vec![outer[0].clone()],
15001 vec![outer[1].clone(), parallel_path_entry("mixed", 2, 2, 0)],
15002 vec![outer[2].clone(), parallel_path_entry("mixed", 2, 2, 1)],
15003 ]
15004 }
15005
15006 #[test]
15007 fn parallel_schedules_every_nested_mixed_leaf_with_stable_metadata() {
15008 let ctx = workflow_context(Vec::new());
15009 let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15010 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15011
15012 assert!(matches!(
15013 call.as_mut().poll(&mut task_context),
15014 Poll::Pending
15015 ));
15016 let commands = ctx.take_commands().expect("parallel commands");
15017 assert_eq!(
15018 commands
15019 .iter()
15020 .map(|command| command["type"].as_str().unwrap_or_default())
15021 .collect::<Vec<_>>(),
15022 [
15023 "schedule_activity",
15024 "start_child_workflow",
15025 "schedule_activity"
15026 ]
15027 );
15028 let paths = nested_parallel_paths();
15029 for (command, path) in commands.iter().zip(paths) {
15030 assert_eq!(command["parallel_group_path"], json!(path));
15031 assert_eq!(
15032 command["parallel_group_id"],
15033 json!(path.last().expect("inner group").parallel_group_id)
15034 );
15035 }
15036 }
15037
15038 fn completed_nested_parallel_history() -> Vec<HistoryEvent> {
15039 let paths = nested_parallel_paths();
15040 let third = parallel_history_event(
15041 "ActivityCompleted",
15042 3,
15043 "activity_type",
15044 "third",
15045 paths[2].clone(),
15046 Some(json!("three")),
15047 );
15048 vec![
15049 parallel_history_event(
15050 "ActivityCompleted",
15051 1,
15052 "activity_type",
15053 "first",
15054 paths[0].clone(),
15055 Some(json!("one")),
15056 ),
15057 parallel_history_event(
15058 "ChildWorkflowScheduled",
15059 2,
15060 "child_workflow_type",
15061 "second",
15062 paths[1].clone(),
15063 None,
15064 ),
15065 parallel_history_event(
15066 "ChildRunCompleted",
15067 2,
15068 "child_workflow_type",
15069 "second",
15070 paths[1].clone(),
15071 Some(json!("two")),
15072 ),
15073 third.clone(),
15074 third,
15075 ]
15076 }
15077
15078 #[test]
15079 fn parallel_replay_rebuilds_input_order_and_tolerates_duplicate_delivery() {
15080 for _restart_or_completed_replay in 0..2 {
15081 let ctx = workflow_context(completed_nested_parallel_history());
15082 let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15083 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15084 let Poll::Ready(Ok(results)) = call.as_mut().poll(&mut task_context) else {
15085 panic!("completed nested parallel history must replay");
15086 };
15087 assert_eq!(
15088 results,
15089 vec![
15090 ParallelResult::Activity(json!("one")),
15091 ParallelResult::Group(vec![
15092 ParallelResult::ChildWorkflow(ChildWorkflowResult {
15093 parent: WorkflowIdentity {
15094 workflow_id: None,
15095 run_id: None,
15096 },
15097 child: WorkflowIdentity {
15098 workflow_id: None,
15099 run_id: None,
15100 },
15101 child_workflow_type: Some("second".to_string()),
15102 result: json!("two"),
15103 }),
15104 ParallelResult::Activity(json!("three")),
15105 ]),
15106 ]
15107 );
15108 assert!(ctx.take_commands().expect("commands").is_empty());
15109 ctx.ensure_history_consumed().expect("history consumed");
15110 }
15111 }
15112
15113 #[test]
15114 fn parallel_failure_keeps_typed_cause_path_and_late_completions() {
15115 let paths = nested_parallel_paths();
15116 let history = vec![
15117 parallel_history_event(
15118 "ActivityCompleted",
15119 1,
15120 "activity_type",
15121 "first",
15122 paths[0].clone(),
15123 Some(json!("one")),
15124 ),
15125 parallel_history_event(
15126 "ChildWorkflowScheduled",
15127 2,
15128 "child_workflow_type",
15129 "second",
15130 paths[1].clone(),
15131 None,
15132 ),
15133 parallel_history_event(
15134 "ChildRunFailed",
15135 2,
15136 "child_workflow_type",
15137 "second",
15138 paths[1].clone(),
15139 None,
15140 ),
15141 parallel_history_event(
15142 "ActivityCompleted",
15143 3,
15144 "activity_type",
15145 "third",
15146 paths[2].clone(),
15147 Some(json!("late")),
15148 ),
15149 ];
15150 let ctx = workflow_context(history);
15151 let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15152 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15153 let outcome = call.as_mut().poll(&mut task_context);
15154 let Poll::Ready(Err(Error::ParallelFailed(failure))) = outcome else {
15155 panic!("one failed child must return a typed partial failure: {outcome:?}");
15156 };
15157 assert_eq!(failure.member_path, [1, 0]);
15158 assert_eq!(failure.group_id, "parallel-calls:1:3");
15159 assert!(matches!(*failure.cause, Error::ChildWorkflowFailed(_)));
15160 assert_eq!(
15161 failure
15162 .completed
15163 .iter()
15164 .map(|completion| completion.member_path.clone())
15165 .collect::<Vec<_>>(),
15166 [vec![0], vec![1, 1]]
15167 );
15168 }
15169
15170 #[test]
15171 fn pending_parallel_history_restarts_without_rescheduling_any_leaf() {
15172 let paths = nested_parallel_paths();
15173 let history = vec![
15174 parallel_history_event(
15175 "ActivityScheduled",
15176 1,
15177 "activity_type",
15178 "first",
15179 paths[0].clone(),
15180 None,
15181 ),
15182 parallel_history_event(
15183 "ChildWorkflowScheduled",
15184 2,
15185 "child_workflow_type",
15186 "second",
15187 paths[1].clone(),
15188 None,
15189 ),
15190 parallel_history_event(
15191 "ActivityScheduled",
15192 3,
15193 "activity_type",
15194 "third",
15195 paths[2].clone(),
15196 None,
15197 ),
15198 ];
15199 for _restart in 0..2 {
15200 let ctx = workflow_context(history.clone());
15201 let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15202 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15203 let outcome = call.as_mut().poll(&mut task_context);
15204 assert!(matches!(outcome, Poll::Pending), "{outcome:?}");
15205 assert!(ctx.take_commands().expect("commands").is_empty());
15206 }
15207 }
15208
15209 fn selection_path(index: usize, key: &str) -> Vec<ParallelGroupMetadata> {
15210 vec![selection_group_entry(
15211 1,
15212 2,
15213 index,
15214 "activity",
15215 &SelectionMemberMetadata {
15216 key: SelectionKey::Name(key.to_string()),
15217 index,
15218 base_sequence: index as u64 + 1,
15219 size: 1,
15220 kind: "activity".to_string(),
15221 },
15222 )]
15223 }
15224
15225 fn selection_activity_event(
15226 event_type: &str,
15227 index: usize,
15228 key: &str,
15229 result: Option<Value>,
15230 ) -> HistoryEvent {
15231 let sequence = index as u64 + 1;
15232 let mut event = parallel_history_event(
15233 event_type,
15234 sequence,
15235 "activity_type",
15236 &format!("{key}-activity"),
15237 selection_path(index, key),
15238 result,
15239 );
15240 event.payload["activity_execution_id"] = json!(format!("activity-{key}"));
15241 event.raw.insert(
15242 "id".to_string(),
15243 json!(if event_type == "ActivityCompleted" {
15244 format!("event-{key}")
15245 } else {
15246 format!("{event_type}-{key}")
15247 }),
15248 );
15249 event
15250 }
15251
15252 fn selection_winner_marker() -> HistoryEvent {
15253 history_event(
15254 "SelectionResolved",
15255 json!({
15256 "selection_group_id": "select-calls:1:2",
15257 "selection_group_base_sequence": 1,
15258 "selection_group_size": 2,
15259 "member_key": "fast",
15260 "member_index": 1,
15261 "member_base_sequence": 2,
15262 "member_size": 1,
15263 "operation_kind": "activity",
15264 "operation_identity": "activity-fast",
15265 "outcome": "completed",
15266 "resolution_event_id": "event-fast",
15267 "resolution_event_type": "ActivityCompleted",
15268 }),
15269 )
15270 }
15271
15272 fn keyed_activity_selection(ctx: &WorkflowContext) -> SelectCall {
15273 ctx.select_keyed(vec![
15274 (
15275 "slow",
15276 ParallelOperation::activity_with_options(
15277 "slow-activity",
15278 ActivityOptions::new().task_queue("default"),
15279 json!([]),
15280 ),
15281 ),
15282 (
15283 "fast",
15284 ParallelOperation::activity_with_options(
15285 "fast-activity",
15286 ActivityOptions::new().task_queue("default"),
15287 json!([]),
15288 ),
15289 ),
15290 ])
15291 }
15292
15293 fn assert_persisted_selection_replay(history: Vec<HistoryEvent>) {
15294 let ctx = workflow_context(history);
15295 let mut call = Box::pin(keyed_activity_selection(&ctx));
15296 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15297 let selected = match call.as_mut().poll(&mut task_context) {
15298 Poll::Ready(Ok(selected)) => selected,
15299 Poll::Ready(Err(error)) => panic!("persisted selection winner must replay: {error:?}"),
15300 Poll::Pending => panic!("persisted selection winner must replay without pending"),
15301 };
15302 assert_eq!(selected.key, SelectionKey::Name("fast".to_string()));
15303 assert_eq!(
15304 selected.value,
15305 Some(ParallelResult::Activity(json!("winner-value")))
15306 );
15307 let slow = selected
15308 .handle(&SelectionKey::Name("slow".to_string()))
15309 .expect("slow handle")
15310 .clone();
15311 let mut await_slow = Box::pin(slow.await_result());
15312 assert!(matches!(
15313 await_slow.as_mut().poll(&mut task_context),
15314 Poll::Ready(Ok(ParallelResult::Activity(value))) if value == json!("loser-value")
15315 ));
15316 assert!(ctx.take_commands().expect("commands").is_empty());
15317 }
15318
15319 const SELECTION_COLD_REPLAY_HISTORY: &str = "DURABLE_WORKFLOW_SELECTION_COLD_REPLAY_HISTORY";
15320
15321 fn canonical_selection_history() -> Vec<HistoryEvent> {
15322 const FIXTURE: &[u8] =
15323 include_bytes!("../tests/fixtures/durable_selection_runtime_history.json");
15324 assert_eq!(
15325 format!("{:x}", Sha256::digest(FIXTURE)),
15326 "51fd8b9c16e978dcef536a5c727b9fdc0ae724d9afc17d9a7837d219f41ee3ba",
15327 );
15328 let fixture: Value = serde_json::from_slice(FIXTURE).expect("canonical selection fixture");
15329
15330 serde_json::from_value(fixture["history"].clone()).expect("canonical selection history")
15331 }
15332
15333 #[test]
15334 fn selection_fresh_process_entrypoint() {
15335 let Ok(path) = std::env::var(SELECTION_COLD_REPLAY_HISTORY) else {
15336 return;
15337 };
15338 let persisted = fs::read(path).expect("persisted selection history");
15339 assert_eq!(
15340 format!("{:x}", Sha256::digest(&persisted)),
15341 "51fd8b9c16e978dcef536a5c727b9fdc0ae724d9afc17d9a7837d219f41ee3ba",
15342 );
15343 let fixture: Value =
15344 serde_json::from_slice(&persisted).expect("valid persisted selection fixture");
15345 let history: Vec<HistoryEvent> = serde_json::from_value(fixture["history"].clone())
15346 .expect("valid persisted selection history");
15347
15348 assert_persisted_selection_replay(history);
15349 }
15350
15351 #[test]
15352 fn selection_starts_every_member_with_stable_keys_and_group_identity() {
15353 let ctx = workflow_context(Vec::new());
15354 let mut call = Box::pin(keyed_activity_selection(&ctx));
15355 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15356
15357 assert!(matches!(
15358 call.as_mut().poll(&mut task_context),
15359 Poll::Pending
15360 ));
15361 let commands = ctx.take_commands().expect("selection commands");
15362 assert_eq!(commands.len(), 2);
15363 assert_eq!(commands[0]["selection_member_key"], json!("slow"));
15364 assert_eq!(commands[1]["selection_member_key"], json!("fast"));
15365 assert!(commands.iter().all(|command| {
15366 command["parallel_group_id"] == json!("select-calls:1:2")
15367 && command["parallel_group_mode"] == json!("select")
15368 }));
15369 }
15370
15371 #[test]
15372 fn selection_key_domain_rejects_empty_authoring_and_malformed_history() {
15373 let ctx = workflow_context(Vec::new());
15374 let mut invalid = Box::pin(ctx.select_keyed(vec![(
15375 "",
15376 ParallelOperation::activity("invalid", json!([])),
15377 )]));
15378 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15379 assert!(matches!(
15380 invalid.as_mut().poll(&mut task_context),
15381 Poll::Ready(Err(Error::InvalidParallelGroup(ParallelGroupError {
15382 reason: "selection_key_invalid",
15383 ..
15384 })))
15385 ));
15386
15387 for invalid_key in [json!(""), json!(-1)] {
15388 let mut event = selection_activity_event("ActivityScheduled", 0, "slow", None);
15389 event.payload["selection_member_key"] = invalid_key.clone();
15390 event.payload["parallel_group_path"][0]["selection_member_key"] = invalid_key;
15391 assert!(matches!(
15392 WorkflowState::new_with_identity(
15393 vec![event],
15394 None,
15395 None,
15396 "rust-workers".to_string(),
15397 DEFAULT_CODEC.to_string(),
15398 None,
15399 ),
15400 Err(Error::NonDeterministicReplay(_))
15401 ));
15402 }
15403 }
15404
15405 #[test]
15406 fn selection_preserves_valid_named_and_numeric_keys() {
15407 let ctx = workflow_context(Vec::new());
15408 let mut selection = Box::pin(ctx.select_keyed(vec![
15409 (
15410 SelectionKey::Index(0),
15411 ParallelOperation::activity("numeric", json!([])),
15412 ),
15413 (
15414 SelectionKey::Name("named".to_string()),
15415 ParallelOperation::timer(Duration::from_secs(1)),
15416 ),
15417 ]));
15418 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15419
15420 assert!(matches!(
15421 selection.as_mut().poll(&mut task_context),
15422 Poll::Pending
15423 ));
15424 let commands = ctx.take_commands().expect("selection commands");
15425 assert_eq!(commands[0]["selection_member_key"], json!(0));
15426 assert_eq!(commands[1]["selection_member_key"], json!("named"));
15427 }
15428
15429 #[test]
15430 fn selection_replays_persisted_winner_and_loser_can_be_awaited_later() {
15431 let history = canonical_selection_history();
15432 assert_persisted_selection_replay(history.clone());
15433
15434 let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
15435 .join("tests/fixtures/durable_selection_runtime_history.json");
15436 let output =
15437 ProcessCommand::new(std::env::current_exe().expect("current Rust test binary"))
15438 .args([
15439 "--exact",
15440 "tests::selection_fresh_process_entrypoint",
15441 "--nocapture",
15442 ])
15443 .env(SELECTION_COLD_REPLAY_HISTORY, &path)
15444 .output()
15445 .expect("run fresh selection replay process");
15446
15447 assert!(
15448 output.status.success(),
15449 "fresh selection replay failed:\nstdout:\n{}\nstderr:\n{}",
15450 String::from_utf8_lossy(&output.stdout),
15451 String::from_utf8_lossy(&output.stderr),
15452 );
15453 }
15454
15455 #[test]
15456 fn selection_waits_durably_when_terminal_members_precede_the_winner_marker() {
15457 let mut history = canonical_selection_history();
15458 history.retain(|event| event.event_type != "SelectionResolved");
15459 let ctx = workflow_context(history);
15460 let mut selection = Box::pin(keyed_activity_selection(&ctx));
15461 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15462
15463 assert!(matches!(
15464 selection.as_mut().poll(&mut task_context),
15465 Poll::Pending
15466 ));
15467 assert!(ctx.take_commands().expect("commands").is_empty());
15468 assert!(
15469 ctx.matched_recorded_pending()
15470 .expect("selection pending state"),
15471 "terminal member history must keep the workflow durably pending until SelectionResolved commits"
15472 );
15473 }
15474
15475 #[test]
15476 fn selection_terminal_condition_history_waits_durably_for_its_winner_marker() {
15477 for (terminal_event, predicate_satisfied, timeout_seconds) in [
15478 ("ConditionWaitSatisfied", true, None),
15479 ("ConditionWaitTimedOut", false, Some(0)),
15480 ] {
15481 let member = SelectionMemberMetadata {
15482 key: SelectionKey::Name("condition".to_string()),
15483 index: 0,
15484 base_sequence: 1,
15485 size: 1,
15486 kind: "condition".to_string(),
15487 };
15488 let path = vec![selection_group_entry(1, 1, 0, "condition", &member)];
15489 let mut payload = json!({
15490 "sequence": 1,
15491 "condition_wait_id": "condition-1",
15492 "condition_wait_occurrence_id": "rust:condition-wait:0",
15493 "condition_key": "ready",
15494 "condition_definition_fingerprint": "sha256:ready-v1",
15495 "parallel_group_path": path,
15496 });
15497 payload
15498 .as_object_mut()
15499 .expect("condition history payload")
15500 .extend(
15501 serde_json::to_value(&path[0])
15502 .expect("condition selection metadata")
15503 .as_object()
15504 .expect("condition selection metadata object")
15505 .clone(),
15506 );
15507 if let Some(timeout_seconds) = timeout_seconds {
15508 payload["timeout_seconds"] = json!(timeout_seconds);
15509 }
15510 let history = vec![
15511 history_event("ConditionWaitOpened", payload.clone()),
15512 history_event(terminal_event, payload),
15513 ];
15514 let ctx = workflow_context(history);
15515 let mut options = ConditionWaitOptions::new("ready", "sha256:ready-v1");
15516 if timeout_seconds.is_some() {
15517 options = options.timeout(Duration::ZERO);
15518 }
15519 let mut selection = Box::pin(ctx.select_keyed(vec![(
15520 "condition",
15521 ParallelOperation::condition(options, move || Ok(predicate_satisfied)),
15522 )]));
15523 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15524
15525 assert!(matches!(
15526 selection.as_mut().poll(&mut task_context),
15527 Poll::Pending
15528 ));
15529 assert!(ctx.take_commands().expect("commands").is_empty());
15530 assert!(
15531 ctx.matched_recorded_pending()
15532 .expect("condition selection pending state"),
15533 "{terminal_event} must keep the workflow durably pending until SelectionResolved commits"
15534 );
15535 }
15536 }
15537
15538 #[test]
15539 fn selection_immediate_condition_members_open_a_durable_wait() {
15540 for predicate_satisfied in [true, false] {
15541 let ctx = workflow_context(Vec::new());
15542 let mut selection = Box::pin(ctx.select_keyed(vec![(
15543 "condition",
15544 ParallelOperation::condition(
15545 ConditionWaitOptions::new("ready", "sha256:ready-v1").timeout(Duration::ZERO),
15546 move || Ok(predicate_satisfied),
15547 ),
15548 )]));
15549 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15550
15551 assert!(matches!(
15552 selection.as_mut().poll(&mut task_context),
15553 Poll::Pending
15554 ));
15555 let commands = ctx.take_commands().expect("condition selection command");
15556 assert_eq!(commands.len(), 1);
15557 assert_eq!(commands[0]["type"], json!("open_condition_wait"));
15558 assert_eq!(commands[0]["timeout_seconds"], json!(0));
15559 assert_eq!(
15560 commands[0]["parallel_group_path"][0]["parallel_group_mode"],
15561 json!("select")
15562 );
15563 }
15564 }
15565
15566 #[test]
15567 fn selection_loser_cancellation_is_explicit_and_idempotent() {
15568 let history = vec![
15569 selection_activity_event("ActivityScheduled", 0, "slow", None),
15570 selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
15571 selection_winner_marker(),
15572 ];
15573 let ctx = workflow_context(history.clone());
15574 let mut call = Box::pin(keyed_activity_selection(&ctx));
15575 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15576 let Poll::Ready(Ok(selected)) = call.as_mut().poll(&mut task_context) else {
15577 panic!("winner must replay");
15578 };
15579 let slow = selected
15580 .handle(&SelectionKey::Name("slow".to_string()))
15581 .expect("slow handle")
15582 .clone();
15583 let mut cancel = Box::pin(slow.cancel());
15584 assert!(matches!(
15585 cancel.as_mut().poll(&mut task_context),
15586 Poll::Pending
15587 ));
15588 assert!(matches!(
15589 cancel.as_mut().poll(&mut task_context),
15590 Poll::Pending
15591 ));
15592 let commands = ctx.take_commands().expect("cancel command");
15593 assert_eq!(commands.len(), 1);
15594 assert_eq!(commands[0]["type"], json!("cancel_selection_operation"));
15595 assert_eq!(commands[0]["member_key"], json!("slow"));
15596
15597 let mut cancelled_history = history;
15598 cancelled_history.push(history_event(
15599 "SelectionOperationCancelled",
15600 json!({
15601 "selection_group_id": "select-calls:1:2",
15602 "member_key": "slow",
15603 "member_index": 0,
15604 "member_base_sequence": 1,
15605 "member_size": 1,
15606 "operation_kind": "activity",
15607 "operation_identity": "activity-slow",
15608 "cancelled_at": "2026-08-27T00:00:00Z",
15609 }),
15610 ));
15611 let replayed = workflow_context(cancelled_history);
15612 let mut call = Box::pin(keyed_activity_selection(&replayed));
15613 let Poll::Ready(Ok(selected)) = call.as_mut().poll(&mut task_context) else {
15614 panic!("winner must replay after cancellation");
15615 };
15616 let slow = selected
15617 .handle(&SelectionKey::Name("slow".to_string()))
15618 .expect("slow handle")
15619 .clone();
15620 let mut cancel = Box::pin(slow.cancel());
15621 assert!(matches!(
15622 cancel.as_mut().poll(&mut task_context),
15623 Poll::Ready(Ok(()))
15624 ));
15625 assert!(replayed.take_commands().expect("commands").is_empty());
15626 }
15627
15628 #[test]
15629 fn selection_cancellation_marker_is_bound_to_every_authored_handle_field() {
15630 let base_history = vec![
15631 selection_activity_event("ActivityScheduled", 0, "slow", None),
15632 selection_activity_event("ActivityScheduled", 1, "fast", None),
15633 selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
15634 selection_winner_marker(),
15635 ];
15636 for (field, corrupt) in [
15637 ("member_key", json!("fast")),
15638 ("member_index", json!(1)),
15639 ("member_base_sequence", json!(3)),
15640 ("member_size", json!(2)),
15641 ("operation_kind", json!("timer")),
15642 ("operation_identity", json!("forged")),
15643 ] {
15644 let mut cancellation = json!({
15645 "selection_group_id": "select-calls:1:2",
15646 "member_key": "slow",
15647 "member_index": 0,
15648 "member_base_sequence": 1,
15649 "member_size": 1,
15650 "operation_kind": "activity",
15651 "operation_identity": "activity-slow",
15652 });
15653 cancellation[field] = corrupt;
15654 let mut history = base_history.clone();
15655 history.push(history_event("SelectionOperationCancelled", cancellation));
15656 let ctx = workflow_context(history);
15657 let mut selection = Box::pin(keyed_activity_selection(&ctx));
15658 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15659
15660 assert!(matches!(
15661 selection.as_mut().poll(&mut task_context),
15662 Poll::Ready(Err(Error::NonDeterministicReplay(_)))
15663 ));
15664 }
15665 }
15666
15667 #[test]
15668 fn selection_child_identity_prefers_the_durable_run_id() {
15669 let ctx = workflow_context(vec![history_event(
15670 "ChildWorkflowScheduled",
15671 json!({
15672 "sequence": 1,
15673 "child_workflow_type": "child",
15674 "child_workflow_instance_id": "child-instance",
15675 "child_workflow_run_id": "child-run",
15676 }),
15677 )]);
15678 let state = ctx.state.lock().expect("workflow state");
15679
15680 assert_eq!(
15681 selection_operation_identity(&state, "child", 1, 1),
15682 "child-run"
15683 );
15684 }
15685
15686 #[test]
15687 fn selection_activity_identity_requires_canonical_execution_id() {
15688 let slow = selection_activity_event("ActivityScheduled", 0, "slow", None);
15689 let mut fast_open = selection_activity_event("ActivityScheduled", 1, "fast", None);
15690 let mut fast_completed =
15691 selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner")));
15692 for event in [&mut fast_open, &mut fast_completed] {
15693 event
15694 .payload
15695 .as_object_mut()
15696 .expect("activity payload")
15697 .remove("activity_execution_id");
15698 event.payload["activity_id"] = json!("forged-activity-id");
15699 }
15700 let mut marker = selection_winner_marker();
15701 marker.payload["operation_identity"] = json!("forged-activity-id");
15702 let ctx = workflow_context(vec![slow, fast_open, fast_completed, marker]);
15703 let mut selection = Box::pin(keyed_activity_selection(&ctx));
15704 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15705
15706 assert!(matches!(
15707 selection.as_mut().poll(&mut task_context),
15708 Poll::Ready(Err(Error::NonDeterministicReplay(_)))
15709 ));
15710 }
15711
15712 #[test]
15713 fn selection_completion_before_cancellation_remains_awaitable() {
15714 let history = vec![
15715 selection_activity_event("ActivityScheduled", 0, "slow", None),
15716 selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
15717 selection_winner_marker(),
15718 selection_activity_event(
15719 "ActivityCompleted",
15720 0,
15721 "slow",
15722 Some(json!("completed-first")),
15723 ),
15724 ];
15725 let ctx = workflow_context(history);
15726 let mut selection = Box::pin(keyed_activity_selection(&ctx));
15727 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15728 let Poll::Ready(Ok(selected)) = selection.as_mut().poll(&mut task_context) else {
15729 panic!("winner must replay");
15730 };
15731 let slow = selected
15732 .handle(&SelectionKey::Name("slow".to_string()))
15733 .expect("slow handle")
15734 .clone();
15735 let mut cancel = Box::pin(slow.cancel());
15736 assert!(matches!(
15737 cancel.as_mut().poll(&mut task_context),
15738 Poll::Ready(Ok(()))
15739 ));
15740 let mut await_slow = Box::pin(slow.await_result());
15741 assert!(matches!(
15742 await_slow.as_mut().poll(&mut task_context),
15743 Poll::Ready(Ok(ParallelResult::Activity(value))) if value == json!("completed-first")
15744 ));
15745 let commands = ctx.take_commands().expect("commands");
15746 assert!(commands.is_empty());
15747 }
15748
15749 #[test]
15750 fn selection_nested_later_failure_before_cancel_remains_the_awaited_failure() {
15751 let nested_member = SelectionMemberMetadata {
15752 key: SelectionKey::Name("nested".to_string()),
15753 index: 0,
15754 base_sequence: 1,
15755 size: 2,
15756 kind: "group".to_string(),
15757 };
15758 let deadline_member = SelectionMemberMetadata {
15759 key: SelectionKey::Name("deadline".to_string()),
15760 index: 1,
15761 base_sequence: 3,
15762 size: 1,
15763 kind: "timer".to_string(),
15764 };
15765 let nested_paths = [
15766 vec![
15767 selection_group_entry(1, 3, 0, "mixed", &nested_member),
15768 parallel_group_entry(1, 2, 0, "activity"),
15769 ],
15770 vec![
15771 selection_group_entry(1, 3, 1, "mixed", &nested_member),
15772 parallel_group_entry(1, 2, 1, "activity"),
15773 ],
15774 ];
15775 let deadline_path = vec![selection_group_entry(1, 3, 2, "mixed", &deadline_member)];
15776 let mut timer_fired = parallel_history_event(
15777 "TimerFired",
15778 3,
15779 "timer_id",
15780 "timer-3",
15781 deadline_path.clone(),
15782 None,
15783 );
15784 timer_fired.payload["delay_seconds"] = json!(0);
15785 timer_fired
15786 .raw
15787 .insert("id".to_string(), json!("timer-fired"));
15788 let mut timer_scheduled = parallel_history_event(
15789 "TimerScheduled",
15790 3,
15791 "timer_id",
15792 "timer-3",
15793 deadline_path,
15794 None,
15795 );
15796 timer_scheduled.payload["delay_seconds"] = json!(0);
15797 let history = vec![
15798 parallel_history_event(
15799 "ActivityScheduled",
15800 1,
15801 "activity_type",
15802 "nested-first",
15803 nested_paths[0].clone(),
15804 None,
15805 ),
15806 parallel_history_event(
15807 "ActivityScheduled",
15808 2,
15809 "activity_type",
15810 "nested-second",
15811 nested_paths[1].clone(),
15812 None,
15813 ),
15814 timer_scheduled,
15815 timer_fired,
15816 history_event(
15817 "SelectionResolved",
15818 json!({
15819 "selection_group_id": "select-calls:1:3",
15820 "selection_group_base_sequence": 1,
15821 "selection_group_size": 3,
15822 "member_key": "deadline",
15823 "member_index": 1,
15824 "member_base_sequence": 3,
15825 "member_size": 1,
15826 "operation_kind": "timer",
15827 "operation_identity": "timer-3",
15828 "outcome": "completed",
15829 "resolution_event_id": "timer-fired",
15830 "resolution_event_type": "TimerFired",
15831 }),
15832 ),
15833 parallel_history_event(
15834 "ActivityFailed",
15835 2,
15836 "activity_type",
15837 "nested-second",
15838 nested_paths[1].clone(),
15839 None,
15840 ),
15841 ];
15842 let ctx = workflow_context(history);
15843 let mut selection = Box::pin(ctx.select_keyed(vec![
15844 (
15845 "nested",
15846 ParallelOperation::group(vec![
15847 ParallelOperation::activity("nested-first", json!([])),
15848 ParallelOperation::activity("nested-second", json!([])),
15849 ]),
15850 ),
15851 ("deadline", ParallelOperation::timer(Duration::ZERO)),
15852 ]));
15853 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15854 let Poll::Ready(Ok(selected)) = selection.as_mut().poll(&mut task_context) else {
15855 panic!("deadline winner must replay");
15856 };
15857 let nested = selected
15858 .handle(&SelectionKey::Name("nested".to_string()))
15859 .expect("nested handle")
15860 .clone();
15861 let mut cancel = Box::pin(nested.cancel());
15862 assert!(matches!(
15863 cancel.as_mut().poll(&mut task_context),
15864 Poll::Ready(Ok(()))
15865 ));
15866 let mut await_nested = Box::pin(nested.await_result());
15867
15868 assert!(matches!(
15869 await_nested.as_mut().poll(&mut task_context),
15870 Poll::Ready(Err(Error::ActivityFailed(_)))
15871 ));
15872 assert!(ctx.take_commands().expect("commands").is_empty());
15873 }
15874
15875 #[test]
15876 fn selection_supports_child_timer_signal_condition_and_nested_groups() {
15877 let ctx = workflow_context(Vec::new());
15878 let mut call = Box::pin(ctx.select(vec![
15879 ParallelOperation::child_workflow(
15880 "child",
15881 ChildWorkflowOptions::new("children"),
15882 json!([]),
15883 ),
15884 ParallelOperation::timer(Duration::from_secs(30)),
15885 ParallelOperation::signal("approval"),
15886 ParallelOperation::condition(
15887 ConditionWaitOptions::new("ready", "sha256:ready"),
15888 || Ok(false),
15889 ),
15890 ParallelOperation::group(vec![
15891 ParallelOperation::activity("nested-one", json!([])),
15892 ParallelOperation::activity("nested-two", json!([])),
15893 ]),
15894 ]));
15895 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15896 assert!(matches!(
15897 call.as_mut().poll(&mut task_context),
15898 Poll::Pending
15899 ));
15900 let commands = ctx.take_commands().expect("selection commands");
15901 assert_eq!(
15902 commands
15903 .iter()
15904 .map(|command| command["type"].as_str().unwrap_or_default())
15905 .collect::<Vec<_>>(),
15906 [
15907 "start_child_workflow",
15908 "start_timer",
15909 "open_signal_wait",
15910 "open_condition_wait",
15911 "schedule_activity",
15912 "schedule_activity",
15913 ]
15914 );
15915 assert!(commands.iter().all(|command| {
15916 command["parallel_group_path"][0]["parallel_group_mode"] == json!("select")
15917 }));
15918 assert_eq!(
15919 commands[4]["parallel_group_path"].as_array().map(Vec::len),
15920 Some(2)
15921 );
15922 assert_eq!(
15923 commands[4]["parallel_group_path"][0]["selection_member_kind"],
15924 json!("group")
15925 );
15926 assert_eq!(
15927 commands[5]["parallel_group_path"][0]["selection_member_kind"],
15928 json!("group")
15929 );
15930
15931 let one_leaf_ctx = workflow_context(Vec::new());
15932 let mut one_leaf = Box::pin(one_leaf_ctx.select(vec![ParallelOperation::group(vec![
15933 ParallelOperation::activity("nested-only", json!([])),
15934 ])]));
15935 assert!(matches!(
15936 one_leaf.as_mut().poll(&mut task_context),
15937 Poll::Pending
15938 ));
15939 let one_leaf_commands = one_leaf_ctx.take_commands().expect("one-leaf commands");
15940 assert_eq!(one_leaf_commands.len(), 1);
15941 assert_eq!(
15942 one_leaf_commands[0]["parallel_group_path"][0]["selection_member_kind"],
15943 json!("group")
15944 );
15945 assert_eq!(
15946 one_leaf_commands[0]["parallel_group_path"][0]["selection_member_size"],
15947 json!(1)
15948 );
15949 }
15950
15951 async fn trip_saga(ctx: WorkflowContext) -> Result<Value> {
15952 let mut saga = ctx.saga();
15953 let outcome = async {
15954 let flight = ctx.activity("trip.reserve-flight", json!([])).await?;
15955 saga.add_compensation("trip.cancel-flight", json!([flight]))?;
15956 let hotel = ctx.activity("trip.reserve-hotel", json!([])).await?;
15957 saga.add_compensation("trip.cancel-hotel", json!([hotel]))?;
15958 ctx.activity("trip.charge", json!([])).await?;
15959 Ok(json!({"status": "booked"}))
15960 }
15961 .await;
15962 saga.finish(outcome).await
15963 }
15964
15965 fn saga_activity(
15966 event_type: &str,
15967 sequence: u64,
15968 activity_type: &str,
15969 result: Option<Value>,
15970 ) -> HistoryEvent {
15971 let mut payload = json!({
15972 "sequence": sequence,
15973 "activity_type": activity_type,
15974 "message": format!("{activity_type} failed"),
15975 "exception_type": "PlannedFailure",
15976 "non_retryable": true,
15977 });
15978 if let Some(result) = result {
15979 payload["result"] = fixture_envelope(result);
15980 }
15981 history_event(event_type, payload)
15982 }
15983
15984 #[test]
15985 fn saga_replays_reverse_compensation_across_restart_and_duplicate_delivery() {
15986 let completed_hotel_compensation = saga_activity(
15987 "ActivityCompleted",
15988 4,
15989 "trip.cancel-hotel",
15990 Some(Value::Null),
15991 );
15992 let history = vec![
15993 saga_activity(
15994 "ActivityCompleted",
15995 1,
15996 "trip.reserve-flight",
15997 Some(json!("flight-1")),
15998 ),
15999 saga_activity(
16000 "ActivityCompleted",
16001 2,
16002 "trip.reserve-hotel",
16003 Some(json!("hotel-1")),
16004 ),
16005 saga_activity("ActivityFailed", 3, "trip.charge", None),
16006 completed_hotel_compensation.clone(),
16007 completed_hotel_compensation,
16008 ];
16009
16010 for _restart in 0..2 {
16011 let ctx = workflow_context(history.clone());
16012 let mut future = Box::pin(trip_saga(ctx.clone()));
16013 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16014 assert!(matches!(
16015 future.as_mut().poll(&mut task_context),
16016 Poll::Pending
16017 ));
16018 let commands = ctx.take_commands().expect("compensation command");
16019 assert_eq!(commands.len(), 1);
16020 assert_eq!(commands[0]["activity_type"], "trip.cancel-flight");
16021 }
16022 }
16023
16024 #[test]
16025 fn saga_compensation_failure_preserves_both_typed_failures() {
16026 let history = vec![
16027 saga_activity(
16028 "ActivityCompleted",
16029 1,
16030 "trip.reserve-flight",
16031 Some(json!("flight-1")),
16032 ),
16033 saga_activity(
16034 "ActivityCompleted",
16035 2,
16036 "trip.reserve-hotel",
16037 Some(json!("hotel-1")),
16038 ),
16039 saga_activity("ActivityFailed", 3, "trip.charge", None),
16040 saga_activity("ActivityFailed", 4, "trip.cancel-hotel", None),
16041 ];
16042 let ctx = workflow_context(history);
16043 let mut future = Box::pin(trip_saga(ctx));
16044 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16045 let Poll::Ready(Err(Error::SagaCompensationFailed(failure))) =
16046 future.as_mut().poll(&mut task_context)
16047 else {
16048 panic!("compensation failure must remain structured");
16049 };
16050 assert!(matches!(
16051 *failure.initiating_failure,
16052 Error::ActivityFailed(_)
16053 ));
16054 assert!(matches!(
16055 *failure.compensation_failure,
16056 Error::ActivityFailed(_)
16057 ));
16058 assert_eq!(failure.compensation_activity_type, "trip.cancel-hotel");
16059 assert_eq!(failure.compensation_registration_order, 2);
16060 }
16061
16062 #[test]
16063 fn saga_compensates_cooperative_cancellation() {
16064 let ctx = workflow_context(vec![saga_activity(
16065 "ActivityCompleted",
16066 1,
16067 "trip.reserve-flight",
16068 Some(json!("flight-1")),
16069 )]);
16070 ctx.state.lock().expect("state").cancel_requested = true;
16071 let run = {
16072 let ctx = ctx.clone();
16073 async move {
16074 let mut saga = ctx.saga();
16075 let outcome = async {
16076 let flight = ctx.activity("trip.reserve-flight", json!([])).await?;
16077 saga.add_compensation("trip.cancel-flight", json!([flight]))?;
16078 ctx.throw_if_cancellation_requested()?;
16079 Ok(json!("unexpected"))
16080 }
16081 .await;
16082 saga.finish(outcome).await
16083 }
16084 };
16085 let mut future = Box::pin(run);
16086 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16087 assert!(matches!(
16088 future.as_mut().poll(&mut task_context),
16089 Poll::Pending
16090 ));
16091 let commands = ctx.take_commands().expect("cancellation compensation");
16092 assert_eq!(commands[0]["activity_type"], "trip.cancel-flight");
16093 }
16094
16095 fn workflow_task(
16096 workflow_type: &str,
16097 history_events: Vec<HistoryEvent>,
16098 payload_codec: &str,
16099 ) -> WorkflowTask {
16100 WorkflowTask {
16101 task_id: format!("wft-{workflow_type}"),
16102 workflow_command_id: None,
16103 workflow_id: Some(format!("wf-{workflow_type}")),
16104 run_id: Some(format!("run-{workflow_type}")),
16105 workflow_type: workflow_type.to_string(),
16106 cancel_requested: false,
16107 payload_codec: payload_codec.to_string(),
16108 arguments: Some(
16109 encode_value_envelope(&json!([]), payload_codec).expect("workflow arguments"),
16110 ),
16111 total_history_events: Some(history_events.len() as u64),
16112 history_size_bytes: None,
16113 continue_as_new_recommended: None,
16114 history_budget_pressure: None,
16115 history_events,
16116 next_history_page_token: None,
16117 workflow_task_attempt: 1,
16118 workflow_signal_id: None,
16119 signal_name: None,
16120 signal_arguments: None,
16121 workflow_update_id: None,
16122 update_name: None,
16123 lease_owner: Some("rust-worker".to_string()),
16124 }
16125 }
16126
16127 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
16128 struct SideEffectProbe {
16129 request_id: String,
16130 attempt: u32,
16131 }
16132
16133 #[test]
16134 fn typed_side_effect_runs_callback_once_and_replay_skips_it() {
16135 let calls = AtomicUsize::new(0);
16136 let ctx = workflow_context(Vec::new());
16137 let value = ctx
16138 .side_effect(|| {
16139 calls.fetch_add(1, Ordering::SeqCst);
16140 SideEffectProbe {
16141 request_id: "request-42".to_string(),
16142 attempt: 3,
16143 }
16144 })
16145 .expect("first side effect");
16146 assert_eq!(value.attempt, 3);
16147 assert_eq!(calls.load(Ordering::SeqCst), 1);
16148 let commands = ctx.take_commands().expect("commands");
16149 assert_eq!(commands.len(), 1);
16150 assert_eq!(commands[0]["type"], "record_side_effect");
16151 assert_eq!(
16152 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
16153 serde_json::to_value(&value).expect("value")
16154 );
16155
16156 let replay = workflow_context(vec![history_event(
16157 "SideEffectRecorded",
16158 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16159 )]);
16160 let replayed: SideEffectProbe = replay
16161 .side_effect(|| {
16162 calls.fetch_add(1, Ordering::SeqCst);
16163 panic!("committed side-effect callbacks must not run during replay")
16164 })
16165 .expect("replayed side effect");
16166 assert_eq!(replayed, value);
16167 assert_eq!(calls.load(Ordering::SeqCst), 1);
16168 assert!(replay.take_commands().expect("commands").is_empty());
16169 replay.ensure_history_consumed().expect("history consumed");
16170 }
16171
16172 #[test]
16173 fn side_effect_uses_avro_envelope_and_uuid_is_replay_stable() {
16174 let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
16175 let value = ctx
16176 .side_effect(|| SideEffectProbe {
16177 request_id: "avro-request".to_string(),
16178 attempt: 1,
16179 })
16180 .expect("Avro side effect");
16181 let uuid = ctx.uuid_v4().expect("deterministic UUID");
16182 let commands = ctx.take_commands().expect("commands");
16183 assert_eq!(commands.len(), 2);
16184 assert_eq!(commands[0]["result"]["codec"], DEFAULT_CODEC);
16185 assert_eq!(commands[1]["result"]["codec"], DEFAULT_CODEC);
16186 assert_eq!(
16187 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
16188 serde_json::to_value(&value).expect("value")
16189 );
16190
16191 let replay = workflow_context_with_codec(
16192 vec![
16193 history_event(
16194 "SideEffectRecorded",
16195 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16196 ),
16197 history_event(
16198 "SideEffectRecorded",
16199 json!({"sequence": 2, "result": commands[1]["result"].clone()}),
16200 ),
16201 ],
16202 DEFAULT_CODEC,
16203 );
16204 let replayed: SideEffectProbe = replay
16205 .side_effect(|| panic!("Avro callback must not run"))
16206 .expect("replayed Avro value");
16207 let replayed_uuid = replay.uuid_v4().expect("replayed UUID");
16208 assert_eq!(replayed, value);
16209 assert_eq!(replayed_uuid, uuid);
16210 assert!(replay.take_commands().expect("commands").is_empty());
16211 }
16212
16213 #[test]
16214 fn typed_side_effect_replay_preserves_bytes_and_maps() {
16215 let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
16216 let value = ctx
16217 .side_effect_avro_value(typed_fidelity_probe)
16218 .expect("typed side effect");
16219 let commands = ctx.take_commands().expect("side-effect command");
16220 assert_eq!(
16221 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
16222 .expect("recorded side effect"),
16223 value
16224 );
16225
16226 let replay = workflow_context_with_codec(
16227 vec![history_event(
16228 "SideEffectRecorded",
16229 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16230 )],
16231 DEFAULT_CODEC,
16232 );
16233 assert_eq!(
16234 replay
16235 .side_effect_avro_value(|| panic!("replay must not invoke callback"))
16236 .expect("replayed typed side effect"),
16237 value
16238 );
16239 }
16240
16241 #[test]
16242 fn ordered_side_effects_share_the_durable_command_stream() {
16243 let first = encode_value_envelope(&json!("first"), DEFAULT_CODEC).expect("first");
16244 let second = encode_value_envelope(&json!(29), DEFAULT_CODEC).expect("second");
16245 let ctx = workflow_context(vec![
16246 history_event(
16247 "SideEffectRecorded",
16248 json!({"sequence": 1, "result": first}),
16249 ),
16250 history_event(
16251 "SideEffectRecorded",
16252 json!({"sequence": 2, "result": second}),
16253 ),
16254 ]);
16255 let first: String = ctx
16256 .side_effect(|| panic!("first callback must not run"))
16257 .expect("first replay");
16258 let second: i32 = ctx
16259 .side_effect(|| panic!("second callback must not run"))
16260 .expect("second replay");
16261 assert_eq!(first, "first");
16262 assert_eq!(second, 29);
16263 ctx.ensure_history_consumed().expect("ordered history");
16264
16265 let reordered = workflow_context(vec![history_event(
16266 "VersionMarkerRecorded",
16267 json!({
16268 "sequence": 1,
16269 "change_id": "before-side-effect",
16270 "version": 1,
16271 "min_supported": 1,
16272 "max_supported": 1,
16273 }),
16274 )]);
16275 let error = reordered
16276 .side_effect(|| "new".to_string())
16277 .expect_err("command reordering must fail");
16278 assert!(matches!(
16279 error,
16280 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16281 if reason == "recorded_command_mismatch"
16282 ));
16283 }
16284
16285 #[test]
16286 fn version_markers_replay_across_upgrades_and_do_not_duplicate() {
16287 let ctx = workflow_context(Vec::new());
16288 assert_eq!(ctx.get_version("checkout-v2", 1, 2).expect("version"), 2);
16289 assert_eq!(ctx.get_version("checkout-v2", 1, 3).expect("cached"), 2);
16290 assert!(ctx.patched("new-search").expect("patch"));
16291 ctx.deprecate_patch("new-search").expect("deprecate patch");
16292 let commands = ctx.take_commands().expect("commands");
16293 assert_eq!(commands.len(), 2);
16294 assert_eq!(commands[0]["type"], "record_version_marker");
16295 assert_eq!(commands[0]["version"], 2);
16296 assert_eq!(commands[1]["change_id"], "new-search");
16297
16298 let replay = workflow_context(vec![history_event(
16299 "VersionMarkerRecorded",
16300 json!({
16301 "sequence": 1,
16302 "change_id": "checkout-v2",
16303 "version": 2,
16304 "min_supported": 1,
16305 "max_supported": 2,
16306 }),
16307 )]);
16308 assert_eq!(replay.get_version("checkout-v2", 1, 4).expect("upgrade"), 2);
16309 assert_eq!(replay.get_version("checkout-v2", 2, 5).expect("repeat"), 2);
16310 assert!(replay.take_commands().expect("commands").is_empty());
16311 replay.ensure_history_consumed().expect("history consumed");
16312 }
16313
16314 #[test]
16315 fn version_markers_reject_incompatible_or_malformed_history() {
16316 let incompatible = workflow_context(vec![history_event(
16317 "VersionMarkerRecorded",
16318 json!({
16319 "sequence": 1,
16320 "change_id": "checkout-v2",
16321 "version": 1,
16322 "min_supported": 1,
16323 "max_supported": 2,
16324 }),
16325 )]);
16326 let error = incompatible
16327 .get_version("checkout-v2", 2, 3)
16328 .expect_err("old version is unsupported");
16329 assert!(matches!(
16330 error,
16331 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16332 if reason == "version_marker_incompatible_range"
16333 ));
16334
16335 for (history, reason) in [
16336 (
16337 vec![history_event("SideEffectRecorded", json!({"sequence": 1}))],
16338 "side_effect_result_missing",
16339 ),
16340 (
16341 vec![history_event(
16342 "SideEffectRecorded",
16343 json!({
16344 "sequence": 1,
16345 "result": {"codec": "avro", "blob": "not-base64"},
16346 }),
16347 )],
16348 "side_effect_payload_incompatible",
16349 ),
16350 (
16351 vec![history_event(
16352 "SideEffectRecorded",
16353 json!({"sequence": 1, "result": {"unwrapped": true}}),
16354 )],
16355 "side_effect_payload_malformed",
16356 ),
16357 (
16358 vec![history_event(
16359 "VersionMarkerRecorded",
16360 json!({
16361 "sequence": 1,
16362 "change_id": "change",
16363 "version": 1,
16364 "min_supported": 2,
16365 "max_supported": 1,
16366 }),
16367 )],
16368 "version_marker_history_range_invalid",
16369 ),
16370 ] {
16371 let error = WorkflowState::new(
16372 history,
16373 "rust-workers".to_string(),
16374 DEFAULT_CODEC.to_string(),
16375 None,
16376 )
16377 .expect_err("malformed history must fail");
16378 assert!(matches!(
16379 error,
16380 Error::NonDeterministicReplay(ReplayFailure { reason: actual, .. })
16381 if actual == reason
16382 ));
16383 }
16384 }
16385
16386 #[test]
16387 fn typed_search_attributes_replay_value_and_type_identity_after_restart() {
16388 let history = vec![history_event(
16389 "SearchAttributesUpserted",
16390 json!({
16391 "sequence": 1,
16392 "attributes": {"customer_tier": "gold"},
16393 "attribute_types": {"customer_tier": "keyword"},
16394 "merged": {"customer_tier": "gold"}
16395 }),
16396 )];
16397
16398 let matching = workflow_context(history.clone());
16399 matching
16400 .upsert_search_attributes(
16401 SearchAttributeUpdate::new()
16402 .keyword("customer_tier", "gold")
16403 .expect("keyword update"),
16404 )
16405 .expect("matching typed update must replay");
16406 matching
16407 .ensure_history_consumed()
16408 .expect("history consumed");
16409
16410 let changed_type = workflow_context(history.clone());
16411 let error = changed_type
16412 .upsert_search_attributes(
16413 SearchAttributeUpdate::new()
16414 .string("customer_tier", "gold")
16415 .expect("string update"),
16416 )
16417 .expect_err("same JSON value with a different declaration must be nondeterministic");
16418 let Error::NonDeterministicReplay(failure) = error else {
16419 panic!("typed identity drift must be a replay failure");
16420 };
16421 assert_eq!(failure.reason, "search_attribute_type_mismatch");
16422 assert_eq!(failure.sequence, Some(1));
16423
16424 let changed_value = workflow_context(history);
16425 let error = changed_value
16426 .upsert_search_attributes(
16427 SearchAttributeUpdate::new()
16428 .keyword("customer_tier", "platinum")
16429 .expect("keyword update"),
16430 )
16431 .expect_err("changed values must be nondeterministic");
16432 let Error::NonDeterministicReplay(failure) = error else {
16433 panic!("value drift must be a replay failure");
16434 };
16435 assert_eq!(failure.reason, "search_attribute_value_mismatch");
16436 }
16437
16438 #[test]
16439 fn legacy_search_attribute_history_keeps_type_identity_unknown() {
16440 let history = vec![history_event(
16441 "SearchAttributesUpserted",
16442 json!({
16443 "sequence": 1,
16444 "attributes": {"customer_tier": "gold"},
16445 "merged": {"customer_tier": "gold"}
16446 }),
16447 )];
16448
16449 for update in [
16450 SearchAttributeUpdate::new()
16451 .keyword("customer_tier", "gold")
16452 .expect("keyword update"),
16453 SearchAttributeUpdate::new()
16454 .string("customer_tier", "gold")
16455 .expect("string update"),
16456 ] {
16457 let restarted = workflow_context(history.clone());
16458 restarted
16459 .upsert_search_attributes(update)
16460 .expect("legacy history constrains values but has unknown type identity");
16461 restarted
16462 .ensure_history_consumed()
16463 .expect("history consumed");
16464 }
16465 }
16466
16467 #[test]
16468 fn search_attribute_command_emits_canonical_types() {
16469 let ctx = workflow_context(Vec::new());
16470 ctx.upsert_search_attributes(
16471 SearchAttributeUpdate::new()
16472 .keyword("customer_tier", "gold")
16473 .expect("keyword update")
16474 .int("attempts", 3)
16475 .expect("int update")
16476 .delete("obsolete")
16477 .expect("delete update"),
16478 )
16479 .expect("valid search attributes");
16480
16481 assert_eq!(
16482 ctx.take_commands().expect("commands"),
16483 vec![json!({
16484 "type": "upsert_search_attributes",
16485 "attributes": {
16486 "attempts": 3,
16487 "customer_tier": "gold",
16488 "obsolete": null
16489 },
16490 "attribute_types": {
16491 "attempts": "int",
16492 "customer_tier": "keyword"
16493 }
16494 })]
16495 );
16496 }
16497
16498 #[test]
16499 fn duplicate_side_effects_and_version_markers_are_rejected() {
16500 let duplicate_side_effect = WorkflowState::new(
16501 vec![
16502 history_event(
16503 "SideEffectRecorded",
16504 json!({"sequence": 1, "result": fixture_envelope(json!(1))}),
16505 ),
16506 history_event(
16507 "SideEffectRecorded",
16508 json!({"sequence": 1, "result": fixture_envelope(json!(2))}),
16509 ),
16510 ],
16511 "rust-workers".to_string(),
16512 DEFAULT_CODEC.to_string(),
16513 None,
16514 )
16515 .expect_err("duplicate side effect");
16516 assert!(matches!(
16517 duplicate_side_effect,
16518 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16519 if reason == "duplicate_side_effect_record"
16520 ));
16521
16522 let marker = |sequence| {
16523 history_event(
16524 "VersionMarkerRecorded",
16525 json!({
16526 "sequence": sequence,
16527 "change_id": "same-change",
16528 "version": 1,
16529 "min_supported": 1,
16530 "max_supported": 1,
16531 }),
16532 )
16533 };
16534 let duplicate_marker = WorkflowState::new(
16535 vec![marker(1), marker(3)],
16536 "rust-workers".to_string(),
16537 DEFAULT_CODEC.to_string(),
16538 None,
16539 )
16540 .expect_err("duplicate marker");
16541 assert!(matches!(
16542 duplicate_marker,
16543 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16544 if reason == "duplicate_version_marker"
16545 ));
16546 }
16547
16548 #[test]
16549 fn workflow_stream_authoring_derives_identity_and_replay_skips_duplicate_append() {
16550 let mut state = WorkflowState::new(
16551 Vec::new(),
16552 "rust-workers".to_string(),
16553 DEFAULT_CODEC.to_string(),
16554 None,
16555 )
16556 .expect("workflow state");
16557 state.workflow_command_identity = "command-7".to_string();
16558 let context = WorkflowContext {
16559 state: Arc::new(Mutex::new(state)),
16560 };
16561 let item =
16562 WorkflowStreamAppendItem::from_reference("s3://bucket/item.avro").item_type("receipt");
16563
16564 context
16565 .append_workflow_stream("output", &[item], Some(10))
16566 .expect("append command");
16567 context
16568 .error_workflow_stream("output", "producer failed", None)
16569 .expect("error command");
16570 let commands = context.take_commands().expect("commands");
16571
16572 assert_eq!(commands[0]["type"], "record_side_effect");
16573 assert_eq!(
16574 commands[0]["workflow_stream"]["command_identity"],
16575 "command-7"
16576 );
16577 assert_eq!(commands[0]["workflow_stream"]["command_ordinal"], 0);
16578 assert_eq!(
16579 commands[0]["workflow_stream"]["items"][0]["idempotency_key"],
16580 "dw-stream:command-7:0:0"
16581 );
16582 assert_eq!(commands[1]["workflow_stream"]["operation"], "error");
16583
16584 let recorded = history_event(
16585 "SideEffectRecorded",
16586 json!({"sequence": 1, "result": fixture_envelope(Value::Null)}),
16587 );
16588 let mut replay_state = WorkflowState::new(
16589 vec![recorded],
16590 "rust-workers".to_string(),
16591 DEFAULT_CODEC.to_string(),
16592 None,
16593 )
16594 .expect("replay state");
16595 replay_state.workflow_command_identity = "command-7".to_string();
16596 let replay_context = WorkflowContext {
16597 state: Arc::new(Mutex::new(replay_state)),
16598 };
16599 replay_context
16600 .append_workflow_stream(
16601 "output",
16602 &[WorkflowStreamAppendItem::from_reference(
16603 "s3://bucket/item.avro",
16604 )],
16605 Some(10),
16606 )
16607 .expect("replayed append");
16608 assert!(replay_context
16609 .take_commands()
16610 .expect("replayed commands")
16611 .is_empty());
16612 }
16613
16614 #[test]
16615 fn workflow_stream_authoring_requires_server_durable_command_identity() {
16616 let context = workflow_context(Vec::new());
16617 let error = context
16618 .append_workflow_stream(
16619 "output",
16620 &[WorkflowStreamAppendItem::from_reference(
16621 "s3://bucket/item.avro",
16622 )],
16623 None,
16624 )
16625 .expect_err("stream append without durable command identity must fail closed");
16626
16627 assert!(matches!(error, Error::MissingWorkflowCommandIdentity));
16628 assert!(context.take_commands().expect("commands").is_empty());
16629 }
16630
16631 #[test]
16632 fn cold_worker_replay_does_not_repeat_committed_side_effects_or_markers() {
16633 fn worker(calls: Arc<AtomicUsize>) -> Worker {
16634 let client = Client::new("http://127.0.0.1:8080").expect("client");
16635 let mut worker = Worker::new(client, "rust-workers");
16636 worker.register_workflow("rust.side-effect-version", move |ctx, _input| {
16637 let calls = Arc::clone(&calls);
16638 async move {
16639 let captured = ctx.side_effect(|| {
16640 calls.fetch_add(1, Ordering::SeqCst);
16641 "captured-once".to_string()
16642 })?;
16643 let version = ctx.get_version("cold-restart", 1, 2)?;
16644 Ok(json!({"captured": captured, "version": version}))
16645 }
16646 });
16647 worker
16648 }
16649
16650 fn task(history_events: Vec<HistoryEvent>) -> WorkflowTask {
16651 WorkflowTask {
16652 task_id: "wft-side-effect-version".to_string(),
16653 workflow_command_id: None,
16654 workflow_id: Some("wf-side-effect-version".to_string()),
16655 run_id: Some("run-side-effect-version".to_string()),
16656 workflow_type: "rust.side-effect-version".to_string(),
16657 cancel_requested: false,
16658 payload_codec: DEFAULT_CODEC.to_string(),
16659 arguments: Some(
16660 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("arguments"),
16661 ),
16662 history_events,
16663 total_history_events: None,
16664 history_size_bytes: None,
16665 continue_as_new_recommended: None,
16666 history_budget_pressure: None,
16667 next_history_page_token: None,
16668 workflow_task_attempt: 1,
16669 workflow_signal_id: None,
16670 signal_name: None,
16671 signal_arguments: None,
16672 workflow_update_id: None,
16673 update_name: None,
16674 lease_owner: Some("rust-worker".to_string()),
16675 }
16676 }
16677
16678 let calls = Arc::new(AtomicUsize::new(0));
16679 let initial = worker(Arc::clone(&calls))
16680 .execute_workflow_task(task(Vec::new()))
16681 .expect("initial execution");
16682 assert_eq!(
16683 initial
16684 .iter()
16685 .map(|command| &command["type"])
16686 .collect::<Vec<_>>(),
16687 vec![
16688 "record_side_effect",
16689 "record_version_marker",
16690 "complete_workflow"
16691 ]
16692 );
16693 assert_eq!(calls.load(Ordering::SeqCst), 1);
16694
16695 let restarted = worker(Arc::clone(&calls));
16696 let replayed = restarted
16697 .execute_workflow_task(task(vec![
16698 history_event(
16699 "SideEffectRecorded",
16700 json!({"sequence": 1, "result": initial[0]["result"].clone()}),
16701 ),
16702 history_event(
16703 "VersionMarkerRecorded",
16704 json!({
16705 "sequence": 2,
16706 "change_id": "cold-restart",
16707 "version": 2,
16708 "min_supported": 1,
16709 "max_supported": 2,
16710 }),
16711 ),
16712 ]))
16713 .expect("cold replay");
16714 assert_eq!(replayed.len(), 1);
16715 assert_eq!(replayed[0]["type"], "complete_workflow");
16716 assert_eq!(calls.load(Ordering::SeqCst), 1);
16717 }
16718
16719 #[test]
16720 fn side_effect_replay_rejects_changed_rust_value_type() {
16721 let result = encode_value_envelope(&json!({"value": 42}), DEFAULT_CODEC).expect("result");
16722 let ctx = workflow_context(vec![history_event(
16723 "SideEffectRecorded",
16724 json!({"sequence": 1, "result": result}),
16725 )]);
16726 let error = ctx
16727 .side_effect::<Vec<String>, _>(|| panic!("callback must not run"))
16728 .expect_err("changed type must fail replay");
16729 assert!(matches!(
16730 error,
16731 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16732 if reason == "side_effect_type_mismatch"
16733 ));
16734 }
16735
16736 fn completed_retry_activity_history() -> Vec<HistoryEvent> {
16737 vec![
16738 history_event(
16739 "ActivityScheduled",
16740 json!({
16741 "sequence": 1,
16742 "activity_type": "flaky",
16743 "activity_execution_id": "act-1",
16744 "activity": {
16745 "id": "act-1",
16746 "sequence": 1,
16747 "type": "flaky",
16748 "queue": "critical-activities",
16749 "execution_mode": null,
16750 "retry_policy": {
16751 "snapshot_version": 1,
16752 "max_attempts": 3,
16753 "backoff_seconds": [2, 4],
16754 "start_to_close_timeout": 30,
16755 "schedule_to_start_timeout": 5,
16756 "schedule_to_close_timeout": 90,
16757 "heartbeat_timeout": 10,
16758 "non_retryable_error_types": ["PermanentError"]
16759 }
16760 }
16761 }),
16762 ),
16763 history_event(
16764 "ActivityStarted",
16765 json!({
16766 "sequence": 1,
16767 "activity_type": "flaky",
16768 "activity_execution_id": "act-1",
16769 "activity_attempt_id": "attempt-1",
16770 "attempt_number": 1
16771 }),
16772 ),
16773 history_event(
16774 "ActivityRetryScheduled",
16775 json!({
16776 "sequence": 1,
16777 "activity_type": "flaky",
16778 "activity_execution_id": "act-1",
16779 "activity_attempt_id": "attempt-1",
16780 "attempt_number": 1,
16781 "retry_after_attempt": 1,
16782 "retry_backoff_seconds": 2,
16783 "failure_category": "activity",
16784 "exception_type": "TransientError"
16785 }),
16786 ),
16787 history_event(
16788 "ActivityStarted",
16789 json!({
16790 "sequence": 1,
16791 "activity_type": "flaky",
16792 "activity_execution_id": "act-1",
16793 "activity_attempt_id": "attempt-2",
16794 "attempt_number": 2
16795 }),
16796 ),
16797 history_event(
16798 "ActivityCompleted",
16799 json!({
16800 "sequence": 1,
16801 "activity_type": "flaky",
16802 "activity_execution_id": "act-1",
16803 "activity_attempt_id": "attempt-2",
16804 "attempt_number": 2,
16805 "payload_codec": DEFAULT_CODEC,
16806 "result": fixture_envelope(json!({"status":"recovered"}))
16807 }),
16808 ),
16809 ]
16810 }
16811
16812 fn retry_activity_options() -> ActivityOptions {
16813 ActivityOptions::new()
16814 .task_queue("critical-activities")
16815 .retry_policy(
16816 ActivityRetryPolicy::new(3)
16817 .backoff_intervals([Duration::from_secs(2), Duration::from_secs(4)])
16818 .non_retryable_error_type("PermanentError"),
16819 )
16820 .start_to_close_timeout(Duration::from_secs(30))
16821 .schedule_to_start_timeout(Duration::from_secs(5))
16822 .schedule_to_close_timeout(Duration::from_secs(90))
16823 .heartbeat_timeout(Duration::from_secs(10))
16824 }
16825
16826 #[test]
16827 fn fixed_avro_value_round_trips_json_values() {
16828 let value = json!({"greeting": "hello", "count": 3, "ok": true});
16829 let envelope = PayloadEnvelope::avro(&value).expect("encode");
16830 assert_eq!(envelope.codec, DEFAULT_CODEC);
16831 assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
16832 }
16833
16834 #[tokio::test]
16835 async fn typed_handler_adapters_round_trip_serde_contracts_on_the_fixed_wire() {
16836 let client = Client::new("http://127.0.0.1:8080").expect("client");
16837 let mut worker = Worker::new(client, "rust-workers");
16838 worker.register_typed_workflow(
16839 "typed.contract.workflow",
16840 |_ctx, input: TypedContract| async move { Ok(input) },
16841 );
16842 worker.register_typed_activity(
16843 "typed.contract.activity",
16844 |_ctx, input: TypedContract| async move { Ok(input) },
16845 );
16846
16847 let expected = typed_contract();
16848 let arguments = AvroValue::Array(vec![
16849 AvroValue::from_serialize(&expected).expect("typed request")
16850 ]);
16851 let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("arguments");
16852 let mut workflow = workflow_task("typed.contract.workflow", Vec::new(), DEFAULT_CODEC);
16853 workflow.arguments = Some(envelope.clone());
16854 let commands = worker
16855 .execute_workflow_task(workflow)
16856 .expect("typed workflow task");
16857 let workflow_result: TypedContract =
16858 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
16859 .expect("workflow result envelope")
16860 .deserialize()
16861 .expect("workflow result type");
16862 assert_eq!(workflow_result, expected);
16863
16864 let activity = ActivityTask {
16865 task_id: "typed-contract-activity".to_string(),
16866 activity_attempt_id: Some("typed-contract-attempt".to_string()),
16867 attempt_id: None,
16868 activity_type: "typed.contract.activity".to_string(),
16869 payload_codec: DEFAULT_CODEC.to_string(),
16870 arguments: Some(envelope),
16871 attempt_number: 1,
16872 lease_owner: Some("rust-worker".to_string()),
16873 };
16874 let activity_result: TypedContract = worker
16875 .execute_activity_task(activity)
16876 .await
16877 .expect("typed activity task")
16878 .deserialize()
16879 .expect("activity result type");
16880 assert_eq!(activity_result, expected);
16881 }
16882
16883 #[tokio::test]
16884 async fn typed_handler_errors_include_handler_name_direction_and_rust_type() {
16885 let client = Client::new("http://127.0.0.1:8080").expect("client");
16886 let mut worker = Worker::new(client, "rust-workers");
16887 worker.register_typed_workflow(
16888 "typed.shape.workflow",
16889 |_ctx, input: TypedContract| async move { Ok(input) },
16890 );
16891 worker.register_typed_activity("typed.unsupported.activity", |_ctx, (): ()| async move {
16892 Ok(f64::NAN)
16893 });
16894
16895 let mut workflow = workflow_task("typed.shape.workflow", Vec::new(), DEFAULT_CODEC);
16896 workflow.arguments = Some(
16897 encode_typed_envelope(
16898 &AvroValue::Array(vec![
16899 AvroValue::String("first".to_string()),
16900 AvroValue::String("second".to_string()),
16901 ]),
16902 DEFAULT_CODEC,
16903 )
16904 .expect("malformed typed arguments"),
16905 );
16906 let commands = worker
16907 .execute_workflow_task(workflow)
16908 .expect("shape mismatch becomes a workflow failure");
16909 let message = commands[0]["message"].as_str().expect("failure message");
16910 assert!(message.contains("workflow handler \"typed.shape.workflow\" input type"));
16911 assert!(message.contains(type_name::<TypedContract>()));
16912 assert!(message.contains("task carried 2 arguments"));
16913
16914 let activity = ActivityTask {
16915 task_id: "typed-unsupported-activity".to_string(),
16916 activity_attempt_id: Some("typed-unsupported-attempt".to_string()),
16917 attempt_id: None,
16918 activity_type: "typed.unsupported.activity".to_string(),
16919 payload_codec: DEFAULT_CODEC.to_string(),
16920 arguments: Some(
16921 encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
16922 .expect("unit arguments"),
16923 ),
16924 attempt_number: 1,
16925 lease_owner: Some("rust-worker".to_string()),
16926 };
16927 let Error::HandlerType {
16928 handler_kind,
16929 handler_name,
16930 value_kind,
16931 rust_type,
16932 message,
16933 } = worker
16934 .execute_activity_task(activity)
16935 .await
16936 .expect_err("non-finite handler output must fail")
16937 else {
16938 panic!("expected contextual handler type failure");
16939 };
16940 assert_eq!(handler_kind, HandlerKind::Activity);
16941 assert_eq!(handler_name, "typed.unsupported.activity");
16942 assert_eq!(value_kind, HandlerValueKind::Result);
16943 assert_eq!(rust_type, type_name::<f64>());
16944 assert!(message.contains("non_finite_float"));
16945 }
16946
16947 #[tokio::test]
16948 async fn typed_replayed_workflow_decodes_input_and_activity_result_losslessly() {
16949 #[derive(Clone, Default)]
16950 struct State {
16951 observed: Option<TypedContract>,
16952 }
16953
16954 let client = Client::new("http://127.0.0.1:8080").expect("client");
16955 let mut worker = Worker::new(client, "rust-workers");
16956 worker.register_typed_replayed_workflow(
16957 "typed.contract.replayed",
16958 State::default,
16959 |ctx, input: TypedContract, state| async move {
16960 let result: TypedContract =
16961 ctx.activity_typed("typed.contract.activity", input).await?;
16962 state.update(|current| current.observed = Some(result.clone()))?;
16963 Ok(result)
16964 },
16965 );
16966 worker.register_replayed_query::<State, _, _>(
16967 "typed.contract.replayed",
16968 "observed",
16969 |_ctx, state, _args| async move {
16970 Ok(json!(state.observed.as_ref().map(|value| value.signed)))
16971 },
16972 );
16973
16974 let expected = typed_contract();
16975 let typed_value = AvroValue::from_serialize(&expected).expect("typed value");
16976 let workflow_arguments =
16977 encode_typed_envelope(&AvroValue::Array(vec![typed_value.clone()]), DEFAULT_CODEC)
16978 .expect("workflow arguments");
16979 let result = encode_typed_envelope(&typed_value, DEFAULT_CODEC).expect("activity result");
16980 let task = QueryTask {
16981 query_task_id: "typed-replay-query".to_string(),
16982 query_task_attempt: 1,
16983 lease_owner: Some("rust-worker".to_string()),
16984 workflow_id: Some("typed-replay".to_string()),
16985 run_id: Some("typed-replay-run".to_string()),
16986 workflow_type: "typed.contract.replayed".to_string(),
16987 query_name: "observed".to_string(),
16988 payload_codec: DEFAULT_CODEC.to_string(),
16989 workflow_arguments: Some(workflow_arguments),
16990 query_arguments: Some(
16991 encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
16992 .expect("query arguments"),
16993 ),
16994 history_events: vec![
16995 history_event(
16996 "ActivityScheduled",
16997 json!({
16998 "sequence": 1,
16999 "activity_type": "typed.contract.activity"
17000 }),
17001 ),
17002 history_event(
17003 "ActivityCompleted",
17004 json!({
17005 "sequence": 1,
17006 "activity_type": "typed.contract.activity",
17007 "payload_codec": DEFAULT_CODEC,
17008 "result": result
17009 }),
17010 ),
17011 ],
17012 history_export: None,
17013 run_status: Some("completed".to_string()),
17014 };
17015
17016 assert_eq!(
17017 worker
17018 .execute_query_task(task)
17019 .await
17020 .expect("typed replay query")
17021 .deserialize::<i64>()
17022 .expect("query result"),
17023 expected.signed
17024 );
17025 }
17026
17027 #[tokio::test]
17028 async fn typed_worker_surfaces_preserve_bytes_and_map_list_identity() {
17029 let client = Client::new("http://127.0.0.1:8080").expect("client");
17030 let mut worker = Worker::new(client, "rust-workers");
17031 worker.register_workflow_avro_value("typed.echo", |_ctx, input| async move { Ok(input) });
17032 worker
17033 .register_activity_avro_value("typed.activity", |_ctx, input| async move { Ok(input) });
17034 worker.register_query_avro_value("typed.echo", "inspect", |_ctx, input| async move {
17035 Ok(input)
17036 });
17037 worker.register_update_avro_value("typed.echo", "replace", |_ctx, input| async move {
17038 Ok(input)
17039 });
17040 worker.register_workflow_avro_value("typed.signal", |ctx, _input| async move {
17041 Ok(AvroValue::Array(
17042 ctx.wait_signal_avro_value("changed").await?,
17043 ))
17044 });
17045
17046 let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
17047 let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed envelope");
17048
17049 let mut workflow = workflow_task("typed.echo", Vec::new(), DEFAULT_CODEC);
17050 workflow.arguments = Some(envelope.clone());
17051 let commands = worker
17052 .execute_workflow_task(workflow)
17053 .expect("typed workflow task");
17054 assert_eq!(commands[0]["type"], "complete_workflow");
17055 assert_eq!(
17056 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17057 .expect("typed workflow result"),
17058 arguments
17059 );
17060
17061 let activity = ActivityTask {
17062 task_id: "activity-typed".to_string(),
17063 activity_attempt_id: Some("attempt-typed".to_string()),
17064 attempt_id: None,
17065 activity_type: "typed.activity".to_string(),
17066 payload_codec: DEFAULT_CODEC.to_string(),
17067 arguments: Some(envelope.clone()),
17068 attempt_number: 1,
17069 lease_owner: Some("rust-worker".to_string()),
17070 };
17071 assert_eq!(
17072 worker
17073 .execute_activity_task(activity)
17074 .await
17075 .expect("typed activity result"),
17076 arguments
17077 );
17078
17079 let query = QueryTask {
17080 query_task_id: "query-typed".to_string(),
17081 query_task_attempt: 1,
17082 lease_owner: Some("rust-worker".to_string()),
17083 workflow_id: Some("typed-1".to_string()),
17084 run_id: Some("run-typed".to_string()),
17085 workflow_type: "typed.echo".to_string(),
17086 query_name: "inspect".to_string(),
17087 payload_codec: DEFAULT_CODEC.to_string(),
17088 workflow_arguments: Some(
17089 encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
17090 .expect("workflow input"),
17091 ),
17092 query_arguments: Some(envelope.clone()),
17093 history_events: Vec::new(),
17094 history_export: None,
17095 run_status: Some("running".to_string()),
17096 };
17097 assert_eq!(
17098 worker
17099 .execute_query_task(query)
17100 .await
17101 .expect("typed query result"),
17102 arguments
17103 );
17104
17105 let mut update = workflow_task(
17106 "typed.echo",
17107 vec![history_event(
17108 "UpdateAccepted",
17109 json!({
17110 "update_id": "update-typed",
17111 "update_name": "replace",
17112 "arguments": envelope.clone(),
17113 }),
17114 )],
17115 DEFAULT_CODEC,
17116 );
17117 update.workflow_update_id = Some("update-typed".to_string());
17118 update.update_name = Some("replace".to_string());
17119 let commands = worker
17120 .execute_workflow_task(update)
17121 .expect("typed update task");
17122 assert_eq!(commands[0]["type"], "complete_update");
17123 assert_eq!(
17124 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17125 .expect("typed update result"),
17126 arguments
17127 );
17128
17129 let mut signal = workflow_task(
17130 "typed.signal",
17131 vec![history_event(
17132 "SignalReceived",
17133 json!({
17134 "signal_id": "signal-typed",
17135 "signal_name": "changed",
17136 "arguments": envelope.clone(),
17137 }),
17138 )],
17139 DEFAULT_CODEC,
17140 );
17141 signal.workflow_signal_id = Some("signal-typed".to_string());
17142 signal.signal_name = Some("changed".to_string());
17143 signal.signal_arguments = Some(envelope);
17144 let commands = worker
17145 .execute_workflow_task(signal)
17146 .expect("typed signal resume");
17147 assert_eq!(
17148 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17149 .expect("typed signal result"),
17150 arguments
17151 );
17152 }
17153
17154 #[tokio::test]
17155 async fn typed_helpers_never_parse_json_inspection_projection() {
17156 let collision_values = projection_collision_probe();
17157 let expected = AvroValue::Array(collision_values.clone());
17158 let envelope = encode_typed_envelope(&expected, DEFAULT_CODEC).expect("collision envelope");
17159
17160 let activity_context = workflow_context_with_codec(
17161 vec![history_event(
17162 "ActivityCompleted",
17163 json!({
17164 "sequence": 1,
17165 "activity_type": "collision.activity",
17166 "payload_codec": DEFAULT_CODEC,
17167 "result": envelope.clone(),
17168 }),
17169 )],
17170 DEFAULT_CODEC,
17171 );
17172 assert_eq!(
17173 activity_context
17174 .activity_avro_value("collision.activity", AvroValue::Array(Vec::new()))
17175 .await
17176 .expect("typed activity collision result"),
17177 expected
17178 );
17179
17180 let signal_context = workflow_context_with_codec(
17181 vec![
17182 history_event(
17183 "SignalWaitOpened",
17184 json!({"sequence": 1, "signal_name": "collision"}),
17185 ),
17186 history_event(
17187 "SignalApplied",
17188 json!({
17189 "sequence": 1,
17190 "signal_name": "collision",
17191 "payload_codec": DEFAULT_CODEC,
17192 "value": envelope.clone(),
17193 }),
17194 ),
17195 ],
17196 DEFAULT_CODEC,
17197 );
17198 assert_eq!(
17199 signal_context
17200 .wait_signal_avro_value("collision")
17201 .await
17202 .expect("typed signal collision arguments"),
17203 collision_values
17204 );
17205
17206 let child_context = workflow_context_with_codec(
17207 vec![
17208 history_event(
17209 "ChildWorkflowScheduled",
17210 json!({
17211 "sequence": 1,
17212 "child_workflow_instance_id": "collision-child",
17213 "child_workflow_run_id": "collision-run",
17214 "child_workflow_type": "collision.child",
17215 }),
17216 ),
17217 history_event(
17218 "ChildRunCompleted",
17219 json!({
17220 "sequence": 1,
17221 "child_workflow_instance_id": "collision-child",
17222 "child_workflow_run_id": "collision-run",
17223 "child_workflow_type": "collision.child",
17224 "payload_codec": DEFAULT_CODEC,
17225 "result": envelope,
17226 }),
17227 ),
17228 ],
17229 DEFAULT_CODEC,
17230 );
17231 let child = child_context
17232 .start_child_workflow_avro_value(
17233 "collision.child",
17234 ChildWorkflowOptions::new("collision-workers"),
17235 AvroValue::Array(Vec::new()),
17236 )
17237 .await
17238 .expect("typed child collision result");
17239 assert_eq!(child.result, expected);
17240 }
17241
17242 #[tokio::test]
17243 async fn replayed_typed_query_keeps_lossless_workflow_and_query_inputs() {
17244 let client = Client::new("http://127.0.0.1:8080").expect("client");
17245 let mut worker = Worker::new(client, "rust-workers");
17246 worker.register_replayed_workflow_avro_value(
17247 "typed.replayed",
17248 || (),
17249 |_ctx, input, _state| async move { Ok(input) },
17250 );
17251 worker.register_replayed_query_avro_value::<(), _, _>(
17252 "typed.replayed",
17253 "inspect",
17254 |ctx, _state, args| async move {
17255 let mut signals = ctx.signals_avro_value("collision");
17256 let signal = signals
17257 .pop()
17258 .map(AvroValue::Array)
17259 .unwrap_or_else(|| AvroValue::Array(Vec::new()));
17260 Ok(AvroValue::Array(vec![
17261 ctx.workflow_input_avro_value().clone(),
17262 signal,
17263 args,
17264 ]))
17265 },
17266 );
17267 let arguments = AvroValue::Array(projection_collision_probe());
17268 let signal_arguments =
17269 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed query signal arguments");
17270 let task = QueryTask {
17271 query_task_id: "query-typed-replay".to_string(),
17272 query_task_attempt: 1,
17273 lease_owner: Some("rust-worker".to_string()),
17274 workflow_id: Some("typed-replay".to_string()),
17275 run_id: Some("run-typed-replay".to_string()),
17276 workflow_type: "typed.replayed".to_string(),
17277 query_name: "inspect".to_string(),
17278 payload_codec: DEFAULT_CODEC.to_string(),
17279 workflow_arguments: Some(
17280 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("workflow arguments"),
17281 ),
17282 query_arguments: Some(
17283 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("query arguments"),
17284 ),
17285 history_events: vec![history_event(
17286 "SignalReceived",
17287 json!({
17288 "signal_id": "collision-signal",
17289 "signal_name": "collision",
17290 "workflow_sequence": 1,
17291 "payload_codec": DEFAULT_CODEC,
17292 "arguments": signal_arguments,
17293 }),
17294 )],
17295 history_export: None,
17296 run_status: Some("completed".to_string()),
17297 };
17298
17299 assert_eq!(
17300 worker
17301 .execute_query_task(task)
17302 .await
17303 .expect("typed replay query"),
17304 AvroValue::Array(vec![arguments.clone(), arguments.clone(), arguments])
17305 );
17306 }
17307
17308 #[test]
17309 fn public_avro_adapter_rejects_non_string_map_keys_before_json_conversion() {
17310 let value = BTreeMap::from([(1_i32, "integer key")]);
17311 let error = PayloadEnvelope::avro(&value)
17312 .expect_err("integer map keys must fail")
17313 .to_string();
17314
17315 assert!(error.contains("invalid_map_key"));
17316 }
17317
17318 #[test]
17319 fn json_tagged_payload_fails_closed_with_actionable_diagnostic() {
17320 let envelope = PayloadEnvelope {
17321 codec: "json".to_string(),
17322 blob: r#"{"greeting":"hello"}"#.to_string(),
17323 };
17324
17325 let error = decode_payload::<Value>(&envelope).expect_err("JSON payload must fail");
17326 let diagnostic = error.to_string();
17327 assert!(diagnostic.contains("unsupported_payload_codec"));
17328 assert!(diagnostic.contains("codec=\"avro\""));
17329 assert!(diagnostic.contains("HTTP document transport"));
17330 }
17331
17332 #[test]
17333 fn untagged_json_payload_value_fails_closed() {
17334 let error = decode_wire_value(&json!({"stale": true}), DEFAULT_CODEC)
17335 .expect_err("untagged JSON payload values must fail");
17336 let diagnostic = error.to_string();
17337 assert!(diagnostic.contains("unsupported_payload_codec"));
17338 assert!(diagnostic.contains("untagged durable payload"));
17339 assert!(diagnostic.contains("HTTP document transport"));
17340 }
17341
17342 #[test]
17343 fn prerelease_avro_payload_without_single_object_frame_is_rejected() {
17344 let envelope = PayloadEnvelope {
17345 codec: DEFAULT_CODEC.to_string(),
17346 blob: BASE64.encode([0x01]),
17347 };
17348
17349 let error = decode_payload::<Value>(&envelope).expect_err("prerelease payload must fail");
17350 assert!(error.to_string().contains("invalid_payload_framing"));
17351 }
17352
17353 #[tokio::test]
17354 async fn workflow_completion_rejects_invalid_payload_slots_without_transport() {
17355 let server = MockWorkerServer::start();
17356 let client = Client::builder(server.base_url())
17357 .timeout(Duration::from_secs(2))
17358 .build()
17359 .expect("client");
17360 let invalid_commands = [
17361 json!({
17362 "type": "complete_workflow",
17363 "result": {"codec": "json", "blob": null}
17364 }),
17365 json!({
17366 "type": "schedule_activity",
17367 "arguments": {"codec": "yaml", "blob": "ignored"}
17368 }),
17369 json!({
17370 "type": "start_child_workflow",
17371 "arguments": {"codec": DEFAULT_CODEC, "blob": null}
17372 }),
17373 json!({"type": "continue_as_new", "arguments": []}),
17374 json!({"type": "complete_update"}),
17375 json!({"type": "record_side_effect", "result": null}),
17376 json!({
17377 "type": "start_service_operation",
17378 "payload_codec": DEFAULT_CODEC,
17379 "request_payload": "raw-avro-bytes"
17380 }),
17381 ];
17382
17383 for command in invalid_commands {
17384 let error = client
17385 .complete_workflow_task("invalid-codec", "rust-worker", 1, vec![command])
17386 .await
17387 .expect_err("invalid durable payload must fail locally");
17388 let diagnostic = error.to_string();
17389 assert!(
17390 diagnostic.contains("unsupported_payload_codec")
17391 || diagnostic.contains("invalid_payload_envelope")
17392 || diagnostic.contains("untagged durable payload"),
17393 "unexpected validation diagnostic: {diagnostic}"
17394 );
17395 }
17396
17397 assert_eq!(
17398 server.request_count("/api/worker/workflow-tasks/invalid-codec/complete"),
17399 0,
17400 "invalid command payloads must not reach HTTP transport"
17401 );
17402 }
17403
17404 #[test]
17405 fn workflow_completion_validates_only_protocol_owned_payload_slots() {
17406 let envelope = fixture_envelope(json!({"codec": "customer-value"}));
17407 let commands = [
17408 json!({"type": "complete_workflow", "result": envelope.clone()}),
17409 json!({"type": "schedule_activity", "arguments": envelope.clone()}),
17410 json!({"type": "start_child_workflow", "arguments": envelope.clone()}),
17411 json!({"type": "continue_as_new", "arguments": envelope.clone()}),
17412 json!({"type": "complete_update", "result": envelope.clone()}),
17413 json!({"type": "record_side_effect", "result": envelope.clone()}),
17414 json!({
17415 "type": "start_service_operation",
17416 "payload_codec": DEFAULT_CODEC,
17417 "request_payload": envelope.clone()
17418 }),
17419 json!({
17420 "type": "complete_workflow",
17421 "result": envelope,
17422 "metadata": {
17423 "codec": "json",
17424 "payload_codec": "customer-codec",
17425 "result": {"codec": "yaml", "blob": null}
17426 }
17427 }),
17428 ];
17429
17430 validate_workflow_task_commands(&commands)
17431 .expect("customer metadata must not become a protocol codec declaration");
17432 }
17433
17434 #[test]
17435 fn valid_avro_tasks_normalize_absent_and_null_arguments_to_empty_lists() {
17436 assert_eq!(
17437 decode_task_avro_arguments(None, DEFAULT_CODEC).expect("absent arguments"),
17438 AvroValue::Array(Vec::new())
17439 );
17440 assert_eq!(
17441 decode_task_avro_arguments(Some(&Value::Null), DEFAULT_CODEC).expect("null arguments"),
17442 AvroValue::Array(Vec::new())
17443 );
17444
17445 let mut signal = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
17446 signal.signal_name = Some("empty-signal".to_string());
17447 signal.signal_arguments = None;
17448 let decoded = decode_resume_signal(&signal)
17449 .expect("valid Avro signal")
17450 .expect("named signal resumes the workflow");
17451 assert!(decoded.arguments.is_empty());
17452 }
17453
17454 #[tokio::test]
17455 async fn malformed_task_level_codecs_become_pre_handler_failures() {
17456 let client = Client::new("http://127.0.0.1:8080").expect("client");
17457 let mut worker = Worker::new(client, "rust-workers");
17458 let handler_calls = Arc::new(AtomicUsize::new(0));
17459
17460 let calls = Arc::clone(&handler_calls);
17461 worker.register_workflow("codec.workflow", move |_ctx, _args| {
17462 calls.fetch_add(1, Ordering::SeqCst);
17463 async move { Ok(Value::Null) }
17464 });
17465 let calls = Arc::clone(&handler_calls);
17466 worker.register_activity("codec.activity", move |_ctx, _args| {
17467 calls.fetch_add(1, Ordering::SeqCst);
17468 async move { Ok(Value::Null) }
17469 });
17470 let calls = Arc::clone(&handler_calls);
17471 worker.register_query("codec.workflow", "known", move |_ctx, _args| {
17472 calls.fetch_add(1, Ordering::SeqCst);
17473 async move { Ok(Value::Null) }
17474 });
17475
17476 let mut failures = Vec::new();
17477 for codec_case in [
17478 InvalidTaskPayloadCodec::Missing,
17479 InvalidTaskPayloadCodec::Null,
17480 InvalidTaskPayloadCodec::NonString,
17481 ] {
17482 let mut workflow = json!({
17483 "task_id": format!("workflow-{}", codec_case.label()),
17484 "workflow_type": "codec.workflow"
17485 });
17486 codec_case.apply(&mut workflow);
17487 match serde_json::from_value::<WorkflowTask>(workflow) {
17488 Ok(task) => match worker.execute_workflow_task(task) {
17489 Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
17490 outcome => failures.push(format!(
17491 "workflow {} codec returned {outcome:?}",
17492 codec_case.label()
17493 )),
17494 },
17495 Err(error) => failures.push(format!(
17496 "workflow {} codec failed transport deserialization: {error}",
17497 codec_case.label()
17498 )),
17499 }
17500
17501 let mut activity = json!({
17502 "task_id": format!("activity-{}", codec_case.label()),
17503 "activity_attempt_id": format!("attempt-{}", codec_case.label()),
17504 "activity_type": "codec.activity",
17505 "attempt_number": 1
17506 });
17507 codec_case.apply(&mut activity);
17508 match serde_json::from_value::<ActivityTask>(activity) {
17509 Ok(task) => match worker.execute_activity_task(task).await {
17510 Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
17511 outcome => failures.push(format!(
17512 "activity {} codec returned {outcome:?}",
17513 codec_case.label()
17514 )),
17515 },
17516 Err(error) => failures.push(format!(
17517 "activity {} codec failed transport deserialization: {error}",
17518 codec_case.label()
17519 )),
17520 }
17521
17522 let mut query = json!({
17523 "query_task_id": format!("query-{}", codec_case.label()),
17524 "workflow_type": "codec.workflow",
17525 "query_name": "known"
17526 });
17527 codec_case.apply(&mut query);
17528 match serde_json::from_value::<QueryTask>(query) {
17529 Ok(task) => match worker.execute_query_task(task).await {
17530 Err(failure) if failure.message.contains("unsupported_payload_codec") => {}
17531 outcome => failures.push(format!(
17532 "query {} codec returned {outcome:?}",
17533 codec_case.label()
17534 )),
17535 },
17536 Err(error) => failures.push(format!(
17537 "query {} codec failed transport deserialization: {error}",
17538 codec_case.label()
17539 )),
17540 }
17541 }
17542
17543 assert!(failures.is_empty(), "{}", failures.join("\n"));
17544 assert_eq!(
17545 handler_calls.load(Ordering::SeqCst),
17546 0,
17547 "invalid task codecs must not invoke a handler"
17548 );
17549 }
17550
17551 #[tokio::test]
17552 async fn polled_malformed_task_codecs_are_settled_without_handler_execution() {
17553 for codec_case in [
17554 InvalidTaskPayloadCodec::Missing,
17555 InvalidTaskPayloadCodec::Null,
17556 InvalidTaskPayloadCodec::NonString,
17557 ] {
17558 let server = MockWorkerServer::invalid_task_payload_codec(codec_case);
17559 let client = Client::builder(server.base_url())
17560 .timeout(Duration::from_secs(2))
17561 .build()
17562 .expect("client");
17563 let mut worker = Worker::new(client, "rust-workers")
17564 .worker_id("codec-worker")
17565 .poll_timeout(Duration::from_millis(10));
17566 let handler_calls = Arc::new(AtomicUsize::new(0));
17567
17568 let calls = Arc::clone(&handler_calls);
17569 worker.register_workflow("codec.workflow", move |_ctx, _args| {
17570 calls.fetch_add(1, Ordering::SeqCst);
17571 async move { Ok(Value::Null) }
17572 });
17573 let calls = Arc::clone(&handler_calls);
17574 worker.register_activity("codec.activity", move |_ctx, _args| {
17575 calls.fetch_add(1, Ordering::SeqCst);
17576 async move { Ok(Value::Null) }
17577 });
17578 let calls = Arc::clone(&handler_calls);
17579 worker.register_query("codec.workflow", "known", move |_ctx, _args| {
17580 calls.fetch_add(1, Ordering::SeqCst);
17581 async move { Ok(Value::Null) }
17582 });
17583
17584 assert_eq!(
17585 worker.run_once().await.expect("invalid tasks are settled"),
17586 3,
17587 "all {} codec tasks must be handled",
17588 codec_case.label()
17589 );
17590 assert_eq!(
17591 handler_calls.load(Ordering::SeqCst),
17592 0,
17593 "{} task codecs must fail before every handler",
17594 codec_case.label()
17595 );
17596
17597 for path in [
17598 "/api/worker/workflow-tasks/codec-workflow/fail",
17599 "/api/worker/activity-tasks/codec-activity/fail",
17600 "/api/worker/query-tasks/codec-query/fail",
17601 ] {
17602 let body = server.request_body(path);
17603 assert!(
17604 body["failure"]["message"]
17605 .as_str()
17606 .is_some_and(|message| message.contains("unsupported_payload_codec")),
17607 "{path} must receive the stable codec diagnostic for the {} case: {body}",
17608 codec_case.label()
17609 );
17610 }
17611 assert_eq!(
17612 server.request_body("/api/worker/query-tasks/codec-query/fail")["failure"]
17613 ["reason"],
17614 "query_payload_decode_failed"
17615 );
17616 for path in [
17617 "/api/worker/workflow-tasks/codec-workflow/complete",
17618 "/api/worker/activity-tasks/codec-activity/complete",
17619 "/api/worker/query-tasks/codec-query/complete",
17620 ] {
17621 assert_eq!(
17622 server.request_count(path),
17623 0,
17624 "invalid {} codec task reached {path}",
17625 codec_case.label()
17626 );
17627 }
17628 }
17629 }
17630
17631 #[tokio::test]
17632 async fn invalid_inbound_codecs_precede_handlers_and_unrelated_outcomes() {
17633 let client = Client::new("http://127.0.0.1:8080").expect("client");
17634 let mut worker = Worker::new(client, "rust-workers");
17635 let handler_calls = Arc::new(AtomicUsize::new(0));
17636
17637 let calls = Arc::clone(&handler_calls);
17638 worker.register_workflow("codec.workflow", move |_ctx, _args| {
17639 calls.fetch_add(1, Ordering::SeqCst);
17640 async move { Ok(Value::Null) }
17641 });
17642 let calls = Arc::clone(&handler_calls);
17643 worker.register_activity("codec.activity", move |_ctx, _args| {
17644 calls.fetch_add(1, Ordering::SeqCst);
17645 async move { Ok(Value::Null) }
17646 });
17647 let calls = Arc::clone(&handler_calls);
17648 worker.register_update("codec.workflow", "known", move |_ctx, _args| {
17649 calls.fetch_add(1, Ordering::SeqCst);
17650 async move { Ok(Value::Null) }
17651 });
17652 let calls = Arc::clone(&handler_calls);
17653 worker.register_query("codec.workflow", "known", move |_ctx, _args| {
17654 calls.fetch_add(1, Ordering::SeqCst);
17655 async move { Ok(Value::Null) }
17656 });
17657
17658 let mut workflow = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
17659 workflow.payload_codec = "json".to_string();
17660 workflow.arguments = None;
17661 let error = worker
17662 .execute_workflow_task(workflow)
17663 .expect_err("task codec must be checked before workflow invocation");
17664 assert!(error.to_string().contains("unsupported_payload_codec"));
17665
17666 let activity = ActivityTask {
17667 task_id: "activity-invalid-codec".to_string(),
17668 activity_attempt_id: None,
17669 attempt_id: None,
17670 activity_type: "codec.activity".to_string(),
17671 payload_codec: "unknown".to_string(),
17672 arguments: None,
17673 attempt_number: 1,
17674 lease_owner: None,
17675 };
17676 let error = worker
17677 .execute_activity_task(activity)
17678 .await
17679 .expect_err("task codec must be checked before activity invocation");
17680 assert!(error.to_string().contains("unsupported_payload_codec"));
17681
17682 let mut update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
17683 update.workflow_update_id = Some("update-invalid-codec".to_string());
17684 update.update_name = Some("known".to_string());
17685 update.history_events.push(history_event(
17686 "UpdateAccepted",
17687 json!({
17688 "update_id": "update-invalid-codec",
17689 "update_name": "known",
17690 "arguments": {"codec": "json", "blob": null}
17691 }),
17692 ));
17693 let error = worker
17694 .execute_workflow_task(update)
17695 .expect_err("nested update codec must be checked before handler lookup");
17696 assert!(error.to_string().contains("unsupported_payload_codec"));
17697
17698 let query: QueryTask = serde_json::from_value(json!({
17699 "query_task_id": "query-invalid-codec",
17700 "workflow_type": "codec.workflow",
17701 "query_name": "known",
17702 "payload_codec": DEFAULT_CODEC,
17703 "workflow_arguments": null,
17704 "query_arguments": null,
17705 "history_export": {
17706 "payloads": {"codec": DEFAULT_CODEC},
17707 "signals": [{
17708 "name": "empty",
17709 "payload_codec": "json",
17710 "arguments": null
17711 }]
17712 }
17713 }))
17714 .expect("query task");
17715 let failure = worker
17716 .execute_query_task(query)
17717 .await
17718 .expect_err("exported signal codec must be checked before query invocation");
17719 assert_eq!(failure.reason, "query_payload_decode_failed");
17720 assert!(failure.message.contains("unsupported_payload_codec"));
17721
17722 let exported_history: QueryTask = serde_json::from_value(json!({
17723 "query_task_id": "query-invalid-history-codec",
17724 "workflow_type": "codec.workflow",
17725 "query_name": "known",
17726 "payload_codec": DEFAULT_CODEC,
17727 "history_export": {
17728 "payloads": {"codec": DEFAULT_CODEC},
17729 "history_events": [{
17730 "type": "ActivityCompleted",
17731 "payload": {"payload_codec": "unknown", "result": null}
17732 }]
17733 }
17734 }))
17735 .expect("query task");
17736 let failure = worker
17737 .execute_query_task(exported_history)
17738 .await
17739 .expect_err("exported history codec must be checked before query invocation");
17740 assert_eq!(failure.reason, "query_payload_decode_failed");
17741 assert!(failure.message.contains("unsupported_payload_codec"));
17742 assert_eq!(handler_calls.load(Ordering::SeqCst), 0);
17743
17744 let mut unknown_workflow = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
17745 unknown_workflow.arguments = None;
17746 unknown_workflow.history_events.push(history_event(
17747 "SignalReceived",
17748 json!({
17749 "signal_name": "empty",
17750 "payload_codec": "json",
17751 "arguments": null
17752 }),
17753 ));
17754 let error = worker
17755 .execute_workflow_task(unknown_workflow)
17756 .expect_err("history codec must precede unknown workflow outcome");
17757 assert!(error.to_string().contains("unsupported_payload_codec"));
17758
17759 let unknown_activity = ActivityTask {
17760 task_id: "activity-unknown".to_string(),
17761 activity_attempt_id: None,
17762 attempt_id: None,
17763 activity_type: "missing".to_string(),
17764 payload_codec: "json".to_string(),
17765 arguments: None,
17766 attempt_number: 1,
17767 lease_owner: None,
17768 };
17769 let error = worker
17770 .execute_activity_task(unknown_activity)
17771 .await
17772 .expect_err("codec must precede unknown activity outcome");
17773 assert!(error.to_string().contains("unsupported_payload_codec"));
17774
17775 let mut unknown_update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
17776 unknown_update.payload_codec = "json".to_string();
17777 unknown_update.arguments = None;
17778 unknown_update.workflow_update_id = Some("update-unknown".to_string());
17779 unknown_update.update_name = Some("missing".to_string());
17780 let error = worker
17781 .execute_workflow_task(unknown_update)
17782 .expect_err("codec must precede fail_update shortcut");
17783 assert!(error.to_string().contains("unsupported_payload_codec"));
17784
17785 let unknown_query: QueryTask = serde_json::from_value(json!({
17786 "query_task_id": "query-unknown",
17787 "workflow_type": "missing",
17788 "query_name": "missing",
17789 "payload_codec": "json",
17790 "workflow_arguments": null,
17791 "query_arguments": null
17792 }))
17793 .expect("query task");
17794 let failure = worker
17795 .execute_query_task(unknown_query)
17796 .await
17797 .expect_err("codec must precede unknown query outcome");
17798 assert_eq!(failure.reason, "query_payload_decode_failed");
17799 assert!(failure.message.contains("unsupported_payload_codec"));
17800 }
17801
17802 #[tokio::test]
17803 async fn invalid_signal_history_payload_aliases_precede_shortcuts() {
17804 let client = Client::new("http://127.0.0.1:8080").expect("client");
17805 let worker = Worker::new(client, "rust-workers");
17806
17807 for event_type in ["SignalReceived", "SignalApplied"] {
17808 for (payload_field, codec) in [
17809 ("value", "json"),
17810 ("input", "unknown"),
17811 ("arguments", "json"),
17812 ] {
17813 let payload = json!({
17814 "signal_name": "empty",
17815 payload_field: {"codec": codec, "blob": null}
17816 });
17817 let workflow = workflow_task(
17818 "missing",
17819 vec![history_event(event_type, payload.clone())],
17820 DEFAULT_CODEC,
17821 );
17822 let error = worker
17823 .execute_workflow_task(workflow)
17824 .expect_err("signal payload codec must precede unknown workflow outcome");
17825 assert!(
17826 error.to_string().contains("unsupported_payload_codec"),
17827 "{event_type}.{payload_field} returned an unrelated workflow error: {error}"
17828 );
17829
17830 let query: QueryTask = serde_json::from_value(json!({
17831 "query_task_id": format!("query-{event_type}-{payload_field}"),
17832 "workflow_type": "missing",
17833 "query_name": "missing",
17834 "payload_codec": DEFAULT_CODEC,
17835 "workflow_arguments": null,
17836 "query_arguments": null,
17837 "history_events": [{
17838 "event_type": event_type,
17839 "payload": payload
17840 }]
17841 }))
17842 .expect("query task");
17843 let failure = worker
17844 .execute_query_task(query)
17845 .await
17846 .expect_err("signal payload codec must precede unknown query outcome");
17847 assert_eq!(
17848 failure.reason, "query_payload_decode_failed",
17849 "{event_type}.{payload_field} returned an unrelated query outcome"
17850 );
17851 assert!(
17852 failure.message.contains("unsupported_payload_codec"),
17853 "{event_type}.{payload_field} returned an unrelated query error: {}",
17854 failure.message
17855 );
17856 }
17857 }
17858 }
17859
17860 #[test]
17861 fn workflow_context_schedules_activity_until_completion_is_in_history() {
17862 let ctx = WorkflowContext {
17863 state: Arc::new(Mutex::new(
17864 WorkflowState::new_with_identity(
17865 Vec::new(),
17866 Some("wf-parent".to_string()),
17867 Some("run-parent".to_string()),
17868 "rust-workers".to_string(),
17869 DEFAULT_CODEC.to_string(),
17870 None,
17871 )
17872 .expect("workflow state"),
17873 )),
17874 };
17875
17876 let mut call = Box::pin(ctx.activity("hello.activity", json!(["Ada"])));
17877 let mut task_context = TaskContext::from_waker(noop_waker_ref());
17878 assert!(matches!(
17879 call.as_mut().poll(&mut task_context),
17880 Poll::Pending
17881 ));
17882
17883 let commands = ctx.take_commands().expect("commands");
17884 assert_eq!(commands[0]["type"], "schedule_activity");
17885 assert_eq!(commands[0]["activity_type"], "hello.activity");
17886 }
17887
17888 #[test]
17889 fn activity_options_encode_retry_policy_queue_and_every_timeout() {
17890 let ctx = workflow_context(Vec::new());
17891 let options = ActivityOptions::new()
17892 .task_queue("payments")
17893 .retry_policy(
17894 ActivityRetryPolicy::new(4)
17895 .exponential_backoff(Duration::from_secs(1), 3, Some(Duration::from_secs(10)))
17896 .non_retryable_error_type("ValidationError"),
17897 )
17898 .start_to_close_timeout(Duration::from_secs(120))
17899 .schedule_to_start_timeout(Duration::from_secs(10))
17900 .schedule_to_close_timeout(Duration::from_secs(300))
17901 .heartbeat_timeout(Duration::from_secs(15));
17902 let mut call = Box::pin(ctx.activity_with_options(
17903 "charge-card",
17904 options,
17905 json!([{"order_id": "o-1"}]),
17906 ));
17907 let mut task_context = TaskContext::from_waker(noop_waker_ref());
17908
17909 assert!(matches!(
17910 call.as_mut().poll(&mut task_context),
17911 Poll::Pending
17912 ));
17913 assert!(matches!(
17914 call.as_mut().poll(&mut task_context),
17915 Poll::Pending
17916 ));
17917
17918 let commands = ctx.take_commands().expect("activity command");
17919 assert_eq!(commands.len(), 1, "one future emits one logical schedule");
17920 assert_eq!(commands[0]["queue"], "payments");
17921 assert_eq!(
17922 commands[0]["retry_policy"],
17923 json!({
17924 "max_attempts": 4,
17925 "backoff_seconds": [1, 3, 9],
17926 "non_retryable_error_types": ["ValidationError"],
17927 })
17928 );
17929 assert_eq!(commands[0]["start_to_close_timeout"], 120);
17930 assert_eq!(commands[0]["schedule_to_start_timeout"], 10);
17931 assert_eq!(commands[0]["schedule_to_close_timeout"], 300);
17932 assert_eq!(commands[0]["heartbeat_timeout"], 15);
17933 }
17934
17935 #[test]
17936 fn activity_options_encode_explicit_and_rounded_backoff_intervals() {
17937 let ctx = workflow_context(Vec::new());
17938 let options = ActivityOptions::new().retry_policy(
17939 ActivityRetryPolicy::new(3)
17940 .backoff_intervals([Duration::from_millis(1), Duration::from_millis(1_001)]),
17941 );
17942 let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
17943 let mut task_context = TaskContext::from_waker(noop_waker_ref());
17944
17945 assert!(matches!(
17946 call.as_mut().poll(&mut task_context),
17947 Poll::Pending
17948 ));
17949 assert_eq!(
17950 ctx.take_commands().expect("command")[0]["retry_policy"]["backoff_seconds"],
17951 json!([1, 2])
17952 );
17953 }
17954
17955 #[test]
17956 fn invalid_activity_options_return_typed_errors_before_emitting_commands() {
17957 let cases = [
17958 (
17959 ActivityOptions::new().task_queue(" "),
17960 ActivityOptionsErrorKind::EmptyTaskQueue,
17961 ),
17962 (
17963 ActivityOptions::new().retry_policy(ActivityRetryPolicy::default()),
17964 ActivityOptionsErrorKind::EmptyRetryPolicy,
17965 ),
17966 (
17967 ActivityOptions::new().retry_policy(ActivityRetryPolicy::new(0)),
17968 ActivityOptionsErrorKind::InvalidMaxAttempts,
17969 ),
17970 (
17971 ActivityOptions::new().retry_policy(ActivityRetryPolicy {
17972 max_attempts: None,
17973 backoff: Some(ActivityBackoff::Explicit(vec![Duration::from_secs(1)])),
17974 non_retryable_error_types: Vec::new(),
17975 }),
17976 ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
17977 ),
17978 (
17979 ActivityOptions::new().retry_policy(
17980 ActivityRetryPolicy::new(2)
17981 .backoff_intervals([Duration::from_secs(1), Duration::from_secs(2)]),
17982 ),
17983 ActivityOptionsErrorKind::TooManyBackoffIntervals,
17984 ),
17985 (
17986 ActivityOptions::new().retry_policy(
17987 ActivityRetryPolicy::new(2).exponential_backoff(
17988 Duration::from_secs(1),
17989 0,
17990 None,
17991 ),
17992 ),
17993 ActivityOptionsErrorKind::InvalidBackoffCoefficient,
17994 ),
17995 (
17996 ActivityOptions::new()
17997 .retry_policy(ActivityRetryPolicy::new(2).non_retryable_error_type(" ")),
17998 ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
17999 ),
18000 (
18001 ActivityOptions::new().retry_policy(
18002 ActivityRetryPolicy::new(10_002).exponential_backoff(
18003 Duration::from_secs(1),
18004 1,
18005 None,
18006 ),
18007 ),
18008 ActivityOptionsErrorKind::BackoffGenerationTooLarge,
18009 ),
18010 (
18011 ActivityOptions::new().retry_policy(
18012 ActivityRetryPolicy::new(2)
18013 .backoff_intervals([Duration::from_secs(i64::MAX as u64 + 1)]),
18014 ),
18015 ActivityOptionsErrorKind::BackoffOverflow,
18016 ),
18017 ];
18018
18019 for (options, expected_kind) in cases {
18020 let ctx = workflow_context(Vec::new());
18021 let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
18022 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18023 let Poll::Ready(Err(Error::InvalidActivityOptions(error))) =
18024 call.as_mut().poll(&mut task_context)
18025 else {
18026 panic!("expected typed activity validation error");
18027 };
18028 assert_eq!(error.kind, expected_kind);
18029 assert!(ctx.take_commands().expect("commands").is_empty());
18030 }
18031 }
18032
18033 #[test]
18034 fn activity_options_validate_positive_and_ordered_timeouts() {
18035 let zero_timeout_cases = [
18036 ActivityOptions::new().start_to_close_timeout(Duration::ZERO),
18037 ActivityOptions::new().schedule_to_start_timeout(Duration::ZERO),
18038 ActivityOptions::new().schedule_to_close_timeout(Duration::ZERO),
18039 ActivityOptions::new().heartbeat_timeout(Duration::ZERO),
18040 ];
18041 for options in zero_timeout_cases {
18042 assert_eq!(
18043 options.validate().expect_err("zero timeout").kind,
18044 ActivityOptionsErrorKind::TimeoutNotPositive
18045 );
18046 }
18047
18048 let ordering_cases = [
18049 ActivityOptions::new()
18050 .heartbeat_timeout(Duration::from_secs(11))
18051 .start_to_close_timeout(Duration::from_secs(10)),
18052 ActivityOptions::new()
18053 .start_to_close_timeout(Duration::from_secs(31))
18054 .schedule_to_close_timeout(Duration::from_secs(30)),
18055 ActivityOptions::new()
18056 .schedule_to_start_timeout(Duration::from_secs(31))
18057 .schedule_to_close_timeout(Duration::from_secs(30)),
18058 ];
18059 for options in ordering_cases {
18060 assert_eq!(
18061 options.validate().expect_err("timeout order").kind,
18062 ActivityOptionsErrorKind::TimeoutOrder
18063 );
18064 }
18065
18066 assert_eq!(
18067 ActivityOptions::new()
18068 .start_to_close_timeout(Duration::from_secs(i64::MAX as u64 + 1))
18069 .validate()
18070 .expect_err("protocol integer overflow")
18071 .kind,
18072 ActivityOptionsErrorKind::TimeoutOverflow
18073 );
18074 }
18075
18076 #[test]
18077 fn replayed_activity_retry_history_completes_without_duplicate_schedule() {
18078 let ctx = workflow_context(completed_retry_activity_history());
18079 let mut call =
18080 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18081 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18082
18083 assert!(matches!(
18084 call.as_mut().poll(&mut task_context),
18085 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18086 ));
18087 assert!(ctx.take_commands().expect("commands").is_empty());
18088 ctx.ensure_history_consumed().expect("history consumed");
18089 }
18090
18091 #[test]
18092 fn duplicate_non_retryable_types_use_one_command_and_replay_representation() {
18093 let mut options = retry_activity_options();
18094 options
18095 .retry_policy
18096 .as_mut()
18097 .expect("retry policy")
18098 .non_retryable_error_types
18099 .extend([" PermanentError ".to_string(), "PermanentError".to_string()]);
18100
18101 let new_ctx = workflow_context(Vec::new());
18102 let mut new_call =
18103 Box::pin(new_ctx.activity_with_options("flaky", options.clone(), json!([])));
18104 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18105 assert!(matches!(
18106 new_call.as_mut().poll(&mut task_context),
18107 Poll::Pending
18108 ));
18109 let commands = new_ctx.take_commands().expect("commands");
18110 assert_eq!(commands.len(), 1);
18111 assert_eq!(
18112 commands[0]["retry_policy"]["non_retryable_error_types"],
18113 json!(["PermanentError"])
18114 );
18115
18116 let replay_ctx = workflow_context(completed_retry_activity_history());
18117 let mut replay_call =
18118 Box::pin(replay_ctx.activity_with_options("flaky", options, json!([])));
18119 assert!(matches!(
18120 replay_call.as_mut().poll(&mut task_context),
18121 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18122 ));
18123 assert!(replay_ctx.take_commands().expect("commands").is_empty());
18124 replay_ctx
18125 .ensure_history_consumed()
18126 .expect("history consumed");
18127 }
18128
18129 #[test]
18130 fn replayed_intermediate_retry_remains_pending_across_restarts() {
18131 let history = completed_retry_activity_history()
18132 .into_iter()
18133 .take(3)
18134 .collect::<Vec<_>>();
18135
18136 for _restart in 0..2 {
18137 let ctx = workflow_context(history.clone());
18138 let mut call =
18139 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18140 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18141 assert!(matches!(
18142 call.as_mut().poll(&mut task_context),
18143 Poll::Pending
18144 ));
18145 assert!(ctx.take_commands().expect("commands").is_empty());
18146 }
18147 }
18148
18149 #[test]
18150 fn replayed_activity_rejects_changed_queue_retry_and_every_timeout_field() {
18151 let mut changed_queue = retry_activity_options();
18152 changed_queue.task_queue = Some("different-queue".to_string());
18153
18154 let mut changed_max_attempts = retry_activity_options();
18155 let retry_policy = changed_max_attempts
18156 .retry_policy
18157 .as_mut()
18158 .expect("retry policy");
18159 retry_policy.max_attempts = Some(4);
18160
18161 let mut changed_backoff = retry_activity_options();
18162 let retry_policy = changed_backoff.retry_policy.as_mut().expect("retry policy");
18163 retry_policy.backoff = Some(ActivityBackoff::Explicit(vec![
18164 Duration::from_secs(3),
18165 Duration::from_secs(4),
18166 ]));
18167
18168 let mut changed_non_retryable_types = retry_activity_options();
18169 let retry_policy = changed_non_retryable_types
18170 .retry_policy
18171 .as_mut()
18172 .expect("retry policy");
18173 retry_policy.non_retryable_error_types = vec!["AnotherPermanentError".to_string()];
18174
18175 let mut changed_start_to_close = retry_activity_options();
18176 changed_start_to_close.start_to_close_timeout = Some(Duration::from_secs(31));
18177 let mut changed_schedule_to_start = retry_activity_options();
18178 changed_schedule_to_start.schedule_to_start_timeout = Some(Duration::from_secs(6));
18179 let mut changed_schedule_to_close = retry_activity_options();
18180 changed_schedule_to_close.schedule_to_close_timeout = Some(Duration::from_secs(91));
18181 let mut changed_heartbeat = retry_activity_options();
18182 changed_heartbeat.heartbeat_timeout = Some(Duration::from_secs(11));
18183
18184 let cases = [
18185 (changed_queue, "activity_task_queue_mismatch"),
18186 (changed_max_attempts, "activity_retry_policy_mismatch"),
18187 (changed_backoff, "activity_retry_policy_mismatch"),
18188 (
18189 changed_non_retryable_types,
18190 "activity_retry_policy_mismatch",
18191 ),
18192 (changed_start_to_close, "activity_retry_policy_mismatch"),
18193 (changed_schedule_to_start, "activity_retry_policy_mismatch"),
18194 (changed_schedule_to_close, "activity_retry_policy_mismatch"),
18195 (changed_heartbeat, "activity_retry_policy_mismatch"),
18196 ];
18197
18198 for (options, expected_reason) in cases {
18199 let ctx = workflow_context(completed_retry_activity_history());
18200 let mut call = Box::pin(ctx.activity_with_options("flaky", options, json!([])));
18201 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18202 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18203 call.as_mut().poll(&mut task_context)
18204 else {
18205 panic!("changed activity options must fail replay");
18206 };
18207 assert_eq!(failure.reason, expected_reason);
18208 assert_eq!(failure.sequence, Some(1));
18209 assert!(ctx.take_commands().expect("commands").is_empty());
18210 }
18211 }
18212
18213 #[test]
18214 fn replayed_activity_rejects_changed_execution_mode_and_snapshot_version() {
18215 let cases = [
18216 (
18217 "execution_mode",
18218 json!("local"),
18219 "activity_execution_mode_mismatch",
18220 ),
18221 (
18222 "snapshot_version",
18223 json!(2),
18224 "activity_retry_policy_mismatch",
18225 ),
18226 ];
18227
18228 for (field, value, expected_reason) in cases {
18229 let mut history = completed_retry_activity_history();
18230 let activity = history[0].payload["activity"]
18231 .as_object_mut()
18232 .expect("activity snapshot");
18233 if field == "execution_mode" {
18234 activity.insert(field.to_string(), value);
18235 } else {
18236 activity["retry_policy"]
18237 .as_object_mut()
18238 .expect("retry snapshot")
18239 .insert(field.to_string(), value);
18240 }
18241
18242 let ctx = workflow_context(history);
18243 let mut call =
18244 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18245 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18246 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18247 call.as_mut().poll(&mut task_context)
18248 else {
18249 panic!("changed {field} must fail replay");
18250 };
18251 assert_eq!(failure.reason, expected_reason);
18252 assert_eq!(failure.sequence, Some(1));
18253 assert!(ctx.take_commands().expect("commands").is_empty());
18254 }
18255 }
18256
18257 #[test]
18258 fn replayed_legacy_activity_treats_missing_option_snapshot_as_unknown() {
18259 let mut history = completed_retry_activity_history();
18260 let activity = history[0].payload["activity"]
18261 .as_object_mut()
18262 .expect("activity snapshot");
18263 activity.remove("execution_mode");
18264 activity.remove("retry_policy");
18265
18266 let mut current = retry_activity_options();
18267 current.start_to_close_timeout = Some(Duration::from_secs(45));
18268 current.schedule_to_start_timeout = Some(Duration::from_secs(8));
18269 current.schedule_to_close_timeout = Some(Duration::from_secs(120));
18270 current.heartbeat_timeout = Some(Duration::from_secs(12));
18271
18272 let ctx = workflow_context(history);
18273 let mut call = Box::pin(ctx.activity_with_options("flaky", current, json!([])));
18274 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18275 assert!(matches!(
18276 call.as_mut().poll(&mut task_context),
18277 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18278 ));
18279 assert!(ctx.take_commands().expect("commands").is_empty());
18280 ctx.ensure_history_consumed().expect("history consumed");
18281 }
18282
18283 #[test]
18284 fn terminal_activity_failed_after_start_returns_typed_failure() {
18285 let history = vec![
18286 history_event(
18287 "ActivityScheduled",
18288 json!({
18289 "sequence": 1,
18290 "activity_type": "flaky",
18291 "activity_execution_id": "act-terminal",
18292 "activity": {
18293 "id": "act-terminal",
18294 "sequence": 1,
18295 "type": "flaky",
18296 "queue": "critical-activities",
18297 "retry_policy": {
18298 "snapshot_version": 1,
18299 "max_attempts": 3,
18300 "backoff_seconds": [2, 4],
18301 "non_retryable_error_types": ["PermanentError"]
18302 }
18303 }
18304 }),
18305 ),
18306 history_event(
18307 "ActivityStarted",
18308 json!({
18309 "sequence": 1,
18310 "activity_type": "flaky",
18311 "activity_execution_id": "act-terminal",
18312 "activity_attempt_id": "attempt-1",
18313 "attempt_number": 1
18314 }),
18315 ),
18316 history_event(
18317 "ActivityFailed",
18318 json!({
18319 "sequence": 1,
18320 "activity_type": "flaky",
18321 "activity_execution_id": "act-terminal",
18322 "activity_attempt_id": "attempt-1",
18323 "attempt_number": 1,
18324 "failure_id": "failure-terminal",
18325 "failure_category": "activity",
18326 "exception_type": "PermanentError",
18327 "message": "cannot retry",
18328 "non_retryable": true
18329 }),
18330 ),
18331 ];
18332 let ctx = workflow_context(history);
18333 let mut call =
18334 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18335 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18336
18337 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18338 call.as_mut().poll(&mut task_context)
18339 else {
18340 panic!("terminal ActivityFailed must settle the activity future");
18341 };
18342 assert_eq!(failure.kind, ActivityFailureKind::Failed);
18343 assert_eq!(
18344 failure.activity_execution_id.as_deref(),
18345 Some("act-terminal")
18346 );
18347 assert_eq!(failure.exception_type.as_deref(), Some("PermanentError"));
18348 assert!(failure.non_retryable);
18349 assert!(ctx.take_commands().expect("commands").is_empty());
18350 ctx.ensure_history_consumed().expect("history consumed");
18351 }
18352
18353 #[test]
18354 fn activity_terminal_events_return_machine_readable_failures() {
18355 let cases = [
18356 (
18357 "ActivityFailed",
18358 json!({
18359 "sequence": 1,
18360 "activity_type": "charge-card",
18361 "activity_execution_id": "act-1",
18362 "activity_attempt_id": "attempt-2",
18363 "attempt_number": 2,
18364 "failure_id": "failure-1",
18365 "failure_category": "activity",
18366 "exception_type": "PaymentDeclined",
18367 "exception_class": "payments.PaymentDeclined",
18368 "message": "card declined",
18369 "non_retryable": true
18370 }),
18371 ActivityFailureKind::Failed,
18372 "activity",
18373 ),
18374 (
18375 "ActivityCancelled",
18376 json!({
18377 "sequence": 1,
18378 "activity_type": "charge-card",
18379 "activity_execution_id": "act-1",
18380 "activity_attempt_id": "attempt-1"
18381 }),
18382 ActivityFailureKind::Cancelled,
18383 "cancelled",
18384 ),
18385 ];
18386
18387 for (event_type, payload, expected_kind, expected_reason) in cases {
18388 let ctx = workflow_context(vec![history_event(event_type, payload)]);
18389 let mut call = Box::pin(ctx.activity("charge-card", json!([])));
18390 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18391 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18392 call.as_mut().poll(&mut task_context)
18393 else {
18394 panic!("expected terminal activity failure");
18395 };
18396 assert_eq!(failure.kind, expected_kind);
18397 assert_eq!(failure.reason, expected_reason);
18398 assert_eq!(failure.activity_execution_id.as_deref(), Some("act-1"));
18399 assert_eq!(failure.activity_type.as_deref(), Some("charge-card"));
18400 }
18401 }
18402
18403 #[test]
18404 fn every_activity_timeout_class_is_typed() {
18405 for timeout_kind in [
18406 "start_to_close",
18407 "schedule_to_start",
18408 "schedule_to_close",
18409 "heartbeat",
18410 ] {
18411 let ctx = workflow_context(vec![history_event(
18412 "ActivityTimedOut",
18413 json!({
18414 "sequence": 1,
18415 "activity_type": "slow",
18416 "activity_execution_id": "act-timeout",
18417 "activity_attempt_id": "attempt-timeout",
18418 "failure_category": "timeout",
18419 "timeout_kind": timeout_kind,
18420 "message": "deadline expired"
18421 }),
18422 )]);
18423 let mut call = Box::pin(ctx.activity("slow", json!([])));
18424 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18425 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18426 call.as_mut().poll(&mut task_context)
18427 else {
18428 panic!("expected timeout failure");
18429 };
18430 assert_eq!(failure.kind, ActivityFailureKind::TimedOut);
18431 assert_eq!(failure.reason, timeout_kind);
18432 assert_eq!(failure.timeout_kind.as_deref(), Some(timeout_kind));
18433 assert_eq!(failure.failure_category.as_deref(), Some("timeout"));
18434 }
18435 }
18436
18437 #[test]
18438 fn workflow_sleep_emits_one_durable_timer_and_rounds_up() {
18439 let ctx = workflow_context(Vec::new());
18440 let mut sleep = Box::pin(ctx.sleep(Duration::from_millis(1_001)));
18441 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18442
18443 assert!(matches!(
18444 sleep.as_mut().poll(&mut task_context),
18445 Poll::Pending
18446 ));
18447 assert!(matches!(
18448 sleep.as_mut().poll(&mut task_context),
18449 Poll::Pending
18450 ));
18451
18452 let commands = ctx.take_commands().expect("timer command");
18453 assert_eq!(
18454 commands,
18455 vec![json!({
18456 "type": "start_timer",
18457 "delay_seconds": 2,
18458 })]
18459 );
18460 }
18461
18462 #[test]
18463 fn workflow_sleep_replays_matching_schedule_and_fire_without_a_command() {
18464 let history = vec![
18465 history_event(
18466 "TimerScheduled",
18467 json!({
18468 "sequence": 1,
18469 "timer_id": "timer-1",
18470 "delay_seconds": 5,
18471 "fire_at": "2026-07-11T12:00:05Z",
18472 }),
18473 ),
18474 history_event(
18475 "TimerFired",
18476 json!({
18477 "sequence": 1,
18478 "timer_id": "timer-1",
18479 "delay_seconds": 5,
18480 "fire_at": "2026-07-11T12:00:05Z",
18481 "fired_at": "2026-07-11T12:00:05Z",
18482 }),
18483 ),
18484 ];
18485
18486 for _restart in 0..2 {
18487 let ctx = workflow_context(history.clone());
18488 let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(5)));
18489 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18490 assert!(matches!(
18491 sleep.as_mut().poll(&mut task_context),
18492 Poll::Ready(Ok(()))
18493 ));
18494 assert!(ctx.take_commands().expect("commands").is_empty());
18495 ctx.ensure_history_consumed().expect("history consumed");
18496 }
18497 }
18498
18499 #[test]
18500 fn workflow_sleep_rejects_changed_delay_during_replay() {
18501 let ctx = workflow_context(vec![
18502 history_event(
18503 "TimerScheduled",
18504 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
18505 ),
18506 history_event(
18507 "TimerFired",
18508 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
18509 ),
18510 ]);
18511 let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(500)));
18512 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18513
18514 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18515 sleep.as_mut().poll(&mut task_context)
18516 else {
18517 panic!("changed timer delay must be rejected");
18518 };
18519 assert_eq!(failure.reason, "timer_delay_mismatch");
18520 assert_eq!(failure.sequence, Some(1));
18521 }
18522
18523 #[test]
18524 fn workflow_condition_wait_emits_published_identity_and_timeout_contract() {
18525 let ctx = workflow_context(Vec::new());
18526 let mut wait = Box::pin(
18527 ctx.wait_condition(
18528 ConditionWaitOptions::new("approval.ready", "sha256:approval-v1")
18529 .timeout(Duration::from_millis(60_001)),
18530 || Ok(false),
18531 ),
18532 );
18533 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18534
18535 assert!(matches!(
18536 wait.as_mut().poll(&mut task_context),
18537 Poll::Pending
18538 ));
18539 assert!(matches!(
18540 wait.as_mut().poll(&mut task_context),
18541 Poll::Pending
18542 ));
18543 assert_eq!(
18544 ctx.take_commands().expect("condition command"),
18545 vec![json!({
18546 "type": "open_condition_wait",
18547 "condition_wait_occurrence_id": "rust:condition-wait:0",
18548 "condition_key": "approval.ready",
18549 "condition_definition_fingerprint": "sha256:approval-v1",
18550 "timeout_seconds": 61,
18551 })]
18552 );
18553 }
18554
18555 #[test]
18556 fn workflow_condition_wait_returns_explicit_immediate_results_without_commands() {
18557 let ctx = workflow_context(Vec::new());
18558 let mut satisfied = Box::pin(wait_condition!(ctx, "already-ready", || Ok(true)));
18559 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18560 assert!(matches!(
18561 satisfied.as_mut().poll(&mut task_context),
18562 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18563 ));
18564
18565 let mut timed_out = Box::pin(wait_condition!(
18566 ctx,
18567 "no-wait",
18568 timeout: Duration::ZERO,
18569 || Ok(false),
18570 ));
18571 assert!(matches!(
18572 timed_out.as_mut().poll(&mut task_context),
18573 Poll::Ready(Ok(ConditionWaitResult::TimedOut))
18574 ));
18575 assert!(ctx.take_commands().expect("commands").is_empty());
18576 }
18577
18578 #[test]
18579 fn signal_and_update_history_reevaluate_open_conditions_after_restart() {
18580 let signal_history = vec![
18581 history_event(
18582 "ConditionWaitOpened",
18583 json!({
18584 "sequence": 4,
18585 "condition_wait_id": "condition:4",
18586 "condition_wait_occurrence_id": "rust:condition-wait:0",
18587 "condition_key": "approval",
18588 "condition_definition_fingerprint": "sha256:approval-v1",
18589 "timeout_seconds": 30,
18590 }),
18591 ),
18592 history_event(
18593 "SignalReceived",
18594 json!({
18595 "workflow_sequence": 4,
18596 "signal_name": "approve",
18597 "arguments": fixture_envelope(json!(["Ada"])),
18598 }),
18599 ),
18600 ];
18601 for _worker_before_or_after_restart in 0..2 {
18602 let ctx = workflow_context(signal_history.clone());
18603 let predicate_ctx = ctx.clone();
18604 let mut wait = Box::pin(
18605 ctx.wait_condition(
18606 ConditionWaitOptions::new("approval", "sha256:approval-v1")
18607 .timeout(Duration::from_secs(30)),
18608 move || Ok(!predicate_ctx.signals("approve")?.is_empty()),
18609 ),
18610 );
18611 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18612 assert!(matches!(
18613 wait.as_mut().poll(&mut task_context),
18614 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18615 ));
18616 assert!(ctx.take_commands().expect("commands").is_empty());
18617 ctx.ensure_history_consumed().expect("condition consumed");
18618 }
18619
18620 let update_history = vec![
18621 history_event(
18622 "ConditionWaitOpened",
18623 json!({
18624 "sequence": 7,
18625 "condition_wait_id": "condition:7",
18626 "condition_wait_occurrence_id": "rust:condition-wait:0",
18627 "condition_key": "update-approval",
18628 "condition_definition_fingerprint": "sha256:update-approval-v1",
18629 }),
18630 ),
18631 history_event(
18632 "UpdateApplied",
18633 json!({
18634 "sequence": 7,
18635 "update_id": "update-1",
18636 "update_name": "approve",
18637 "arguments": fixture_envelope(json!([true])),
18638 }),
18639 ),
18640 ];
18641 let ctx = workflow_context(update_history);
18642 let predicate_ctx = ctx.clone();
18643 let mut wait = Box::pin(ctx.wait_condition(
18644 ConditionWaitOptions::new("update-approval", "sha256:update-approval-v1"),
18645 move || {
18646 Ok(predicate_ctx
18647 .updates("approve")?
18648 .first()
18649 .and_then(|arguments| arguments.first())
18650 .and_then(Value::as_bool)
18651 == Some(true))
18652 },
18653 ));
18654 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18655 assert!(matches!(
18656 wait.as_mut().poll(&mut task_context),
18657 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18658 ));
18659 assert!(ctx.take_commands().expect("commands").is_empty());
18660 ctx.ensure_history_consumed().expect("condition consumed");
18661 }
18662
18663 #[test]
18664 fn condition_wait_preserves_open_satisfied_and_timed_out_replay_states() {
18665 let open_history = vec![
18666 history_event(
18667 "ConditionWaitOpened",
18668 json!({
18669 "sequence": 3,
18670 "condition_wait_id": "condition:3",
18671 "condition_wait_occurrence_id": "rust:condition-wait:0",
18672 "condition_key": "two-votes",
18673 "condition_definition_fingerprint": "sha256:two-votes-v1",
18674 "timeout_seconds": 120,
18675 }),
18676 ),
18677 history_event(
18678 "SignalReceived",
18679 json!({
18680 "workflow_sequence": 3,
18681 "signal_name": "vote",
18682 "arguments": fixture_envelope(json!(["first"])),
18683 }),
18684 ),
18685 ];
18686 for _worker_before_or_after_restart in 0..2 {
18687 let ctx = workflow_context(open_history.clone());
18688 let predicate_ctx = ctx.clone();
18689 let mut wait = Box::pin(
18690 ctx.wait_condition(
18691 ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1")
18692 .timeout(Duration::from_secs(120)),
18693 move || Ok(predicate_ctx.signals("vote")?.len() >= 2),
18694 ),
18695 );
18696 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18697 assert!(matches!(
18698 wait.as_mut().poll(&mut task_context),
18699 Poll::Pending
18700 ));
18701 assert_eq!(
18702 ctx.take_commands().expect("reopened condition"),
18703 vec![json!({
18704 "type": "open_condition_wait",
18705 "condition_wait_occurrence_id": "rust:condition-wait:0",
18706 "condition_key": "two-votes",
18707 "condition_definition_fingerprint": "sha256:two-votes-v1",
18708 "timeout_seconds": 120,
18709 })]
18710 );
18711 }
18712
18713 let satisfied_ctx = workflow_context(vec![
18714 history_event(
18715 "ConditionWaitOpened",
18716 json!({
18717 "sequence": 5,
18718 "condition_wait_id": "condition:5",
18719 "condition_wait_occurrence_id": "rust:condition-wait:0",
18720 "condition_key": "approval",
18721 "condition_definition_fingerprint": "sha256:approval-v1",
18722 }),
18723 ),
18724 history_event(
18725 "ConditionWaitSatisfied",
18726 json!({
18727 "sequence": 5,
18728 "condition_wait_id": "condition:5",
18729 "condition_wait_occurrence_id": "rust:condition-wait:0",
18730 "condition_key": "approval",
18731 "condition_definition_fingerprint": "sha256:approval-v1",
18732 }),
18733 ),
18734 ]);
18735 let mut satisfied = Box::pin(satisfied_ctx.wait_condition(
18736 ConditionWaitOptions::new("approval", "sha256:approval-v1"),
18737 || Ok(false),
18738 ));
18739 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18740 assert!(matches!(
18741 satisfied.as_mut().poll(&mut task_context),
18742 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18743 ));
18744
18745 let timed_out_ctx = workflow_context(vec![
18746 history_event(
18747 "ConditionWaitOpened",
18748 json!({
18749 "sequence": 8,
18750 "condition_wait_id": "condition:8",
18751 "condition_wait_occurrence_id": "rust:condition-wait:0",
18752 "condition_key": "approval-timeout",
18753 "condition_definition_fingerprint": "sha256:approval-timeout-v1",
18754 "timeout_seconds": 5,
18755 }),
18756 ),
18757 history_event(
18758 "TimerScheduled",
18759 json!({
18760 "sequence": 9,
18761 "timer_id": "condition-timer:9",
18762 "timer_kind": "condition_timeout",
18763 "condition_wait_id": "condition:8",
18764 "delay_seconds": 5,
18765 }),
18766 ),
18767 history_event(
18768 "TimerFired",
18769 json!({
18770 "sequence": 9,
18771 "timer_id": "condition-timer:9",
18772 "timer_kind": "condition_timeout",
18773 "condition_wait_id": "condition:8",
18774 "delay_seconds": 5,
18775 }),
18776 ),
18777 ]);
18778 let mut timed_out = Box::pin(
18779 timed_out_ctx.wait_condition(
18780 ConditionWaitOptions::new("approval-timeout", "sha256:approval-timeout-v1")
18781 .timeout(Duration::from_secs(5)),
18782 || Ok(true),
18783 ),
18784 );
18785 assert!(matches!(
18786 timed_out.as_mut().poll(&mut task_context),
18787 Poll::Ready(Ok(ConditionWaitResult::TimedOut))
18788 ));
18789 }
18790
18791 #[test]
18792 fn condition_wait_replays_repeated_physical_opens_as_one_logical_wait() {
18793 let history = vec![
18794 history_event(
18795 "ConditionWaitOpened",
18796 json!({
18797 "sequence": 3,
18798 "condition_wait_id": "condition:3",
18799 "condition_wait_occurrence_id": "rust:condition-wait:0",
18800 "condition_key": "two-votes",
18801 "condition_definition_fingerprint": "sha256:two-votes-v1",
18802 }),
18803 ),
18804 history_event(
18805 "SignalReceived",
18806 json!({
18807 "workflow_sequence": 3,
18808 "signal_name": "vote",
18809 "arguments": fixture_envelope(json!(["first"])),
18810 }),
18811 ),
18812 history_event(
18813 "ConditionWaitSatisfied",
18814 json!({
18815 "sequence": 3,
18816 "condition_wait_id": "condition:3",
18817 "condition_wait_occurrence_id": "rust:condition-wait:0",
18818 "condition_key": "two-votes",
18819 "condition_definition_fingerprint": "sha256:two-votes-v1",
18820 }),
18821 ),
18822 history_event(
18823 "ConditionWaitOpened",
18824 json!({
18825 "sequence": 5,
18826 "condition_wait_id": "condition:5",
18827 "condition_wait_occurrence_id": "rust:condition-wait:0",
18828 "condition_key": "two-votes",
18829 "condition_definition_fingerprint": "sha256:two-votes-v1",
18830 }),
18831 ),
18832 history_event(
18833 "SignalReceived",
18834 json!({
18835 "workflow_sequence": 5,
18836 "signal_name": "vote",
18837 "arguments": fixture_envelope(json!(["second"])),
18838 }),
18839 ),
18840 history_event(
18841 "ConditionWaitSatisfied",
18842 json!({
18843 "sequence": 5,
18844 "condition_wait_id": "condition:5",
18845 "condition_wait_occurrence_id": "rust:condition-wait:0",
18846 "condition_key": "two-votes",
18847 "condition_definition_fingerprint": "sha256:two-votes-v1",
18848 }),
18849 ),
18850 ];
18851 for _cold_worker_or_restart in 0..2 {
18852 let ctx = workflow_context(history.clone());
18853 let predicate_ctx = ctx.clone();
18854 let mut wait = Box::pin(ctx.wait_condition(
18855 ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1"),
18856 move || Ok(predicate_ctx.signals("vote")?.len() >= 2),
18857 ));
18858 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18859
18860 assert!(matches!(
18861 wait.as_mut().poll(&mut task_context),
18862 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18863 ));
18864 assert!(ctx.take_commands().expect("commands").is_empty());
18865 ctx.ensure_history_consumed()
18866 .expect("every physical wait-open is consumed");
18867 }
18868 }
18869
18870 #[test]
18871 fn condition_wait_replays_update_driven_physical_opens_as_one_occurrence() {
18872 let history = vec![
18873 history_event(
18874 "ConditionWaitOpened",
18875 json!({
18876 "sequence": 3,
18877 "condition_wait_id": "condition:3",
18878 "condition_wait_occurrence_id": "rust:condition-wait:0",
18879 "condition_key": "approved",
18880 "condition_definition_fingerprint": "sha256:approved-v1",
18881 }),
18882 ),
18883 history_event(
18884 "UpdateApplied",
18885 json!({
18886 "sequence": 3,
18887 "update_id": "update-1",
18888 "update_name": "approve",
18889 "arguments": fixture_envelope(json!([false])),
18890 }),
18891 ),
18892 history_event(
18893 "ConditionWaitOpened",
18894 json!({
18895 "sequence": 5,
18896 "condition_wait_id": "condition:5",
18897 "condition_wait_occurrence_id": "rust:condition-wait:0",
18898 "condition_key": "approved",
18899 "condition_definition_fingerprint": "sha256:approved-v1",
18900 }),
18901 ),
18902 history_event(
18903 "UpdateApplied",
18904 json!({
18905 "sequence": 5,
18906 "update_id": "update-2",
18907 "update_name": "approve",
18908 "arguments": fixture_envelope(json!([true])),
18909 }),
18910 ),
18911 ];
18912
18913 for _cold_worker_or_restart in 0..2 {
18914 let ctx = workflow_context(history.clone());
18915 let predicate_ctx = ctx.clone();
18916 let mut wait = Box::pin(ctx.wait_condition(
18917 ConditionWaitOptions::new("approved", "sha256:approved-v1"),
18918 move || {
18919 Ok(predicate_ctx
18920 .updates("approve")?
18921 .last()
18922 .and_then(|arguments| arguments.first())
18923 .and_then(Value::as_bool)
18924 == Some(true))
18925 },
18926 ));
18927 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18928
18929 assert!(matches!(
18930 wait.as_mut().poll(&mut task_context),
18931 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18932 ));
18933 assert!(ctx.take_commands().expect("commands").is_empty());
18934 ctx.ensure_history_consumed()
18935 .expect("every update-driven reopen is consumed");
18936 }
18937 }
18938
18939 #[test]
18940 fn condition_wait_replay_keeps_every_adjacent_authored_occurrence_distinct() {
18941 for (first_key, first_fingerprint, second_key, second_fingerprint) in [
18942 ("shared", "sha256:first", "shared", "sha256:second"),
18943 ("first", "sha256:shared", "second", "sha256:shared"),
18944 ("shared", "sha256:shared", "shared", "sha256:shared"),
18945 ("first", "sha256:first", "second", "sha256:second"),
18946 ] {
18947 let history = vec![
18948 history_event(
18949 "ConditionWaitOpened",
18950 json!({
18951 "sequence": 3,
18952 "condition_wait_id": "condition:3",
18953 "condition_wait_occurrence_id": "rust:condition-wait:0",
18954 "condition_key": first_key,
18955 "condition_definition_fingerprint": first_fingerprint,
18956 }),
18957 ),
18958 history_event(
18959 "ConditionWaitSatisfied",
18960 json!({
18961 "sequence": 3,
18962 "condition_wait_id": "condition:3",
18963 "condition_wait_occurrence_id": "rust:condition-wait:0",
18964 "condition_key": first_key,
18965 "condition_definition_fingerprint": first_fingerprint,
18966 }),
18967 ),
18968 history_event(
18969 "ConditionWaitOpened",
18970 json!({
18971 "sequence": 4,
18972 "condition_wait_id": "condition:4",
18973 "condition_wait_occurrence_id": "rust:condition-wait:1",
18974 "condition_key": second_key,
18975 "condition_definition_fingerprint": second_fingerprint,
18976 }),
18977 ),
18978 history_event(
18979 "ConditionWaitSatisfied",
18980 json!({
18981 "sequence": 4,
18982 "condition_wait_id": "condition:4",
18983 "condition_wait_occurrence_id": "rust:condition-wait:1",
18984 "condition_key": second_key,
18985 "condition_definition_fingerprint": second_fingerprint,
18986 }),
18987 ),
18988 ];
18989 for _cold_worker_or_restart in 0..2 {
18990 let ctx = workflow_context(history.clone());
18991 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18992 let mut first = Box::pin(ctx.wait_condition(
18993 ConditionWaitOptions::new(first_key, first_fingerprint),
18994 || Ok(false),
18995 ));
18996 assert!(matches!(
18997 first.as_mut().poll(&mut task_context),
18998 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18999 ));
19000
19001 let mut second = Box::pin(ctx.wait_condition(
19002 ConditionWaitOptions::new(second_key, second_fingerprint),
19003 || Ok(false),
19004 ));
19005 assert!(matches!(
19006 second.as_mut().poll(&mut task_context),
19007 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19008 ));
19009 assert!(ctx.take_commands().expect("commands").is_empty());
19010 ctx.ensure_history_consumed()
19011 .expect("each authored wait consumes one occurrence");
19012 }
19013 }
19014 }
19015
19016 #[test]
19017 fn cold_workers_replay_adjacent_condition_waits_from_one_loop_call_site() {
19018 fn worker() -> Worker {
19019 let client = Client::new("http://127.0.0.1:8080").expect("client");
19020 let mut worker = Worker::new(client, "rust-workers");
19021 worker.register_workflow("rust.condition-loop", |ctx, _input| async move {
19022 let mut outcomes = Vec::new();
19023 for _ in 0..2 {
19024 outcomes.push(
19025 ctx.wait_condition(
19026 ConditionWaitOptions::new("shared", "sha256:shared"),
19027 || Ok(false),
19028 )
19029 .await?,
19030 );
19031 }
19032 Ok(json!(outcomes))
19033 });
19034 worker
19035 }
19036
19037 let task = workflow_task(
19038 "rust.condition-loop",
19039 vec![
19040 history_event(
19041 "ConditionWaitOpened",
19042 json!({
19043 "sequence": 1,
19044 "condition_wait_id": "condition:1",
19045 "condition_wait_occurrence_id": "rust:condition-wait:0",
19046 "condition_key": "shared",
19047 "condition_definition_fingerprint": "sha256:shared",
19048 }),
19049 ),
19050 history_event(
19051 "ConditionWaitSatisfied",
19052 json!({
19053 "sequence": 1,
19054 "condition_wait_id": "condition:1",
19055 "condition_wait_occurrence_id": "rust:condition-wait:0",
19056 "condition_key": "shared",
19057 "condition_definition_fingerprint": "sha256:shared",
19058 }),
19059 ),
19060 history_event(
19061 "ConditionWaitOpened",
19062 json!({
19063 "sequence": 2,
19064 "condition_wait_id": "condition:2",
19065 "condition_wait_occurrence_id": "rust:condition-wait:1",
19066 "condition_key": "shared",
19067 "condition_definition_fingerprint": "sha256:shared",
19068 }),
19069 ),
19070 history_event(
19071 "ConditionWaitSatisfied",
19072 json!({
19073 "sequence": 2,
19074 "condition_wait_id": "condition:2",
19075 "condition_wait_occurrence_id": "rust:condition-wait:1",
19076 "condition_key": "shared",
19077 "condition_definition_fingerprint": "sha256:shared",
19078 }),
19079 ),
19080 ],
19081 DEFAULT_CODEC,
19082 );
19083
19084 for _cold_worker_or_restart in 0..2 {
19085 let commands = worker()
19086 .execute_workflow_task(task.clone())
19087 .expect("adjacent loop waits replay deterministically");
19088 assert_eq!(commands.len(), 1);
19089 assert_eq!(commands[0]["type"], "complete_workflow");
19090 assert_eq!(
19091 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
19092 json!(["satisfied", "satisfied"])
19093 );
19094 }
19095 }
19096
19097 #[test]
19098 fn condition_wait_replay_rejects_identity_predicate_and_timeout_changes() {
19099 let history = vec![history_event(
19100 "ConditionWaitOpened",
19101 json!({
19102 "sequence": 12,
19103 "condition_wait_id": "condition:12",
19104 "condition_wait_occurrence_id": "rust:condition-wait:0",
19105 "condition_key": "approval",
19106 "condition_definition_fingerprint": "sha256:approval-v1",
19107 "timeout_seconds": 30,
19108 }),
19109 )];
19110 for (options, expected_reason) in [
19111 (
19112 ConditionWaitOptions::new("changed", "sha256:approval-v1")
19113 .timeout(Duration::from_secs(30)),
19114 "condition_wait_key_mismatch",
19115 ),
19116 (
19117 ConditionWaitOptions::new("approval", "sha256:approval-v2")
19118 .timeout(Duration::from_secs(30)),
19119 "condition_wait_predicate_mismatch",
19120 ),
19121 (
19122 ConditionWaitOptions::new("approval", "sha256:approval-v1")
19123 .timeout(Duration::from_secs(29)),
19124 "condition_wait_timeout_mismatch",
19125 ),
19126 ] {
19127 let ctx = workflow_context(history.clone());
19128 let mut wait = Box::pin(ctx.wait_condition(options, || Ok(false)));
19129 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19130 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
19131 wait.as_mut().poll(&mut task_context)
19132 else {
19133 panic!("changed condition definition must fail replay");
19134 };
19135 assert_eq!(failure.reason, expected_reason);
19136 assert_eq!(failure.sequence, Some(12));
19137 }
19138 }
19139
19140 #[test]
19141 fn condition_wait_history_requires_the_canonical_predicate_fingerprint() {
19142 let error = WorkflowState::new(
19143 vec![history_event(
19144 "ConditionWaitOpened",
19145 json!({
19146 "sequence": 12,
19147 "condition_wait_id": "condition:12",
19148 "condition_wait_occurrence_id": "rust:condition-wait:0",
19149 "condition_key": "approval",
19150 }),
19151 )],
19152 "rust-workers".to_string(),
19153 DEFAULT_CODEC.to_string(),
19154 None,
19155 )
19156 .expect_err("condition history without a predicate fingerprint must fail");
19157
19158 assert!(matches!(
19159 error,
19160 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19161 if reason == "condition_wait_predicate_fingerprint_missing"
19162 ));
19163 }
19164
19165 #[test]
19166 fn condition_wait_history_requires_authored_occurrence_identity() {
19167 let error = WorkflowState::new(
19168 vec![history_event(
19169 "ConditionWaitOpened",
19170 json!({
19171 "sequence": 12,
19172 "condition_wait_id": "condition:12",
19173 "condition_key": "approval",
19174 "condition_definition_fingerprint": "sha256:approval-v1",
19175 }),
19176 )],
19177 "rust-workers".to_string(),
19178 DEFAULT_CODEC.to_string(),
19179 None,
19180 )
19181 .expect_err("condition history without occurrence identity must fail");
19182
19183 assert!(matches!(
19184 error,
19185 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19186 if reason == "condition_wait_occurrence_id_missing"
19187 ));
19188 }
19189
19190 #[test]
19191 fn typed_search_attribute_updates_validate_emit_and_replay() {
19192 let update = SearchAttributeUpdate::new()
19193 .keyword("OrderStatus", " waiting ")
19194 .expect("keyword")
19195 .int("Attempt", 3)
19196 .expect("int")
19197 .bool("Escalated", false)
19198 .expect("bool")
19199 .keyword_list("Regions", ["us-east", "eu-west"])
19200 .expect("list")
19201 .datetime("UpdatedAt", "2026-08-22T04:00:00Z")
19202 .expect("datetime")
19203 .delete("LegacyStatus")
19204 .expect("delete");
19205 let ctx = workflow_context(Vec::new());
19206 ctx.upsert_search_attributes(update.clone())
19207 .expect("typed update");
19208 assert_eq!(
19209 ctx.take_commands().expect("search-attribute command"),
19210 vec![json!({
19211 "type": "upsert_search_attributes",
19212 "attributes": {
19213 "Attempt": 3,
19214 "Escalated": false,
19215 "LegacyStatus": null,
19216 "OrderStatus": "waiting",
19217 "Regions": ["us-east", "eu-west"],
19218 "UpdatedAt": "2026-08-22T04:00:00Z",
19219 },
19220 "attribute_types": {
19221 "Attempt": "int",
19222 "Escalated": "bool",
19223 "OrderStatus": "keyword",
19224 "Regions": "keyword_list",
19225 "UpdatedAt": "datetime",
19226 },
19227 })]
19228 );
19229
19230 let replay = workflow_context(vec![history_event(
19231 "SearchAttributesUpserted",
19232 json!({
19233 "sequence": 6,
19234 "attributes": {
19235 "Attempt": 3,
19236 "Escalated": false,
19237 "LegacyStatus": null,
19238 "OrderStatus": "waiting",
19239 "Regions": ["us-east", "eu-west"],
19240 "UpdatedAt": "2026-08-22T04:00:00Z",
19241 },
19242 "attribute_types": {
19243 "Attempt": "int",
19244 "Escalated": "bool",
19245 "OrderStatus": "keyword",
19246 "Regions": "keyword_list",
19247 "UpdatedAt": "datetime",
19248 },
19249 "merged": {},
19250 }),
19251 )]);
19252 replay
19253 .upsert_search_attributes(update)
19254 .expect("matching update replays");
19255 assert!(replay.take_commands().expect("commands").is_empty());
19256 replay.ensure_history_consumed().expect("history consumed");
19257
19258 let type_drift = workflow_context(vec![history_event(
19259 "SearchAttributesUpserted",
19260 json!({
19261 "sequence": 7,
19262 "attributes": {"OrderStatus": "waiting"},
19263 "attribute_types": {"OrderStatus": "keyword"},
19264 "merged": {"OrderStatus": "waiting"},
19265 }),
19266 )]);
19267 let error = type_drift
19268 .upsert_search_attributes(
19269 SearchAttributeUpdate::new()
19270 .string("OrderStatus", "waiting")
19271 .expect("string update"),
19272 )
19273 .expect_err("same JSON value with a changed type must fail replay");
19274 assert!(matches!(
19275 error,
19276 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19277 if reason == "search_attribute_type_mismatch"
19278 ));
19279
19280 let malformed_types = WorkflowState::new(
19281 vec![history_event(
19282 "SearchAttributesUpserted",
19283 json!({
19284 "sequence": 8,
19285 "attributes": {"OrderStatus": "waiting"},
19286 "attribute_types": {"OrderStatus": "unsupported"},
19287 "merged": {"OrderStatus": "waiting"},
19288 }),
19289 )],
19290 "rust-workers".to_string(),
19291 DEFAULT_CODEC.to_string(),
19292 None,
19293 )
19294 .expect_err("unsupported search-attribute type metadata must fail");
19295 assert!(matches!(
19296 malformed_types,
19297 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19298 if reason == "search_attribute_types_malformed"
19299 ));
19300
19301 assert!(matches!(
19302 SearchAttributeUpdate::new().keyword("bad key", "value"),
19303 Err(SearchAttributeUpdateError::InvalidKey(_))
19304 ));
19305 assert!(matches!(
19306 SearchAttributeUpdate::new().float("Ratio", f64::NAN),
19307 Err(SearchAttributeUpdateError::NonFiniteFloat(_))
19308 ));
19309 assert!(matches!(
19310 SearchAttributeUpdate::new().keyword("UnicodeKeyword", "é".repeat(128)),
19311 Err(SearchAttributeUpdateError::ValueTooLong { .. })
19312 ));
19313 assert!(matches!(
19314 SearchAttributeUpdate::new().datetime("UpdatedAt", "2026-02-30T04:00:00Z"),
19315 Err(SearchAttributeUpdateError::InvalidDateTime(_))
19316 ));
19317 assert!(matches!(
19318 workflow_context(Vec::new()).upsert_search_attributes(SearchAttributeUpdate::new()),
19319 Err(Error::InvalidSearchAttributeUpdate(
19320 SearchAttributeUpdateError::Empty
19321 ))
19322 ));
19323 }
19324
19325 #[test]
19326 fn typed_search_attribute_text_uses_the_runtime_byte_limit() {
19327 let ascii = "a".repeat(MAX_SEARCH_ATTRIBUTE_STRING_LENGTH);
19328 let utf8 = "é".repeat(MAX_SEARCH_ATTRIBUTE_STRING_LENGTH / 2);
19329
19330 assert!(SearchAttributeUpdate::new()
19331 .string("AsciiDescription", ascii)
19332 .is_ok());
19333 assert!(SearchAttributeUpdate::new()
19334 .string("Utf8Description", utf8)
19335 .is_ok());
19336 assert!(matches!(
19337 SearchAttributeUpdate::new().string(
19338 "TooLongDescription",
19339 "é".repeat((MAX_SEARCH_ATTRIBUTE_STRING_LENGTH / 2) + 1),
19340 ),
19341 Err(SearchAttributeUpdateError::ValueTooLong {
19342 kind: "string",
19343 limit: MAX_SEARCH_ATTRIBUTE_STRING_LENGTH,
19344 ..
19345 })
19346 ));
19347 }
19348
19349 #[test]
19350 fn workflow_history_rejects_unpaired_or_mismatched_timer_events() {
19351 let lone_fire = WorkflowState::new(
19352 vec![history_event(
19353 "TimerFired",
19354 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19355 )],
19356 "rust-workers".to_string(),
19357 DEFAULT_CODEC.to_string(),
19358 None,
19359 )
19360 .expect_err("TimerFired requires TimerScheduled");
19361 assert!(matches!(
19362 lone_fire,
19363 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19364 if reason == "timer_schedule_missing_or_duplicate"
19365 ));
19366
19367 let wrong_identity = WorkflowState::new(
19368 vec![
19369 history_event(
19370 "TimerScheduled",
19371 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19372 ),
19373 history_event(
19374 "TimerFired",
19375 json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
19376 ),
19377 ],
19378 "rust-workers".to_string(),
19379 DEFAULT_CODEC.to_string(),
19380 None,
19381 )
19382 .expect_err("fire must match scheduled timer identity");
19383 assert!(matches!(
19384 wrong_identity,
19385 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19386 if reason == "timer_identity_mismatch"
19387 ));
19388
19389 let duplicate_fire = WorkflowState::new(
19390 vec![
19391 history_event(
19392 "TimerScheduled",
19393 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19394 ),
19395 history_event(
19396 "TimerFired",
19397 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19398 ),
19399 history_event(
19400 "TimerFired",
19401 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19402 ),
19403 ],
19404 "rust-workers".to_string(),
19405 DEFAULT_CODEC.to_string(),
19406 None,
19407 )
19408 .expect_err("a durable timer cannot fire twice");
19409 assert!(matches!(
19410 duplicate_fire,
19411 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19412 if reason == "duplicate_timer_fire"
19413 ));
19414
19415 let wrong_fired_delay = WorkflowState::new(
19416 vec![
19417 history_event(
19418 "TimerScheduled",
19419 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19420 ),
19421 history_event(
19422 "TimerFired",
19423 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 6}),
19424 ),
19425 ],
19426 "rust-workers".to_string(),
19427 DEFAULT_CODEC.to_string(),
19428 None,
19429 )
19430 .expect_err("timer schedule and fire delays must agree");
19431 assert!(matches!(
19432 wrong_fired_delay,
19433 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19434 if reason == "timer_history_delay_mismatch"
19435 ));
19436 }
19437
19438 #[test]
19439 fn replay_rejects_activity_moved_before_recorded_timer() {
19440 let ctx = workflow_context(vec![
19441 history_event(
19442 "TimerScheduled",
19443 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19444 ),
19445 history_event(
19446 "TimerFired",
19447 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19448 ),
19449 history_event(
19450 "ActivityCompleted",
19451 json!({
19452 "sequence": 2,
19453 "activity_type": "after-timer",
19454 "payload_codec": DEFAULT_CODEC,
19455 "result": fixture_envelope(json!("done")),
19456 }),
19457 ),
19458 ]);
19459 let mut activity = Box::pin(ctx.activity("after-timer", json!([])));
19460 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19461
19462 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
19463 activity.as_mut().poll(&mut task_context)
19464 else {
19465 panic!("reordered durable command must be rejected");
19466 };
19467 assert_eq!(failure.reason, "recorded_command_mismatch");
19468 assert_eq!(failure.sequence, Some(1));
19469 assert_eq!(failure.expected.as_deref(), Some("timer"));
19470 assert_eq!(failure.actual.as_deref(), Some("activity:after-timer"));
19471 }
19472
19473 #[test]
19474 fn workflow_context_emits_a_typed_named_signal_wait() {
19475 let ctx = workflow_context(Vec::new());
19476 let mut signal = Box::pin(ctx.wait_signal("finish"));
19477 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19478
19479 assert!(matches!(
19480 signal.as_mut().poll(&mut task_context),
19481 Poll::Pending
19482 ));
19483 assert_eq!(
19484 ctx.take_commands().expect("signal-wait command"),
19485 vec![json!({
19486 "type": "open_signal_wait",
19487 "signal_name": "finish",
19488 })]
19489 );
19490 }
19491
19492 #[test]
19493 fn runtime_message_stream_transport_cannot_be_opened_as_a_user_signal() {
19494 let ctx = workflow_context(Vec::new());
19495 let mut signal = Box::pin(ctx.wait_signal(MESSAGE_STREAM_SIGNAL));
19496 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19497
19498 let Poll::Ready(Err(Error::Codec(message))) = signal.as_mut().poll(&mut task_context)
19499 else {
19500 panic!("runtime-reserved signal should be rejected");
19501 };
19502 assert!(message.contains("reserved by the workflow runtime"));
19503 assert!(ctx.take_commands().expect("commands").is_empty());
19504 }
19505
19506 #[tokio::test]
19507 async fn runtime_message_stream_transport_cannot_be_sent_as_a_user_signal() {
19508 let client = Client::builder("http://127.0.0.1:9")
19509 .build()
19510 .expect("client");
19511 let error = client
19512 .signal_workflow("workflow-1", MESSAGE_STREAM_SIGNAL, json!(["forged"]))
19513 .await
19514 .expect_err("runtime-reserved signal should be rejected before transport");
19515
19516 assert!(
19517 matches!(error, Error::Codec(ref message) if message.contains("reserved by the workflow runtime"))
19518 );
19519 }
19520
19521 #[test]
19522 fn message_stream_worker_task_consumes_current_contiguous_bounded_batch() {
19523 fn delivery(message_id: &str, position: u64, value: &str) -> Value {
19524 let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
19525 value.to_string(),
19526 )]))
19527 .expect("message payload");
19528 json!({
19529 "schema": MESSAGE_STREAM_SCHEMA,
19530 "stream_name": "orders",
19531 "message_id": message_id,
19532 "position": position,
19533 "payload_envelope": payload,
19534 })
19535 }
19536
19537 fn opened(sequence: u64) -> HistoryEvent {
19538 history_event(
19539 "SignalWaitOpened",
19540 json!({
19541 "sequence": sequence,
19542 "signal_name": MESSAGE_STREAM_SIGNAL,
19543 }),
19544 )
19545 }
19546
19547 fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
19548 history_event(
19549 "SignalApplied",
19550 json!({
19551 "sequence": sequence,
19552 "signal_name": MESSAGE_STREAM_SIGNAL,
19553 "value": fixture_envelope(json!([delivery])),
19554 }),
19555 )
19556 }
19557
19558 fn received(delivery: Value) -> HistoryEvent {
19559 history_event(
19560 "SignalReceived",
19561 json!({
19562 "signal_name": MESSAGE_STREAM_SIGNAL,
19563 "arguments": fixture_envelope(json!([delivery])),
19564 "payload_codec": DEFAULT_CODEC,
19565 }),
19566 )
19567 }
19568
19569 let client = Client::new("http://127.0.0.1:8080").expect("client");
19570 let mut worker = Worker::new(client, "rust-workers");
19571 worker.register_workflow("rust.message-stream-batch", |ctx, _input| async move {
19572 let messages = ctx.message_stream("orders")?.receive(2).await?;
19573 Ok(json!(messages
19574 .into_iter()
19575 .map(|message| message.message_id)
19576 .collect::<Vec<_>>()))
19577 });
19578
19579 let first = delivery("message-1", 1, "one");
19580 let second = delivery("message-2", 2, "two");
19581 let batch = worker
19582 .execute_workflow_task_decision(workflow_task(
19583 "rust.message-stream-batch",
19584 vec![
19585 opened(1),
19586 received(first.clone()),
19587 applied(1, first.clone()),
19588 received(first.clone()),
19589 received(second),
19590 ],
19591 DEFAULT_CODEC,
19592 ))
19593 .expect("worker task consumes the available batch");
19594
19595 assert_eq!(batch.commands.len(), 1);
19596 assert_eq!(batch.commands[0]["type"], "complete_workflow");
19597 assert_eq!(
19598 decode_wire_value(&batch.commands[0]["result"], DEFAULT_CODEC)
19599 .expect("workflow result"),
19600 json!(["message-1", "message-2"])
19601 );
19602 assert_eq!(
19603 batch.message_stream_cursors,
19604 vec![json!({"stream_name": "orders", "through_position": 2})]
19605 );
19606 assert!(batch.message_stream_waits.is_empty());
19607
19608 let partial = worker
19609 .execute_workflow_task_decision(workflow_task(
19610 "rust.message-stream-batch",
19611 vec![opened(1), received(first.clone()), applied(1, first)],
19612 DEFAULT_CODEC,
19613 ))
19614 .expect("worker task returns without waiting for a missing second item");
19615 assert_eq!(partial.commands.len(), 1);
19616 assert_eq!(partial.commands[0]["type"], "complete_workflow");
19617 assert_eq!(
19618 decode_wire_value(&partial.commands[0]["result"], DEFAULT_CODEC)
19619 .expect("workflow result"),
19620 json!(["message-1"])
19621 );
19622 assert_eq!(
19623 partial.message_stream_cursors,
19624 vec![json!({"stream_name": "orders", "through_position": 1})]
19625 );
19626 assert!(partial.message_stream_waits.is_empty());
19627 }
19628
19629 #[test]
19630 fn message_stream_replay_preserves_partial_batch_boundary_before_later_wait() {
19631 fn delivery(message_id: &str, position: u64, value: &str) -> Value {
19632 let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
19633 value.to_string(),
19634 )]))
19635 .expect("message payload");
19636 json!({
19637 "schema": MESSAGE_STREAM_SCHEMA,
19638 "stream_name": "orders",
19639 "message_id": message_id,
19640 "position": position,
19641 "payload_envelope": payload,
19642 })
19643 }
19644
19645 fn opened(sequence: u64) -> HistoryEvent {
19646 history_event(
19647 "SignalWaitOpened",
19648 json!({
19649 "sequence": sequence,
19650 "signal_name": MESSAGE_STREAM_SIGNAL,
19651 }),
19652 )
19653 }
19654
19655 fn received(delivery: Value) -> HistoryEvent {
19656 history_event(
19657 "SignalReceived",
19658 json!({
19659 "signal_name": MESSAGE_STREAM_SIGNAL,
19660 "arguments": fixture_envelope(json!([delivery])),
19661 "payload_codec": DEFAULT_CODEC,
19662 }),
19663 )
19664 }
19665
19666 fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
19667 history_event(
19668 "SignalApplied",
19669 json!({
19670 "sequence": sequence,
19671 "signal_name": MESSAGE_STREAM_SIGNAL,
19672 "value": fixture_envelope(json!([delivery])),
19673 }),
19674 )
19675 }
19676
19677 let client = Client::new("http://127.0.0.1:8080").expect("client");
19678 let mut worker = Worker::new(client, "rust-workers");
19679 worker.register_workflow(
19680 "rust.message-stream-partial-batches",
19681 |ctx, _input| async move {
19682 let stream = ctx.message_stream("orders")?;
19683 let first = stream.receive(10).await?;
19684 let second = stream.receive(10).await?;
19685 Ok(json!([
19686 first
19687 .into_iter()
19688 .map(|message| message.message_id)
19689 .collect::<Vec<_>>(),
19690 second
19691 .into_iter()
19692 .map(|message| message.message_id)
19693 .collect::<Vec<_>>(),
19694 ]))
19695 },
19696 );
19697
19698 let first = delivery("message-1", 1, "one");
19699 let second = delivery("message-2", 2, "two");
19700 let decision = worker
19701 .execute_workflow_task_decision(workflow_task(
19702 "rust.message-stream-partial-batches",
19703 vec![
19704 opened(1),
19705 received(first.clone()),
19706 applied(1, first),
19707 opened(2),
19708 received(second.clone()),
19709 applied(2, second),
19710 ],
19711 DEFAULT_CODEC,
19712 ))
19713 .expect("cold replay preserves both authored receive boundaries");
19714
19715 assert_eq!(decision.commands.len(), 1);
19716 assert_eq!(decision.commands[0]["type"], "complete_workflow");
19717 assert_eq!(
19718 decode_wire_value(&decision.commands[0]["result"], DEFAULT_CODEC)
19719 .expect("workflow result"),
19720 json!([["message-1"], ["message-2"]])
19721 );
19722 assert_eq!(
19723 decision.message_stream_cursors,
19724 vec![json!({"stream_name": "orders", "through_position": 2})]
19725 );
19726 assert!(decision.message_stream_waits.is_empty());
19727 }
19728
19729 #[test]
19730 fn empty_message_stream_opens_internal_signal_wait_and_reports_position() {
19731 let ctx = workflow_context(Vec::new());
19732 let stream = ctx.message_stream("orders").expect("message stream");
19733 let mut receive = Box::pin(stream.receive(10));
19734 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19735
19736 assert!(matches!(
19737 receive.as_mut().poll(&mut task_context),
19738 Poll::Pending
19739 ));
19740 assert_eq!(
19741 ctx.take_commands().expect("message-stream wait command"),
19742 vec![json!({
19743 "type": "open_signal_wait",
19744 "signal_name": MESSAGE_STREAM_SIGNAL,
19745 })]
19746 );
19747 let (cursors, waits) = ctx.message_stream_metadata().expect("stream metadata");
19748 assert!(cursors.is_empty());
19749 assert_eq!(
19750 waits,
19751 vec![json!({"stream_name": "orders", "after_position": 0})]
19752 );
19753 }
19754
19755 #[test]
19756 fn continue_as_new_cursor_checkpoint_preserves_global_pending_position() {
19757 let ctx = workflow_context(vec![history_event(
19758 "SignalReceived",
19759 json!({
19760 "signal_name": MESSAGE_STREAM_SIGNAL,
19761 "arguments": fixture_envelope(json!([{
19762 "schema": MESSAGE_STREAM_CURSOR_SCHEMA,
19763 "stream_name": "orders",
19764 "through_position": 2,
19765 }])),
19766 "payload_codec": DEFAULT_CODEC,
19767 }),
19768 )]);
19769 let stream = ctx.message_stream("orders").expect("message stream");
19770 let mut receive = Box::pin(stream.receive(10));
19771 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19772
19773 assert!(matches!(
19774 receive.as_mut().poll(&mut task_context),
19775 Poll::Pending
19776 ));
19777 let (cursors, waits) = ctx.message_stream_metadata().expect("stream metadata");
19778 assert_eq!(
19779 cursors,
19780 vec![json!({"stream_name": "orders", "through_position": 2})]
19781 );
19782 assert_eq!(
19783 waits,
19784 vec![json!({"stream_name": "orders", "after_position": 2})]
19785 );
19786 }
19787
19788 #[test]
19789 fn message_stream_delivery_preserves_typed_avro_arguments_across_replay() {
19790 let mut empty_map = BTreeMap::new();
19791 let mut nested = BTreeMap::new();
19792 nested.insert(
19793 "value".to_string(),
19794 AvroValue::Array(vec![AvroValue::Bytes(b"nested".to_vec())]),
19795 );
19796 let values = vec![
19797 AvroValue::Bytes(vec![0, 255]),
19798 AvroValue::Long(1),
19799 AvroValue::Double(1.0),
19800 AvroValue::Array(Vec::new()),
19801 AvroValue::Map(std::mem::take(&mut empty_map)),
19802 AvroValue::Map(nested),
19803 ];
19804 let payload = encode_avro_value(&AvroValue::Array(values.clone())).expect("payload");
19805 let transport = vec![json!({
19806 "schema": MESSAGE_STREAM_SCHEMA,
19807 "stream_name": "orders",
19808 "message_id": "message-1",
19809 "position": 1,
19810 "payload_envelope": payload,
19811 })];
19812
19813 for _ in 0..2 {
19814 let Some(MessageStreamDelivery::Message(message)) =
19815 decode_message_stream_delivery(transport.clone()).expect("delivery")
19816 else {
19817 panic!("message delivery expected");
19818 };
19819 assert_eq!(message.arguments, values);
19820 assert!(matches!(message.arguments[1], AvroValue::Long(1)));
19821 assert!(matches!(message.arguments[2], AvroValue::Double(1.0)));
19822 }
19823 }
19824
19825 #[test]
19826 fn cold_worker_replacement_consumes_message_stream_wait_arrivals_once_in_order() {
19827 fn delivery(message_id: &str, position: u64, value: &str) -> Value {
19828 let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
19829 value.to_string(),
19830 )]))
19831 .expect("message payload");
19832 json!({
19833 "schema": MESSAGE_STREAM_SCHEMA,
19834 "stream_name": "orders",
19835 "message_id": message_id,
19836 "position": position,
19837 "payload_envelope": payload,
19838 })
19839 }
19840
19841 fn opened(sequence: u64) -> HistoryEvent {
19842 history_event(
19843 "SignalWaitOpened",
19844 json!({
19845 "sequence": sequence,
19846 "signal_name": MESSAGE_STREAM_SIGNAL,
19847 }),
19848 )
19849 }
19850
19851 fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
19852 history_event(
19853 "SignalApplied",
19854 json!({
19855 "sequence": sequence,
19856 "signal_name": MESSAGE_STREAM_SIGNAL,
19857 "value": fixture_envelope(json!([delivery])),
19858 }),
19859 )
19860 }
19861
19862 fn worker() -> Worker {
19863 let client = Client::new("http://127.0.0.1:8080").expect("client");
19864 let mut worker = Worker::new(client, "rust-workers");
19865 worker.register_workflow("rust.message-stream", |ctx, _input| async move {
19866 let stream = ctx.message_stream("orders")?;
19867 let first = stream.receive_one().await?;
19868 let second = stream.receive_one().await?;
19869 Ok(json!([first.message_id, second.message_id]))
19870 });
19871 worker
19872 }
19873
19874 fn task_with_resume(history: Vec<HistoryEvent>, delivery: Value) -> WorkflowTask {
19875 let mut task = workflow_task("rust.message-stream", history, DEFAULT_CODEC);
19876 task.signal_name = Some(MESSAGE_STREAM_SIGNAL.to_string());
19877 task.signal_arguments = Some(fixture_envelope(json!([delivery])));
19878 task
19879 }
19880
19881 let waiting = worker()
19882 .execute_workflow_task_decision(workflow_task(
19883 "rust.message-stream",
19884 Vec::new(),
19885 DEFAULT_CODEC,
19886 ))
19887 .expect("first worker opens the stream wait");
19888 assert_eq!(
19889 waiting.commands,
19890 vec![json!({
19891 "type": "open_signal_wait",
19892 "signal_name": MESSAGE_STREAM_SIGNAL,
19893 })]
19894 );
19895 assert!(waiting.message_stream_cursors.is_empty());
19896 assert_eq!(
19897 waiting.message_stream_waits,
19898 vec![json!({"stream_name": "orders", "after_position": 0})]
19899 );
19900
19901 let first_delivery = delivery("message-1", 1, "one");
19902 let first_arrival = worker()
19903 .execute_workflow_task_decision(task_with_resume(
19904 vec![opened(1)],
19905 first_delivery.clone(),
19906 ))
19907 .expect("replacement worker consumes the first arrival");
19908 assert_eq!(
19909 first_arrival.commands,
19910 vec![json!({
19911 "type": "open_signal_wait",
19912 "signal_name": MESSAGE_STREAM_SIGNAL,
19913 })]
19914 );
19915 assert_eq!(
19916 first_arrival.message_stream_cursors,
19917 vec![json!({"stream_name": "orders", "through_position": 1})]
19918 );
19919 assert_eq!(
19920 first_arrival.message_stream_waits,
19921 vec![json!({"stream_name": "orders", "after_position": 1})]
19922 );
19923
19924 let second_delivery = delivery("message-2", 2, "two");
19925 let first_applied = applied(1, first_delivery);
19926 let completed = worker()
19927 .execute_workflow_task_decision(task_with_resume(
19928 vec![opened(1), first_applied.clone(), opened(2)],
19929 second_delivery.clone(),
19930 ))
19931 .expect("next replacement worker consumes the second arrival");
19932 assert_eq!(completed.commands.len(), 1);
19933 assert_eq!(completed.commands[0]["type"], "complete_workflow");
19934 assert_eq!(
19935 decode_wire_value(&completed.commands[0]["result"], DEFAULT_CODEC)
19936 .expect("workflow result"),
19937 json!(["message-1", "message-2"])
19938 );
19939 assert_eq!(
19940 completed.message_stream_cursors,
19941 vec![json!({"stream_name": "orders", "through_position": 2})]
19942 );
19943 assert!(completed.message_stream_waits.is_empty());
19944
19945 let replay_history = vec![
19946 opened(1),
19947 first_applied,
19948 opened(2),
19949 applied(2, second_delivery),
19950 ];
19951 for _cold_worker_or_restart in 0..2 {
19952 let replayed = worker()
19953 .execute_workflow_task_decision(workflow_task(
19954 "rust.message-stream",
19955 replay_history.clone(),
19956 DEFAULT_CODEC,
19957 ))
19958 .expect("cold worker replays each logical message exactly once");
19959 assert_eq!(replayed.commands.len(), 1);
19960 assert_eq!(
19961 decode_wire_value(&replayed.commands[0]["result"], DEFAULT_CODEC)
19962 .expect("replayed workflow result"),
19963 json!(["message-1", "message-2"])
19964 );
19965 assert_eq!(
19966 replayed.message_stream_cursors,
19967 vec![json!({"stream_name": "orders", "through_position": 2})]
19968 );
19969 assert!(replayed.message_stream_waits.is_empty());
19970 }
19971 }
19972
19973 #[test]
19974 fn message_stream_capability_and_completion_require_protocol_one_fifteen() {
19975 assert!(!worker_protocol_supports_message_streams("1.14"));
19976 assert!(worker_protocol_supports_message_streams("1.15"));
19977 assert!(worker_protocol_supports_message_streams("1.16"));
19978 assert!(worker_protocol_supports_message_streams(
19979 WORKER_PROTOCOL_VERSION
19980 ));
19981 assert_eq!(MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION, "1.15");
19982 }
19983
19984 #[test]
19985 fn condition_wait_history_cannot_be_consumed_as_a_typed_signal_wait() {
19986 let ctx = workflow_context(vec![
19987 history_event(
19988 "ConditionWaitOpened",
19989 json!({
19990 "sequence": 1,
19991 "condition_wait_id": "condition:1",
19992 "condition_wait_occurrence_id": "rust:condition-wait:0",
19993 "condition_key": "signal:finish",
19994 "condition_definition_fingerprint": "sha256:signal-finish-v1",
19995 }),
19996 ),
19997 history_event(
19998 "ConditionWaitSatisfied",
19999 json!({
20000 "sequence": 1,
20001 "condition_wait_id": "condition:1",
20002 "condition_wait_occurrence_id": "rust:condition-wait:0",
20003 "condition_key": "signal:finish",
20004 "condition_definition_fingerprint": "sha256:signal-finish-v1",
20005 }),
20006 ),
20007 history_event(
20008 "SignalReceived",
20009 json!({"signal_name": "finish", "arguments": []}),
20010 ),
20011 ]);
20012 let mut signal = Box::pin(ctx.wait_signal("finish"));
20013 let mut task_context = TaskContext::from_waker(noop_waker_ref());
20014
20015 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
20016 signal.as_mut().poll(&mut task_context)
20017 else {
20018 panic!("condition history must not resolve as a typed signal wait");
20019 };
20020 assert_eq!(failure.reason, "recorded_command_mismatch");
20021 assert_eq!(failure.expected.as_deref(), Some("condition wait"));
20022 }
20023
20024 #[test]
20025 fn replay_orders_signal_waits_and_timers_in_one_command_stream() {
20026 let signal_then_timer = vec![
20027 history_event(
20028 "SignalWaitOpened",
20029 json!({"sequence": 1, "signal_name": "go"}),
20030 ),
20031 history_event(
20032 "SignalApplied",
20033 json!({
20034 "sequence": 1,
20035 "signal_name": "go",
20036 "value": fixture_envelope(json!(["now"])),
20037 }),
20038 ),
20039 history_event(
20040 "TimerScheduled",
20041 json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
20042 ),
20043 history_event(
20044 "TimerFired",
20045 json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
20046 ),
20047 ];
20048
20049 let ctx = workflow_context(signal_then_timer.clone());
20050 let mut signal = Box::pin(ctx.wait_signal("go"));
20051 let mut task_context = TaskContext::from_waker(noop_waker_ref());
20052 assert!(matches!(
20053 signal.as_mut().poll(&mut task_context),
20054 Poll::Ready(Ok(arguments)) if arguments == vec![json!("now")]
20055 ));
20056 let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5)));
20057 assert!(matches!(
20058 timer.as_mut().poll(&mut task_context),
20059 Poll::Ready(Ok(()))
20060 ));
20061 ctx.ensure_history_consumed()
20062 .expect("signal and timer history consumed in order");
20063
20064 let reordered = workflow_context(signal_then_timer);
20065 let mut timer_first = Box::pin(reordered.sleep(Duration::from_secs(5)));
20066 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
20067 timer_first.as_mut().poll(&mut task_context)
20068 else {
20069 panic!("timer cannot consume signal-wait-first history");
20070 };
20071 assert_eq!(failure.reason, "recorded_command_mismatch");
20072 assert_eq!(failure.sequence, Some(1));
20073 assert_eq!(failure.expected.as_deref(), Some("signal wait"));
20074
20075 let timer_then_signal = vec![
20076 history_event(
20077 "TimerScheduled",
20078 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20079 ),
20080 history_event(
20081 "TimerFired",
20082 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20083 ),
20084 history_event(
20085 "SignalWaitOpened",
20086 json!({"sequence": 2, "signal_name": "go"}),
20087 ),
20088 history_event(
20089 "SignalApplied",
20090 json!({
20091 "sequence": 2,
20092 "signal_name": "go",
20093 "value": fixture_envelope(json!([])),
20094 }),
20095 ),
20096 ];
20097 let reordered = workflow_context(timer_then_signal);
20098 let mut signal_first = Box::pin(reordered.wait_signal("go"));
20099 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
20100 signal_first.as_mut().poll(&mut task_context)
20101 else {
20102 panic!("signal wait cannot consume timer-first history");
20103 };
20104 assert_eq!(failure.reason, "recorded_command_mismatch");
20105 assert_eq!(failure.sequence, Some(1));
20106 assert_eq!(failure.expected.as_deref(), Some("timer"));
20107 }
20108
20109 #[test]
20110 fn workflow_history_rejects_duplicate_or_colliding_command_sequences() {
20111 let duplicate_timer = WorkflowState::new(
20112 vec![
20113 history_event(
20114 "TimerScheduled",
20115 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20116 ),
20117 history_event(
20118 "TimerScheduled",
20119 json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
20120 ),
20121 ],
20122 "rust-workers".to_string(),
20123 DEFAULT_CODEC.to_string(),
20124 None,
20125 )
20126 .expect_err("one workflow sequence cannot schedule two timers");
20127 assert!(matches!(
20128 duplicate_timer,
20129 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20130 if reason == "timer_schedule_missing_or_duplicate"
20131 ));
20132
20133 let colliding_kinds = WorkflowState::new(
20134 vec![
20135 history_event(
20136 "TimerScheduled",
20137 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20138 ),
20139 history_event(
20140 "ActivityCompleted",
20141 json!({"sequence": 1, "activity_type": "same-sequence"}),
20142 ),
20143 ],
20144 "rust-workers".to_string(),
20145 DEFAULT_CODEC.to_string(),
20146 None,
20147 )
20148 .expect_err("one workflow sequence cannot identify two command kinds");
20149 assert!(matches!(
20150 colliding_kinds,
20151 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20152 if reason == "durable_command_sequence_collision"
20153 ));
20154
20155 let duplicate_signal_wait = WorkflowState::new(
20156 vec![
20157 history_event(
20158 "SignalWaitOpened",
20159 json!({"sequence": 1, "signal_name": "go"}),
20160 ),
20161 history_event(
20162 "SignalWaitOpened",
20163 json!({"sequence": 1, "signal_name": "go"}),
20164 ),
20165 ],
20166 "rust-workers".to_string(),
20167 DEFAULT_CODEC.to_string(),
20168 None,
20169 )
20170 .expect_err("one workflow sequence cannot open two signal waits");
20171 assert!(matches!(
20172 duplicate_signal_wait,
20173 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20174 if reason == "signal_wait_open_missing_or_duplicate"
20175 ));
20176 }
20177
20178 #[test]
20179 fn workflow_history_accepts_a_first_command_after_global_sequence_gaps() {
20180 let result = encode_value_envelope(&json!({"captured": true}), DEFAULT_CODEC)
20181 .expect("side-effect result");
20182 let ctx = workflow_context(vec![history_event(
20183 "SideEffectRecorded",
20184 json!({"sequence": 99, "result": result}),
20185 )]);
20186
20187 let replayed: Value = ctx
20188 .side_effect(|| panic!("recorded side effect must not run"))
20189 .expect("positive global workflow sequence is valid");
20190 assert_eq!(replayed, json!({"captured": true}));
20191 ctx.ensure_history_consumed().expect("history consumed");
20192 }
20193
20194 #[test]
20195 fn workflow_history_rejects_zero_and_descending_command_sequences() {
20196 let result =
20197 encode_value_envelope(&json!("captured"), DEFAULT_CODEC).expect("side-effect result");
20198 let zero = WorkflowState::new(
20199 vec![history_event(
20200 "SideEffectRecorded",
20201 json!({"sequence": 0, "result": result.clone()}),
20202 )],
20203 "rust-workers".to_string(),
20204 DEFAULT_CODEC.to_string(),
20205 None,
20206 )
20207 .expect_err("durable command sequences must be positive");
20208 assert!(matches!(
20209 zero,
20210 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20211 if reason == "durable_command_sequence_invalid"
20212 ));
20213
20214 let descending = WorkflowState::new(
20215 vec![
20216 history_event(
20217 "SideEffectRecorded",
20218 json!({"sequence": 3, "result": result}),
20219 ),
20220 history_event(
20221 "VersionMarkerRecorded",
20222 json!({
20223 "sequence": 2,
20224 "change_id": "descending-marker",
20225 "version": 1,
20226 "min_supported": 1,
20227 "max_supported": 1,
20228 }),
20229 ),
20230 ],
20231 "rust-workers".to_string(),
20232 DEFAULT_CODEC.to_string(),
20233 None,
20234 )
20235 .expect_err("new durable commands must remain strictly ordered");
20236 let Error::NonDeterministicReplay(failure) = descending else {
20237 panic!("expected typed replay failure");
20238 };
20239 assert_eq!(failure.reason, "durable_command_sequence_mismatch");
20240 assert_eq!(failure.sequence, Some(2));
20241 assert_eq!(
20242 failure.expected.as_deref(),
20243 Some("workflow sequence greater than 3")
20244 );
20245 assert_eq!(failure.actual.as_deref(), Some("2"));
20246 }
20247
20248 #[test]
20249 fn workflow_task_replay_completes_after_signals_create_sequence_gaps() {
20250 fn worker() -> Worker {
20251 let client = Client::new("http://127.0.0.1:8080").expect("client");
20252 let mut worker = Worker::new(client, "rust-workers");
20253 worker.register_workflow("rust.finish-after-gaps", |ctx, _input| async move {
20254 ctx.wait_signal("finish").await?;
20255 let marker: String =
20256 ctx.side_effect(|| panic!("recorded side effect must not run"))?;
20257 assert_eq!(marker, "after-finish");
20258 Ok(json!("finished"))
20259 });
20260 worker
20261 }
20262
20263 let marker = encode_value_envelope(&json!("after-finish"), DEFAULT_CODEC)
20264 .expect("side-effect result");
20265 let task = workflow_task(
20266 "rust.finish-after-gaps",
20267 vec![
20268 history_event(
20269 "SignalWaitOpened",
20270 json!({"sequence": 1, "signal_name": "finish"}),
20271 ),
20272 history_event(
20273 "SignalReceived",
20274 json!({
20275 "signal_id": "increment-3",
20276 "signal_name": "increment",
20277 "workflow_sequence": 2,
20278 "payload_codec": DEFAULT_CODEC,
20279 "arguments": fixture_envelope(json!([3])),
20280 }),
20281 ),
20282 history_event(
20283 "SignalReceived",
20284 json!({
20285 "signal_id": "increment-5",
20286 "signal_name": "increment",
20287 "workflow_sequence": 3,
20288 "payload_codec": DEFAULT_CODEC,
20289 "arguments": fixture_envelope(json!([5])),
20290 }),
20291 ),
20292 history_event(
20293 "SignalReceived",
20294 json!({
20295 "signal_id": "finish",
20296 "signal_name": "finish",
20297 "workflow_sequence": 4,
20298 "payload_codec": DEFAULT_CODEC,
20299 "arguments": fixture_envelope(json!([])),
20300 }),
20301 ),
20302 history_event(
20303 "SignalApplied",
20304 json!({
20305 "sequence": 1,
20306 "signal_id": "finish",
20307 "signal_name": "finish",
20308 "payload_codec": DEFAULT_CODEC,
20309 "value": fixture_envelope(json!([])),
20310 }),
20311 ),
20312 history_event(
20313 "SideEffectRecorded",
20314 json!({"sequence": 5, "result": marker}),
20315 ),
20316 ],
20317 DEFAULT_CODEC,
20318 );
20319
20320 for _original_or_cold_worker in 0..2 {
20321 let commands = worker()
20322 .execute_workflow_task(task.clone())
20323 .expect("signal gaps preserve deterministic replay");
20324 assert_eq!(commands.len(), 1, "replay emits only terminal completion");
20325 assert_eq!(commands[0]["type"], "complete_workflow");
20326 assert_eq!(
20327 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
20328 json!("finished")
20329 );
20330 }
20331 }
20332
20333 #[test]
20334 fn workflow_sleep_rejects_unrepresentable_rounded_duration() {
20335 let ctx = workflow_context(Vec::new());
20336 let mut sleep = Box::pin(ctx.start_timer(Duration::new(u64::MAX, 1)));
20337 let mut task_context = TaskContext::from_waker(noop_waker_ref());
20338 assert!(matches!(
20339 sleep.as_mut().poll(&mut task_context),
20340 Poll::Ready(Err(Error::TimerDurationOverflow))
20341 ));
20342 assert!(ctx.take_commands().expect("commands").is_empty());
20343 }
20344
20345 #[test]
20346 fn workflow_memo_update_emits_canonical_command_and_replays_once() {
20347 let entries = AvroValue::Map(BTreeMap::from([
20348 ("text".to_string(), AvroValue::String("same".to_string())),
20349 (
20350 "nested".to_string(),
20351 AvroValue::Map(BTreeMap::from([
20352 ("beta".to_string(), AvroValue::Long(2)),
20353 ("alpha".to_string(), AvroValue::Long(1)),
20354 ])),
20355 ),
20356 ("long".to_string(), AvroValue::Long(7)),
20357 ("double".to_string(), AvroValue::Double(7.0)),
20358 ("binary".to_string(), AvroValue::Bytes(b"same".to_vec())),
20359 ]));
20360 let ctx = workflow_context(Vec::new());
20361 ctx.upsert_memo(entries.clone()).expect("valid memo update");
20362 let commands = ctx.take_commands().expect("commands");
20363
20364 assert_eq!(commands.len(), 1);
20365 assert_eq!(commands[0]["type"], "upsert_memo");
20366 let server_entries = json!({
20367 "codec": "avro",
20368 "blob": "wwHioz3/VYAiNw4KDGJpbmFyeQgIc2FtZQxkb3VibGUGAAAAAAAAHEAIbG9uZwQODG5lc3RlZA4ECmFscGhhBAIIYmV0YQQEAAh0ZXh0CghzYW1lAA==",
20369 });
20370 assert_eq!(
20371 commands[0]["entries"]
20372 .as_object()
20373 .expect("entries envelope")
20374 .keys()
20375 .collect::<Vec<_>>(),
20376 vec!["blob", "codec"]
20377 );
20378 assert_eq!(commands[0]["entries"], server_entries);
20379 let wire_entries =
20380 decode_wire_avro_value(&commands[0]["entries"], DEFAULT_CODEC).expect("memo entries");
20381 assert_eq!(wire_entries, entries);
20382
20383 let history = vec![history_event(
20384 "MemoUpserted",
20385 json!({
20386 "sequence": 1,
20387 "entries": server_entries.clone(),
20388 "merged": server_entries,
20389 }),
20390 )];
20391 let replay = workflow_context(history.clone());
20392 replay
20393 .upsert_memo(entries.clone())
20394 .expect("matching replay identity");
20395 assert!(replay.take_commands().expect("replay commands").is_empty());
20396
20397 let changed_types = AvroValue::Map(BTreeMap::from([
20398 ("text".to_string(), AvroValue::Bytes(b"same".to_vec())),
20399 (
20400 "nested".to_string(),
20401 AvroValue::Map(BTreeMap::from([
20402 ("alpha".to_string(), AvroValue::Long(1)),
20403 ("beta".to_string(), AvroValue::Long(2)),
20404 ])),
20405 ),
20406 ("long".to_string(), AvroValue::Double(7.0)),
20407 ("double".to_string(), AvroValue::Long(7)),
20408 ("binary".to_string(), AvroValue::String("same".to_string())),
20409 ]));
20410 let error = workflow_context(history)
20411 .upsert_memo(changed_types)
20412 .expect_err("memo replay identity must preserve Avro value types");
20413 assert!(matches!(
20414 error,
20415 Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20416 ));
20417 }
20418
20419 #[test]
20420 fn workflow_memo_update_rejects_changed_replay_identity_and_invalid_keys() {
20421 let original = encode_value_envelope(&json!({"stage": "original"}), DEFAULT_CODEC)
20422 .expect("memo envelope");
20423 let replay = workflow_context(vec![history_event(
20424 "MemoUpserted",
20425 json!({
20426 "sequence": 1,
20427 "entries": original.clone(),
20428 "merged": original
20429 }),
20430 )]);
20431 let error = replay
20432 .upsert_memo(json!({"stage": "changed"}))
20433 .expect_err("changed memo update must fail replay");
20434 assert!(matches!(
20435 error,
20436 Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20437 ));
20438
20439 let invalid = workflow_context(Vec::new())
20440 .upsert_memo(
20441 json!({"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx": true}),
20442 )
20443 .expect_err("oversized key");
20444 assert!(matches!(invalid, Error::InvalidMemoUpdate(_)));
20445 }
20446
20447 #[test]
20448 fn workflow_memo_replay_distinguishes_signed_zero_identity() {
20449 let negative_zero = AvroValue::Map(BTreeMap::from([(
20450 "reading".to_string(),
20451 AvroValue::Double(-0.0),
20452 )]));
20453 let negative_zero_envelope =
20454 encode_typed_envelope(&negative_zero, DEFAULT_CODEC).expect("negative zero envelope");
20455 let history = vec![history_event(
20456 "MemoUpserted",
20457 json!({
20458 "sequence": 1,
20459 "entries": negative_zero_envelope.clone(),
20460 "merged": negative_zero_envelope,
20461 }),
20462 )];
20463
20464 workflow_context(history.clone())
20465 .upsert_memo(negative_zero)
20466 .expect("matching negative-zero history identity");
20467
20468 let error = workflow_context(history)
20469 .upsert_memo(AvroValue::Map(BTreeMap::from([(
20470 "reading".to_string(),
20471 AvroValue::Double(0.0),
20472 )])))
20473 .expect_err("positive zero must not consume negative-zero memo history");
20474 assert!(matches!(
20475 error,
20476 Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20477 ));
20478 }
20479
20480 #[test]
20481 fn workflow_memo_capability_requires_flag_and_command_advertisement() {
20482 let supported = json!({
20483 "workflow_memo_updates": {"supported": true, "minimum_protocol_version": "1.14"},
20484 "supported_workflow_task_commands": ["complete_workflow", "upsert_memo"]
20485 });
20486 assert!(runtime_supports_workflow_memo_updates(Some(&supported)));
20487 assert!(!runtime_supports_workflow_memo_updates(Some(&json!({
20488 "workflow_memo_updates": {"supported": false},
20489 "supported_workflow_task_commands": ["upsert_memo"]
20490 }))));
20491 assert!(commands_use_workflow_memo_updates(&[json!({
20492 "type": "upsert_memo",
20493 "entries": {"stage": "processing"}
20494 })]));
20495 }
20496
20497 #[test]
20498 fn workflow_task_replay_completes_without_rescheduling_recorded_commands() {
20499 let client = Client::new("http://127.0.0.1:8080").expect("client");
20500 let mut worker = Worker::new(client, "rust-workers");
20501 worker.register_workflow("rust.timer", |ctx, _input| async move {
20502 ctx.sleep(Duration::from_secs(5)).await?;
20503 ctx.activity("after-timer", json!([])).await
20504 });
20505
20506 let task = |history_events| WorkflowTask {
20507 task_id: "wft-rust-timer-1".to_string(),
20508 workflow_command_id: None,
20509 workflow_id: Some("wf-rust-timer".to_string()),
20510 run_id: Some("run-rust-timer".to_string()),
20511 workflow_type: "rust.timer".to_string(),
20512 cancel_requested: false,
20513 payload_codec: DEFAULT_CODEC.to_string(),
20514 arguments: Some(
20515 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
20516 ),
20517 history_events,
20518 total_history_events: None,
20519 history_size_bytes: None,
20520 continue_as_new_recommended: None,
20521 history_budget_pressure: None,
20522 next_history_page_token: None,
20523 workflow_task_attempt: 1,
20524 workflow_signal_id: None,
20525 signal_name: None,
20526 signal_arguments: None,
20527 workflow_update_id: None,
20528 update_name: None,
20529 lease_owner: Some("rust-worker".to_string()),
20530 };
20531
20532 let initial = worker
20533 .execute_workflow_task(task(Vec::new()))
20534 .expect("initial timer task");
20535 assert_eq!(
20536 initial,
20537 vec![json!({"type": "start_timer", "delay_seconds": 5})]
20538 );
20539
20540 let activity_result =
20541 encode_value_envelope(&json!("done"), DEFAULT_CODEC).expect("activity result");
20542 let replayed = worker
20543 .execute_workflow_task(task(vec![
20544 history_event(
20545 "TimerScheduled",
20546 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20547 ),
20548 history_event(
20549 "TimerFired",
20550 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20551 ),
20552 history_event(
20553 "ActivityCompleted",
20554 json!({
20555 "sequence": 2,
20556 "activity_type": "after-timer",
20557 "payload_codec": DEFAULT_CODEC,
20558 "result": activity_result,
20559 }),
20560 ),
20561 ]))
20562 .expect("replayed workflow task");
20563 assert_eq!(replayed.len(), 1);
20564 assert_eq!(replayed[0]["type"], "complete_workflow");
20565 assert_eq!(
20566 decode_wire_value(&replayed[0]["result"], DEFAULT_CODEC).expect("result"),
20567 json!("done")
20568 );
20569 }
20570
20571 #[test]
20572 fn workflow_continue_as_new_emits_arguments_type_and_queue_once() {
20573 let client = Client::new("http://127.0.0.1:8080").expect("client");
20574 let mut worker = Worker::new(client, "rust-workers");
20575 worker.register_workflow("rust.continue", |ctx, _input| async move {
20576 ctx.continue_as_new_with_options(
20577 ContinueAsNewOptions::new()
20578 .workflow_type("rust.next")
20579 .task_queue("next-workers"),
20580 json!([2, {"cursor": "next"}]),
20581 )
20582 });
20583
20584 let commands = worker
20585 .execute_workflow_task(workflow_task("rust.continue", Vec::new(), DEFAULT_CODEC))
20586 .expect("continue-as-new command");
20587
20588 assert_eq!(commands.len(), 1);
20589 assert_eq!(commands[0]["type"], "continue_as_new");
20590 assert_eq!(commands[0]["workflow_type"], "rust.next");
20591 assert_eq!(commands[0]["queue"], "next-workers");
20592 assert_eq!(
20593 decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC)
20594 .expect("continue-as-new arguments"),
20595 json!([2, {"cursor": "next"}])
20596 );
20597 }
20598
20599 #[test]
20600 fn continue_as_new_preserves_typed_arguments() {
20601 let client = Client::new("http://127.0.0.1:8080").expect("client");
20602 let mut worker = Worker::new(client, "rust-workers");
20603 worker.register_workflow_avro_value("rust.typed-continue", |ctx, _input| async move {
20604 ctx.continue_as_new(AvroValue::Array(vec![typed_fidelity_probe()]))?;
20605 unreachable!("continue-as-new returns a control-flow error")
20606 });
20607
20608 let commands = worker
20609 .execute_workflow_task(workflow_task(
20610 "rust.typed-continue",
20611 Vec::new(),
20612 DEFAULT_CODEC,
20613 ))
20614 .expect("typed continue-as-new command");
20615
20616 assert_eq!(commands[0]["type"], "continue_as_new");
20617 assert_eq!(
20618 decode_wire_avro_value(&commands[0]["arguments"], DEFAULT_CODEC)
20619 .expect("typed continue arguments"),
20620 AvroValue::Array(vec![typed_fidelity_probe()])
20621 );
20622 }
20623
20624 #[test]
20625 fn recorded_continue_as_new_is_consumed_without_duplicate_successor_command() {
20626 let client = Client::new("http://127.0.0.1:8080").expect("client");
20627 let mut worker = Worker::new(client, "rust-workers");
20628 worker.register_workflow("rust.continue", |ctx, _input| async move {
20629 ctx.continue_as_new(json!([2]))
20630 });
20631 let task = workflow_task(
20632 "rust.continue",
20633 vec![history_event(
20634 "WorkflowContinuedAsNew",
20635 json!({"sequence": 1, "continued_to_run_id": "run-next"}),
20636 )],
20637 DEFAULT_CODEC,
20638 );
20639
20640 for _worker_restart_or_redelivery in 0..2 {
20641 let commands = worker
20642 .execute_workflow_task(task.clone())
20643 .expect("recorded transition replays");
20644 assert!(
20645 commands.is_empty(),
20646 "replay must not emit another successor"
20647 );
20648 }
20649 }
20650
20651 #[test]
20652 fn continue_as_new_rejects_invalid_overrides_before_emitting_a_command() {
20653 let ctx = workflow_context(Vec::new());
20654 let error = ctx
20655 .continue_as_new_with_options(ContinueAsNewOptions::new().task_queue(" "), json!([1]))
20656 .expect_err("blank queue must be rejected");
20657
20658 let Error::InvalidContinueAsNewOptions(error) = error else {
20659 panic!("expected typed continue-as-new validation error");
20660 };
20661 assert_eq!(error.field, "task_queue");
20662 assert!(ctx.take_commands().expect("commands").is_empty());
20663 }
20664
20665 #[test]
20666 fn workflow_context_exposes_server_history_budget() {
20667 let client = Client::new("http://127.0.0.1:8080").expect("client");
20668 let mut worker = Worker::new(client, "rust-workers");
20669 worker.register_workflow("rust.history-budget", |ctx, _input| async move {
20670 let budget = ctx.history_budget()?;
20671 Ok(json!({
20672 "events": budget.event_count,
20673 "bytes": budget.size_bytes,
20674 "recommended": budget.continue_as_new_recommended,
20675 "pressure": budget.pressure,
20676 }))
20677 });
20678 let task: WorkflowTask = serde_json::from_value(json!({
20679 "task_id": "task-history-budget",
20680 "workflow_type": "rust.history-budget",
20681 "payload_codec": DEFAULT_CODEC,
20682 "history_events": [],
20683 "total_history_events": 480,
20684 "history_size_bytes": 1_048_576,
20685 "continue_as_new_recommended": true,
20686 "history_budget_pressure": "continue_as_new_recommended",
20687 }))
20688 .expect("published workflow task");
20689
20690 let commands = worker
20691 .execute_workflow_task(task)
20692 .expect("history-budget workflow");
20693 let result = decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("result");
20694 assert_eq!(result["events"], 480);
20695 assert_eq!(result["bytes"], 1_048_576);
20696 assert_eq!(result["recommended"], true);
20697 assert_eq!(result["pressure"], "continue_as_new_recommended");
20698 }
20699
20700 #[test]
20701 fn uncaught_workflow_handler_error_emits_terminal_failure_command() {
20702 let client = Client::new("http://127.0.0.1:8080").expect("client");
20703 let mut worker = Worker::new(client, "rust-workers");
20704 worker.register_workflow("rust.failing", |_ctx, _input| async move {
20705 Err(Error::Codec("rust_conformance_failure".to_string()))
20706 });
20707 let task = WorkflowTask {
20708 task_id: "wft-rust-failing-1".to_string(),
20709 workflow_command_id: None,
20710 workflow_id: Some("wf-rust-failing".to_string()),
20711 run_id: Some("run-rust-failing".to_string()),
20712 workflow_type: "rust.failing".to_string(),
20713 cancel_requested: false,
20714 payload_codec: DEFAULT_CODEC.to_string(),
20715 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
20716 history_events: Vec::new(),
20717 total_history_events: Some(0),
20718 history_size_bytes: None,
20719 continue_as_new_recommended: None,
20720 history_budget_pressure: None,
20721 next_history_page_token: None,
20722 workflow_task_attempt: 1,
20723 workflow_signal_id: None,
20724 signal_name: None,
20725 signal_arguments: None,
20726 workflow_update_id: None,
20727 update_name: None,
20728 lease_owner: Some("rust-worker".to_string()),
20729 };
20730
20731 let commands = worker
20732 .execute_workflow_task(task)
20733 .expect("handler failure becomes a workflow command");
20734
20735 assert_eq!(commands.len(), 1);
20736 assert_eq!(commands[0]["type"], "fail_workflow");
20737 assert_eq!(commands[0]["exception_type"], "RustWorkflowError");
20738 assert_eq!(commands[0]["exception_class"], "durable_workflow::Error");
20739 assert_eq!(commands[0]["non_retryable"], false);
20740 assert_eq!(
20741 commands[0]["message"],
20742 "codec error: rust_conformance_failure"
20743 );
20744 assert_eq!(
20745 commands[0]["exception"]["message"],
20746 "codec error: rust_conformance_failure"
20747 );
20748 }
20749
20750 #[test]
20751 fn ordinary_handler_error_preserves_commands_queued_in_the_same_decision() {
20752 let client = Client::new("http://127.0.0.1:8080").expect("client");
20753 let mut worker = Worker::new(client, "rust-workers");
20754 worker.register_workflow("rust.failing-after-side-effect", |ctx, _input| async move {
20755 let _: String = ctx.side_effect(|| "captured".to_string())?;
20756 Err(Error::WorkerLoop("application failure".to_string()))
20757 });
20758
20759 let commands = worker
20760 .execute_workflow_task(workflow_task(
20761 "rust.failing-after-side-effect",
20762 Vec::new(),
20763 DEFAULT_CODEC,
20764 ))
20765 .expect("ordinary failure remains a workflow decision");
20766
20767 assert_eq!(commands.len(), 2);
20768 assert_eq!(commands[0]["type"], "record_side_effect");
20769 assert_eq!(commands[1]["type"], "fail_workflow");
20770 }
20771
20772 #[test]
20773 fn handler_error_cannot_hide_an_unconsumed_committed_side_effect() {
20774 let client = Client::new("http://127.0.0.1:8080").expect("client");
20775 let mut worker = Worker::new(client, "rust-workers");
20776 worker.register_workflow("rust.removed-side-effect", |_ctx, _input| async move {
20777 Err(Error::WorkerLoop("application failure".to_string()))
20778 });
20779 let result =
20780 encode_value_envelope(&json!("committed"), DEFAULT_CODEC).expect("side-effect result");
20781
20782 let error = worker
20783 .execute_workflow_task(workflow_task(
20784 "rust.removed-side-effect",
20785 vec![history_event(
20786 "SideEffectRecorded",
20787 json!({"sequence": 1, "result": result}),
20788 )],
20789 DEFAULT_CODEC,
20790 ))
20791 .expect_err("removed committed history must not become fail_workflow");
20792
20793 let Error::NonDeterministicReplay(failure) = error else {
20794 panic!("expected typed replay failure");
20795 };
20796 assert_eq!(failure.reason, "recorded_commands_unconsumed");
20797 assert_eq!(failure.sequence, Some(1));
20798 assert_eq!(failure.expected.as_deref(), Some("side effect"));
20799 }
20800
20801 #[test]
20802 fn replay_error_discards_side_effect_queued_before_incompatible_marker_check() {
20803 let client = Client::new("http://127.0.0.1:8080").expect("client");
20804 let mut worker = Worker::new(client, "rust-workers");
20805 worker.register_workflow(
20806 "rust.side-effect-before-marker-error",
20807 |ctx, _input| async move {
20808 assert_eq!(ctx.get_version("restart-safe", 1, 1)?, 1);
20809 let _: String = ctx.side_effect(|| "must-not-commit".to_string())?;
20810 ctx.get_version("restart-safe", 2, 2)?;
20811 Ok(Value::Null)
20812 },
20813 );
20814
20815 let error = worker
20816 .execute_workflow_task(workflow_task(
20817 "rust.side-effect-before-marker-error",
20818 vec![history_event(
20819 "VersionMarkerRecorded",
20820 json!({
20821 "sequence": 1,
20822 "change_id": "restart-safe",
20823 "version": 1,
20824 "min_supported": 1,
20825 "max_supported": 1,
20826 }),
20827 )],
20828 DEFAULT_CODEC,
20829 ))
20830 .expect_err("replay error must return no queued workflow commands");
20831
20832 let Error::NonDeterministicReplay(failure) = error else {
20833 panic!("expected typed replay failure");
20834 };
20835 assert_eq!(failure.reason, "version_marker_incompatible_range");
20836 assert_eq!(failure.sequence, Some(1));
20837 }
20838
20839 #[test]
20840 fn workflow_task_replay_keeps_recorded_unfired_timer_pending_without_rescheduling() {
20841 let client = Client::new("http://127.0.0.1:8080").expect("client");
20842 let mut worker = Worker::new(client, "rust-workers");
20843 worker.register_workflow("rust.timer.pending", |ctx, _input| async move {
20844 ctx.sleep(Duration::from_secs(5)).await?;
20845 Ok(json!({"status": "timer fired"}))
20846 });
20847
20848 let task = WorkflowTask {
20849 task_id: "wft-rust-timer-pending".to_string(),
20850 workflow_command_id: None,
20851 workflow_id: Some("wf-rust-timer".to_string()),
20852 run_id: Some("run-rust-timer".to_string()),
20853 workflow_type: "rust.timer.pending".to_string(),
20854 cancel_requested: false,
20855 payload_codec: DEFAULT_CODEC.to_string(),
20856 arguments: Some(
20857 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
20858 ),
20859 history_events: vec![history_event(
20860 "TimerScheduled",
20861 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20862 )],
20863 total_history_events: Some(1),
20864 history_size_bytes: None,
20865 continue_as_new_recommended: None,
20866 history_budget_pressure: None,
20867 next_history_page_token: None,
20868 workflow_task_attempt: 1,
20869 workflow_signal_id: None,
20870 signal_name: None,
20871 signal_arguments: None,
20872 workflow_update_id: None,
20873 update_name: None,
20874 lease_owner: Some("rust-worker".to_string()),
20875 };
20876
20877 for _redelivery_or_restart in 0..2 {
20878 let commands = worker
20879 .execute_workflow_task(task.clone())
20880 .expect("recorded timer remains pending");
20881 assert!(
20882 commands.is_empty(),
20883 "recorded timer must not be rescheduled"
20884 );
20885 }
20886 }
20887
20888 #[test]
20889 fn workflow_task_rejects_recorded_command_removed_from_workflow_code() {
20890 let client = Client::new("http://127.0.0.1:8080").expect("client");
20891 let mut worker = Worker::new(client, "rust-workers");
20892 worker.register_workflow("rust.timer.removed", |_ctx, _input| async move {
20893 Ok(json!({"status": "completed"}))
20894 });
20895 let task = WorkflowTask {
20896 task_id: "wft-rust-timer-removed".to_string(),
20897 workflow_command_id: None,
20898 workflow_id: Some("wf-rust-timer".to_string()),
20899 run_id: Some("run-rust-timer".to_string()),
20900 workflow_type: "rust.timer.removed".to_string(),
20901 cancel_requested: false,
20902 payload_codec: DEFAULT_CODEC.to_string(),
20903 arguments: Some(
20904 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
20905 ),
20906 history_events: vec![
20907 history_event(
20908 "TimerScheduled",
20909 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20910 ),
20911 history_event(
20912 "TimerFired",
20913 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20914 ),
20915 ],
20916 total_history_events: Some(2),
20917 history_size_bytes: None,
20918 continue_as_new_recommended: None,
20919 history_budget_pressure: None,
20920 next_history_page_token: None,
20921 workflow_task_attempt: 1,
20922 workflow_signal_id: None,
20923 signal_name: None,
20924 signal_arguments: None,
20925 workflow_update_id: None,
20926 update_name: None,
20927 lease_owner: Some("rust-worker".to_string()),
20928 };
20929
20930 let Error::NonDeterministicReplay(failure) = worker
20931 .execute_workflow_task(task)
20932 .expect_err("removed timer must fail replay")
20933 else {
20934 panic!("expected typed replay failure");
20935 };
20936 assert_eq!(failure.reason, "recorded_commands_unconsumed");
20937 assert_eq!(failure.sequence, Some(1));
20938 }
20939
20940 #[test]
20941 fn workflow_context_emits_explicit_child_workflow_contract() {
20942 let ctx = WorkflowContext {
20943 state: Arc::new(Mutex::new(
20944 WorkflowState::new_with_identity(
20945 Vec::new(),
20946 Some("wf-parent".to_string()),
20947 Some("run-parent".to_string()),
20948 "parent-workers".to_string(),
20949 DEFAULT_CODEC.to_string(),
20950 None,
20951 )
20952 .expect("workflow state"),
20953 )),
20954 };
20955 let options = ChildWorkflowOptions::new("python-workers")
20956 .parent_close_policy(ParentClosePolicy::RequestCancel)
20957 .retry_policy(ChildWorkflowRetryPolicy {
20958 max_attempts: Some(3),
20959 backoff_seconds: vec![1, 5],
20960 non_retryable_error_types: vec!["ValidationError".to_string()],
20961 })
20962 .execution_timeout_seconds(600)
20963 .run_timeout_seconds(120);
20964 let mut call = Box::pin(ctx.start_child_workflow(
20965 "python.fulfil-order",
20966 options,
20967 json!([{"order_id": "order-42"}]),
20968 ));
20969 let mut task_context = TaskContext::from_waker(noop_waker_ref());
20970
20971 assert!(matches!(
20972 call.as_mut().poll(&mut task_context),
20973 Poll::Pending
20974 ));
20975 let commands = ctx.take_commands().expect("commands");
20976 assert_eq!(commands.len(), 1);
20977 let command = &commands[0];
20978 assert_eq!(command["type"], "start_child_workflow");
20979 assert_eq!(command["workflow_type"], "python.fulfil-order");
20980 assert_eq!(command["queue"], "python-workers");
20981 assert_eq!(command["parent_close_policy"], "request_cancel");
20982 assert_eq!(command["retry_policy"]["max_attempts"], 3);
20983 assert_eq!(command["execution_timeout_seconds"], 600);
20984 assert_eq!(command["run_timeout_seconds"], 120);
20985 assert_eq!(
20986 decode_wire_value(&command["arguments"], DEFAULT_CODEC).expect("child args"),
20987 json!([{"order_id": "order-42"}])
20988 );
20989 }
20990
20991 fn child_parent_worker() -> Worker {
20992 let client = Client::new("http://127.0.0.1:8080").expect("client");
20993 let mut worker = Worker::new(client, "rust-parent-workers");
20994 worker.register_workflow("rust.parent", |ctx, _input| async move {
20995 let child = ctx
20996 .start_child_workflow(
20997 "python.child",
20998 ChildWorkflowOptions::new("python-child-workers")
20999 .parent_close_policy(ParentClosePolicy::Terminate),
21000 json!([{"codec_probe": [1, true, "rust"]}]),
21001 )
21002 .await?;
21003 Ok(json!({
21004 "parent_workflow_id": child.parent.workflow_id,
21005 "parent_run_id": child.parent.run_id,
21006 "child_workflow_id": child.child.workflow_id,
21007 "child_run_id": child.child.run_id,
21008 "child_workflow_type": child.child_workflow_type,
21009 "result": child.result,
21010 }))
21011 });
21012 worker
21013 }
21014
21015 fn child_parent_task(event_type: &str, payload: Value) -> WorkflowTask {
21016 WorkflowTask {
21017 task_id: "wft-child-parent".to_string(),
21018 workflow_command_id: None,
21019 workflow_id: Some("wf-parent".to_string()),
21020 run_id: Some("run-parent".to_string()),
21021 workflow_type: "rust.parent".to_string(),
21022 cancel_requested: false,
21023 payload_codec: DEFAULT_CODEC.to_string(),
21024 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21025 history_events: vec![
21026 HistoryEvent {
21027 event_type: "ChildWorkflowScheduled".to_string(),
21028 payload: json!({
21029 "sequence": 1,
21030 "child_call_id": "call-child",
21031 "child_workflow_instance_id": "wf-child",
21032 "child_workflow_run_id": "run-child",
21033 "child_workflow_type": "python.child",
21034 }),
21035 raw: HashMap::new(),
21036 },
21037 HistoryEvent {
21038 event_type: event_type.to_string(),
21039 payload,
21040 raw: HashMap::new(),
21041 },
21042 ],
21043 total_history_events: Some(2),
21044 history_size_bytes: None,
21045 continue_as_new_recommended: None,
21046 history_budget_pressure: None,
21047 next_history_page_token: None,
21048 workflow_task_attempt: 1,
21049 workflow_signal_id: None,
21050 signal_name: None,
21051 signal_arguments: None,
21052 workflow_update_id: None,
21053 update_name: None,
21054 lease_owner: Some("rust-worker".to_string()),
21055 }
21056 }
21057
21058 #[test]
21059 fn committed_child_result_replays_without_starting_a_duplicate() {
21060 let worker = child_parent_worker();
21061 let task = child_parent_task(
21062 "ChildRunCompleted",
21063 json!({
21064 "sequence": 1,
21065 "child_call_id": "call-child",
21066 "child_workflow_instance_id": "wf-child",
21067 "child_workflow_run_id": "run-child",
21068 "child_workflow_type": "python.child",
21069 "payload_codec": DEFAULT_CODEC,
21070 "result": fixture_envelope(json!({"from":"python","ok":true})),
21071 }),
21072 );
21073
21074 for _restart in 0..2 {
21075 let commands = worker
21076 .execute_workflow_task(task.clone())
21077 .expect("replayed parent task");
21078 assert_eq!(commands.len(), 1);
21079 assert_eq!(commands[0]["type"], "complete_workflow");
21080 assert!(!commands
21081 .iter()
21082 .any(|command| command["type"] == "start_child_workflow"));
21083 let output =
21084 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
21085 assert_eq!(output["parent_workflow_id"], "wf-parent");
21086 assert_eq!(output["parent_run_id"], "run-parent");
21087 assert_eq!(output["child_workflow_id"], "wf-child");
21088 assert_eq!(output["child_run_id"], "run-child");
21089 assert_eq!(output["result"], json!({"from": "python", "ok": true}));
21090 }
21091 }
21092
21093 #[test]
21094 fn typed_child_arguments_and_results_survive_replay() {
21095 let client = Client::new("http://127.0.0.1:8080").expect("client");
21096 let mut worker = Worker::new(client, "rust-parent-workers");
21097 worker.register_workflow_avro_value("rust.typed-parent", |ctx, _input| async move {
21098 let child = ctx
21099 .start_child_workflow_avro_value(
21100 "python.typed-child",
21101 ChildWorkflowOptions::new("python-workers"),
21102 AvroValue::Array(vec![typed_fidelity_probe()]),
21103 )
21104 .await?;
21105 Ok(child.result)
21106 });
21107
21108 let initial = worker
21109 .execute_workflow_task(workflow_task(
21110 "rust.typed-parent",
21111 Vec::new(),
21112 DEFAULT_CODEC,
21113 ))
21114 .expect("typed child start");
21115 assert_eq!(initial[0]["type"], "start_child_workflow");
21116 assert_eq!(
21117 decode_wire_avro_value(&initial[0]["arguments"], DEFAULT_CODEC)
21118 .expect("typed child arguments"),
21119 AvroValue::Array(vec![typed_fidelity_probe()])
21120 );
21121
21122 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
21123 .expect("typed child result");
21124 let task = workflow_task(
21125 "rust.typed-parent",
21126 vec![
21127 history_event(
21128 "ChildWorkflowScheduled",
21129 json!({
21130 "sequence": 1,
21131 "child_call_id": "call-typed",
21132 "child_workflow_instance_id": "wf-child",
21133 "child_workflow_run_id": "run-child",
21134 "child_workflow_type": "python.typed-child",
21135 }),
21136 ),
21137 history_event(
21138 "ChildRunCompleted",
21139 json!({
21140 "sequence": 1,
21141 "child_call_id": "call-typed",
21142 "child_workflow_instance_id": "wf-child",
21143 "child_workflow_run_id": "run-child",
21144 "child_workflow_type": "python.typed-child",
21145 "payload_codec": DEFAULT_CODEC,
21146 "result": result,
21147 }),
21148 ),
21149 ],
21150 DEFAULT_CODEC,
21151 );
21152
21153 let commands = worker
21154 .execute_workflow_task(task)
21155 .expect("typed child replay");
21156 assert_eq!(commands[0]["type"], "complete_workflow");
21157 assert_eq!(
21158 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
21159 .expect("typed parent result"),
21160 typed_fidelity_probe()
21161 );
21162 }
21163
21164 #[test]
21165 fn pending_child_replays_after_restart_without_starting_a_duplicate() {
21166 let worker = child_parent_worker();
21167 let mut task = child_parent_task("unused", Value::Null);
21168 task.history_events.truncate(1);
21169 task.total_history_events = Some(1);
21170
21171 for _redelivery_or_restart in 0..2 {
21172 let commands = worker
21173 .execute_workflow_task(task.clone())
21174 .expect("recorded child remains pending");
21175 assert!(
21176 commands.is_empty(),
21177 "recorded pending child must not be started again"
21178 );
21179 }
21180 }
21181
21182 #[test]
21183 fn child_cancellation_becomes_stable_parent_failure_command() {
21184 let worker = child_parent_worker();
21185 let task = child_parent_task(
21186 "ChildRunCancelled",
21187 json!({
21188 "sequence": 1,
21189 "child_workflow_instance_id": "wf-child",
21190 "child_workflow_run_id": "run-child",
21191 "child_workflow_type": "python.child",
21192 "failure_id": "failure-child",
21193 "failure_category": "cancelled",
21194 "message": "cancelled by parent-close policy",
21195 }),
21196 );
21197
21198 let commands = worker
21199 .execute_workflow_task(task)
21200 .expect("parent settlement");
21201 assert_eq!(commands.len(), 1);
21202 assert_eq!(commands[0]["type"], "fail_workflow");
21203 assert_eq!(commands[0]["exception_type"], "ChildWorkflowCancelled");
21204 assert_eq!(
21205 commands[0]["exception"]["properties"]["reason"],
21206 "cancelled"
21207 );
21208 assert_eq!(
21209 commands[0]["exception"]["properties"]["child_workflow_run_id"],
21210 "run-child"
21211 );
21212 }
21213
21214 #[test]
21215 fn workflow_can_handle_typed_child_failure() {
21216 let client = Client::new("http://127.0.0.1:8080").expect("client");
21217 let mut worker = Worker::new(client, "rust-parent-workers");
21218 worker.register_workflow("rust.handled-parent", |ctx, _input| async move {
21219 match ctx
21220 .start_child_workflow(
21221 "python.child",
21222 ChildWorkflowOptions::new("python-child-workers"),
21223 json!([]),
21224 )
21225 .await
21226 {
21227 Err(Error::ChildWorkflowFailed(failure)) => Ok(json!({
21228 "reason": failure.reason,
21229 "failure_id": failure.failure_id,
21230 "exception_class": failure.exception_class,
21231 "child_run_id": failure.child_workflow_run_id,
21232 })),
21233 Err(error) => Err(error),
21234 Ok(_) => Err(Error::WorkerLoop(
21235 "child unexpectedly succeeded".to_string(),
21236 )),
21237 }
21238 });
21239 let mut task = child_parent_task(
21240 "ChildRunFailed",
21241 json!({
21242 "sequence": 1,
21243 "child_workflow_instance_id": "wf-child",
21244 "child_workflow_run_id": "run-child",
21245 "child_workflow_type": "python.child",
21246 "failure_id": "failure-child",
21247 "failure_category": "child_workflow",
21248 "message": "payment rejected",
21249 "exception": {
21250 "type": "PaymentRejected",
21251 "class": "payments.PaymentRejected",
21252 "message": "payment rejected"
21253 }
21254 }),
21255 );
21256 task.workflow_type = "rust.handled-parent".to_string();
21257
21258 let commands = worker.execute_workflow_task(task).expect("handled failure");
21259 assert_eq!(commands[0]["type"], "complete_workflow");
21260 let output =
21261 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
21262 assert_eq!(output["reason"], "child_workflow");
21263 assert_eq!(output["failure_id"], "failure-child");
21264 assert_eq!(output["exception_class"], "payments.PaymentRejected");
21265 assert_eq!(output["child_run_id"], "run-child");
21266 }
21267
21268 #[test]
21269 fn rust_hello_world_uses_signal_arguments_from_resume_payload() {
21270 let client = Client::new("http://127.0.0.1:8080").expect("client");
21271 let mut worker = Worker::new(client, "rust-workers");
21272
21273 worker.register_workflow("rust.hello_workflow", |ctx, _input| async move {
21274 let signal = ctx.wait_signal("start").await?;
21275 let name = signal
21276 .first()
21277 .and_then(|value| value.as_str())
21278 .unwrap_or("world");
21279 let greeting = ctx.activity("rust.hello_activity", json!([name])).await?;
21280 Ok(json!({
21281 "greeting": greeting,
21282 "language": "rust"
21283 }))
21284 });
21285
21286 let signal_arguments =
21287 encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC).expect("signal arguments");
21288 let task = WorkflowTask {
21289 task_id: "wft-rust-signal-1".to_string(),
21290 workflow_command_id: None,
21291 workflow_id: Some("wf-rust-hello".to_string()),
21292 run_id: Some("run-rust-hello".to_string()),
21293 workflow_type: "rust.hello_workflow".to_string(),
21294 cancel_requested: false,
21295 payload_codec: DEFAULT_CODEC.to_string(),
21296 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21297 history_events: vec![HistoryEvent {
21298 event_type: "SignalReceived".to_string(),
21299 payload: json!({
21300 "signal_id": "sig-rust-1",
21301 "signal_name": "start"
21302 }),
21303 raw: HashMap::new(),
21304 }],
21305 total_history_events: Some(1),
21306 history_size_bytes: None,
21307 continue_as_new_recommended: None,
21308 history_budget_pressure: None,
21309 next_history_page_token: None,
21310 workflow_task_attempt: 1,
21311 workflow_signal_id: Some("sig-rust-1".to_string()),
21312 signal_name: Some("start".to_string()),
21313 signal_arguments: Some(signal_arguments),
21314 workflow_update_id: None,
21315 update_name: None,
21316 lease_owner: Some("rust-worker".to_string()),
21317 };
21318
21319 let commands = worker.execute_workflow_task(task).expect("workflow task");
21320
21321 assert_eq!(commands.len(), 1);
21322 assert_eq!(commands[0]["type"], "schedule_activity");
21323 assert_eq!(commands[0]["activity_type"], "rust.hello_activity");
21324 assert_eq!(
21325 decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC).expect("activity args"),
21326 json!(["Rust"])
21327 );
21328 }
21329
21330 #[test]
21331 fn workflow_task_appends_paginated_history_events() {
21332 let mut task = WorkflowTask {
21333 task_id: "wft-rust-pages-1".to_string(),
21334 workflow_command_id: None,
21335 workflow_id: Some("wf-rust-pages".to_string()),
21336 run_id: Some("run-rust-pages".to_string()),
21337 workflow_type: "rust.hello_workflow".to_string(),
21338 cancel_requested: false,
21339 payload_codec: DEFAULT_CODEC.to_string(),
21340 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21341 history_events: vec![HistoryEvent {
21342 event_type: "WorkflowStarted".to_string(),
21343 payload: json!({}),
21344 raw: HashMap::new(),
21345 }],
21346 total_history_events: Some(3),
21347 history_size_bytes: None,
21348 continue_as_new_recommended: None,
21349 history_budget_pressure: None,
21350 next_history_page_token: Some("MQ==".to_string()),
21351 workflow_task_attempt: 1,
21352 workflow_signal_id: None,
21353 signal_name: None,
21354 signal_arguments: None,
21355 workflow_update_id: None,
21356 update_name: None,
21357 lease_owner: Some("rust-worker".to_string()),
21358 };
21359
21360 task.append_history_page(WorkflowTaskHistoryPage {
21361 history_events: vec![
21362 HistoryEvent {
21363 event_type: "SignalReceived".to_string(),
21364 payload: json!({
21365 "signal_id": "sig-rust-1",
21366 "signal_name": "start",
21367 "arguments": encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC)
21368 .expect("signal arguments")
21369 }),
21370 raw: HashMap::new(),
21371 },
21372 HistoryEvent {
21373 event_type: "MarkerRecorded".to_string(),
21374 payload: json!({"sequence": 3}),
21375 raw: HashMap::new(),
21376 },
21377 ],
21378 total_history_events: Some(3),
21379 next_history_page_token: None,
21380 });
21381
21382 assert_eq!(task.history_events.len(), 3);
21383 assert_eq!(task.total_history_events, Some(3));
21384 assert_eq!(task.next_history_page_token, None);
21385
21386 let signal = task
21387 .history_events
21388 .iter()
21389 .find(|event| event.event_type == "SignalReceived")
21390 .expect("signal event");
21391 assert_eq!(
21392 decode_signal_event_arguments(signal, DEFAULT_CODEC).expect("signal arguments"),
21393 vec![AvroValue::String("Rust".to_string())]
21394 );
21395 }
21396
21397 #[tokio::test]
21398 async fn query_handler_reads_ordered_cross_codec_signals_without_commands() {
21399 let client = Client::new("http://127.0.0.1:8080").expect("client");
21400 let mut worker = Worker::new(client, "rust-workers");
21401 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21402 worker.register_query("counter", "current", |ctx, _args| async move {
21403 let mut count = 0_i64;
21404 for signal in ctx.signal_events() {
21405 let value = signal
21406 .arguments
21407 .first()
21408 .and_then(Value::as_i64)
21409 .unwrap_or_default();
21410 match signal.name.as_str() {
21411 "increment" => count += value,
21412 "set" => count = value,
21413 _ => {}
21414 }
21415 }
21416 Ok(json!(count))
21417 });
21418
21419 let task = QueryTask {
21420 query_task_id: "query-rust-counter".to_string(),
21421 query_task_attempt: 1,
21422 lease_owner: Some("rust-worker".to_string()),
21423 workflow_id: Some("counter-1".to_string()),
21424 run_id: Some("run-counter-1".to_string()),
21425 workflow_type: "counter".to_string(),
21426 query_name: "current".to_string(),
21427 payload_codec: DEFAULT_CODEC.to_string(),
21428 workflow_arguments: Some(
21429 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
21430 ),
21431 query_arguments: Some(
21432 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("query arguments"),
21433 ),
21434 history_events: vec![
21435 HistoryEvent {
21436 event_type: "SignalReceived".to_string(),
21437 payload: json!({
21438 "signal_id": "php-signal-1",
21439 "signal_name": "increment",
21440 "workflow_sequence": 1,
21441 "payload_codec": DEFAULT_CODEC,
21442 "arguments": encode_value_envelope(&json!([3]), DEFAULT_CODEC).expect("php avro signal")
21443 }),
21444 raw: HashMap::new(),
21445 },
21446 HistoryEvent {
21447 event_type: "SignalReceived".to_string(),
21448 payload: json!({
21449 "signal_id": "python-signal-2",
21450 "signal_name": "increment",
21451 "workflow_sequence": 2,
21452 "payload_codec": DEFAULT_CODEC,
21453 "arguments": encode_value_envelope(&json!([5]), DEFAULT_CODEC).expect("python avro signal")
21454 }),
21455 raw: HashMap::new(),
21456 },
21457 HistoryEvent {
21458 event_type: "SignalReceived".to_string(),
21459 payload: json!({
21460 "signal_id": "rust-signal-3",
21461 "signal_name": "set",
21462 "workflow_sequence": 3,
21463 "payload_codec": DEFAULT_CODEC,
21464 "arguments": encode_value_envelope(&json!([0]), DEFAULT_CODEC).expect("rust avro signal")
21465 }),
21466 raw: HashMap::new(),
21467 },
21468 ],
21469 history_export: None,
21470 run_status: Some("completed".to_string()),
21471 };
21472
21473 let result = worker.execute_query_task(task).await.expect("query result");
21474 assert_eq!(result.into_json().expect("query projection"), json!(0));
21475 }
21476
21477 #[tokio::test]
21478 async fn replayed_queries_read_running_completed_and_cold_restarted_instance_state() {
21479 let worker = replay_counter_worker();
21480 let running_history = json!([
21481 {
21482 "type": "ActivityCompleted",
21483 "payload": {
21484 "sequence": 1,
21485 "activity_type": "load-counter",
21486 "payload_codec": DEFAULT_CODEC,
21487 "result": fixture_envelope(json!("loaded"))
21488 }
21489 },
21490 {
21491 "type": "SignalWaitOpened",
21492 "payload": {
21493 "sequence": 3,
21494 "signal_name": "increment"
21495 }
21496 },
21497 {
21498 "type": "SignalReceived",
21499 "payload": {
21500 "signal_id": "signal-3",
21501 "signal_name": "increment",
21502 "workflow_sequence": 2,
21503 "payload_codec": DEFAULT_CODEC,
21504 "arguments": fixture_envelope(json!([3]))
21505 }
21506 },
21507 {
21508 "type": "SignalApplied",
21509 "payload": {
21510 "sequence": 3,
21511 "signal_id": "signal-3",
21512 "signal_name": "increment",
21513 "payload_codec": DEFAULT_CODEC,
21514 "value": fixture_envelope(json!([3]))
21515 }
21516 }
21517 ]);
21518
21519 let running = worker
21520 .execute_query_task(replay_counter_query(
21521 "current",
21522 running_history.clone(),
21523 "running",
21524 ))
21525 .await
21526 .expect("running replay query");
21527 assert_eq!(
21528 running.clone().into_json().expect("query projection"),
21529 json!({"loaded": "loaded", "count": 3, "finished": false})
21530 );
21531
21532 let detached = worker
21533 .execute_query_task(replay_counter_query(
21534 "detached-mutation",
21535 running_history.clone(),
21536 "running",
21537 ))
21538 .await
21539 .expect("query mutates only its detached state clone");
21540 assert_eq!(detached.into_json().expect("query projection"), json!(999));
21541 let failed = worker
21542 .execute_query_task(replay_counter_query(
21543 "failed-mutation",
21544 running_history.clone(),
21545 "running",
21546 ))
21547 .await
21548 .expect_err("failed query");
21549 assert_eq!(failed.reason, "query_rejected");
21550 let unchanged = worker
21551 .execute_query_task(replay_counter_query("current", running_history, "running"))
21552 .await
21553 .expect("later query reconstructs unchanged state");
21554 assert_eq!(unchanged, running);
21555
21556 let restarted_worker = replay_counter_worker();
21557 let empty_arguments = fixture_envelope(json!([]));
21558 let loaded_result = fixture_envelope(json!("loaded"));
21559 let signal_three = fixture_blob(json!([3]));
21560 let signal_five = fixture_blob(json!([5]));
21561 let restarted_task: QueryTask = serde_json::from_value(json!({
21562 "query_task_id": "query-after-restart",
21563 "workflow_id": "counter-1",
21564 "run_id": "run-counter-1",
21565 "workflow_type": "replay-counter",
21566 "query_name": "current",
21567 "payload_codec": DEFAULT_CODEC,
21568 "workflow_arguments": empty_arguments.clone(),
21569 "query_arguments": empty_arguments,
21570 "history_events": [],
21571 "history_export": {
21572 "payloads": {"codec": DEFAULT_CODEC},
21573 "history_events": [
21574 {
21575 "type": "ActivityCompleted",
21576 "payload": {
21577 "sequence": 1,
21578 "activity_type": "load-counter",
21579 "payload_codec": DEFAULT_CODEC,
21580 "result": null
21581 }
21582 },
21583 {
21584 "type": "SignalWaitOpened",
21585 "payload": {
21586 "sequence": 3,
21587 "signal_name": "increment"
21588 }
21589 },
21590 {
21591 "type": "SignalReceived",
21592 "payload": {
21593 "signal_id": "signal-3",
21594 "signal_name": "increment",
21595 "workflow_sequence": 2
21596 }
21597 },
21598 {
21599 "type": "SignalApplied",
21600 "payload": {
21601 "sequence": 3,
21602 "signal_id": "signal-3",
21603 "signal_name": "increment"
21604 }
21605 },
21606 {
21607 "type": "SignalWaitOpened",
21608 "payload": {
21609 "sequence": 5,
21610 "signal_name": "increment"
21611 }
21612 },
21613 {
21614 "type": "SignalReceived",
21615 "payload": {
21616 "signal_id": "signal-5",
21617 "signal_name": "increment",
21618 "workflow_sequence": 4
21619 }
21620 },
21621 {
21622 "type": "SignalApplied",
21623 "payload": {
21624 "sequence": 5,
21625 "signal_id": "signal-5",
21626 "signal_name": "increment"
21627 }
21628 }
21629 ],
21630 "activities": [{
21631 "sequence": 1,
21632 "activity_type": "load-counter",
21633 "payload_codec": DEFAULT_CODEC,
21634 "result": loaded_result
21635 }],
21636 "signals": [
21637 {
21638 "id": "signal-3",
21639 "name": "increment",
21640 "workflow_sequence": 2,
21641 "payload_codec": DEFAULT_CODEC,
21642 "arguments": signal_three
21643 },
21644 {
21645 "id": "signal-5",
21646 "name": "increment",
21647 "workflow_sequence": 4,
21648 "payload_codec": DEFAULT_CODEC,
21649 "arguments": signal_five
21650 }
21651 ]
21652 },
21653 "run_status": "completed"
21654 }))
21655 .expect("cold replay query task");
21656 let completed = restarted_worker
21657 .execute_query_task(restarted_task)
21658 .await
21659 .expect("completed cold replay query");
21660 assert_eq!(
21661 completed.into_json().expect("query projection"),
21662 json!({"loaded": "loaded", "count": 8, "finished": true})
21663 );
21664 }
21665
21666 #[tokio::test]
21667 async fn replayed_query_replay_failures_are_machine_readable() {
21668 let worker = replay_counter_worker();
21669 let task = replay_counter_query(
21670 "current",
21671 json!([{
21672 "type": "ActivityCompleted",
21673 "payload": {
21674 "sequence": 1,
21675 "payload_codec": DEFAULT_CODEC,
21676 "result": {"codec": DEFAULT_CODEC, "blob": "{"}
21677 }
21678 }]),
21679 "running",
21680 );
21681 let failure = worker
21682 .execute_query_task(task)
21683 .await
21684 .expect_err("invalid replay history payload");
21685 assert_eq!(failure.reason, "query_payload_decode_failed");
21686 assert_eq!(failure.failure_type, "QueryPayloadDecodeFailed");
21687 assert!(failure.message.contains("invalid_payload_framing"));
21688 }
21689
21690 #[tokio::test]
21691 async fn query_task_restores_compact_history_from_export() {
21692 let client = Client::new("http://127.0.0.1:8080").expect("client");
21693 let mut worker = Worker::new(client, "rust-workers");
21694 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21695 worker.register_query("counter", "current", |ctx, _args| async move {
21696 Ok(json!(ctx.signals("increment")[0][0]))
21697 });
21698 let empty_arguments = fixture_envelope(json!([]));
21699 let exported_signal = fixture_blob(json!([9]));
21700 let task: QueryTask = serde_json::from_value(json!({
21701 "query_task_id": "query-export",
21702 "workflow_type": "counter",
21703 "query_name": "current",
21704 "payload_codec": DEFAULT_CODEC,
21705 "workflow_arguments": empty_arguments.clone(),
21706 "query_arguments": empty_arguments,
21707 "history_events": [],
21708 "history_export": {
21709 "payloads": {"codec": DEFAULT_CODEC},
21710 "history_events": [{
21711 "type": "SignalReceived",
21712 "payload": {"signal_id": "signal-export", "signal_name": "increment"}
21713 }],
21714 "signals": [{
21715 "id": "signal-export",
21716 "name": "increment",
21717 "status": "applied",
21718 "workflow_sequence": 1,
21719 "payload_codec": DEFAULT_CODEC,
21720 "arguments": exported_signal
21721 }]
21722 }
21723 }))
21724 .expect("query task");
21725
21726 let result = worker.execute_query_task(task).await.expect("query result");
21727 assert_eq!(result.into_json().expect("query projection"), json!(9));
21728 }
21729
21730 #[tokio::test]
21731 async fn query_task_failures_have_stable_reasons() {
21732 let client = Client::new("http://127.0.0.1:8080").expect("client");
21733 let mut worker = Worker::new(client, "rust-workers");
21734 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21735 worker.register_query(
21736 "counter",
21737 "current",
21738 |_ctx, _args| async move { Ok(json!(0)) },
21739 );
21740
21741 let base_task = QueryTask {
21742 query_task_id: "query-errors".to_string(),
21743 query_task_attempt: 1,
21744 lease_owner: None,
21745 workflow_id: Some("counter-errors".to_string()),
21746 run_id: Some("run-errors".to_string()),
21747 workflow_type: "counter".to_string(),
21748 query_name: "missing".to_string(),
21749 payload_codec: DEFAULT_CODEC.to_string(),
21750 workflow_arguments: Some(fixture_envelope(json!([]))),
21751 query_arguments: Some(fixture_envelope(json!([]))),
21752 history_events: Vec::new(),
21753 history_export: None,
21754 run_status: Some("running".to_string()),
21755 };
21756
21757 let unknown = worker
21758 .execute_query_task(base_task.clone())
21759 .await
21760 .expect_err("unknown query");
21761 assert_eq!(unknown.reason, "rejected_unknown_query");
21762
21763 let mut malformed = base_task;
21764 malformed.query_name = "current".to_string();
21765 malformed.query_arguments = Some(json!({"codec": DEFAULT_CODEC, "blob": "{"}));
21766 let malformed = worker
21767 .execute_query_task(malformed)
21768 .await
21769 .expect_err("malformed payload");
21770 assert_eq!(malformed.reason, "query_payload_decode_failed");
21771
21772 let client = Client::new("http://127.0.0.1:8080").expect("client");
21773 let mut unavailable_worker = Worker::new(client, "rust-workers");
21774 unavailable_worker
21775 .register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21776 let empty_arguments = fixture_envelope(json!([]));
21777 let unavailable_task: QueryTask = serde_json::from_value(json!({
21778 "query_task_id": "query-unavailable",
21779 "workflow_type": "counter",
21780 "query_name": "current",
21781 "payload_codec": DEFAULT_CODEC,
21782 "workflow_arguments": empty_arguments.clone(),
21783 "query_arguments": empty_arguments
21784 }))
21785 .expect("query task");
21786 let unavailable = unavailable_worker
21787 .execute_query_task(unavailable_task)
21788 .await
21789 .expect_err("query handler unavailable");
21790 assert_eq!(unavailable.reason, "query_handler_unavailable");
21791 }
21792
21793 #[tokio::test]
21794 async fn client_query_decodes_result_and_typed_failure() {
21795 let server = MockWorkerServer::start();
21796 let client = Client::builder(server.base_url())
21797 .timeout(Duration::from_secs(2))
21798 .build()
21799 .expect("client");
21800
21801 let result = client
21802 .query_workflow("counter-1", "current", json!([]))
21803 .await
21804 .expect("query result");
21805 assert_eq!(result, json!({"count": 8}));
21806
21807 let error = client
21808 .query_workflow("counter-1", "missing", json!([]))
21809 .await
21810 .expect_err("unknown query");
21811 let Error::QueryFailed(failure) = error else {
21812 panic!("expected typed query failure");
21813 };
21814 assert_eq!(failure.status, 404);
21815 assert_eq!(failure.reason, "rejected_unknown_query");
21816 }
21817
21818 #[tokio::test]
21819 async fn public_client_surfaces_send_and_receive_lossless_avro_values() {
21820 let server = MockWorkerServer::start();
21821 let client = Client::builder(server.base_url())
21822 .timeout(Duration::from_secs(2))
21823 .build()
21824 .expect("client");
21825 let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
21826
21827 client
21828 .start_workflow(
21829 "typed.echo",
21830 "rust-workers",
21831 "typed-start",
21832 arguments.clone(),
21833 )
21834 .await
21835 .expect("typed workflow start");
21836 assert_eq!(
21837 decode_wire_avro_value(
21838 &server.request_body("/api/workflows")["input"],
21839 DEFAULT_CODEC,
21840 )
21841 .expect("typed start input"),
21842 arguments
21843 );
21844
21845 client
21846 .signal_workflow("typed-1", "changed", arguments.clone())
21847 .await
21848 .expect("typed signal");
21849 assert_eq!(
21850 decode_wire_avro_value(
21851 &server.request_body("/api/workflows/typed-1/signal/changed")["input"],
21852 DEFAULT_CODEC,
21853 )
21854 .expect("typed signal input"),
21855 arguments
21856 );
21857
21858 assert_eq!(
21859 client
21860 .query_workflow_avro_value("typed-1", "inspect", arguments.clone())
21861 .await
21862 .expect("typed query"),
21863 typed_fidelity_probe()
21864 );
21865 assert_eq!(
21866 decode_wire_avro_value(
21867 &server.request_body("/api/workflows/typed-1/query/inspect")["input"],
21868 DEFAULT_CODEC,
21869 )
21870 .expect("typed query input"),
21871 arguments
21872 );
21873
21874 assert_eq!(
21875 client
21876 .update_workflow_avro_value(
21877 "typed-1",
21878 "replace",
21879 arguments.clone(),
21880 Some("typed-request"),
21881 )
21882 .await
21883 .expect("typed update"),
21884 typed_fidelity_probe()
21885 );
21886 let update = server.request_body("/api/workflows/typed-1/update/replace");
21887 assert_eq!(update["request_id"], "typed-request");
21888 assert_eq!(
21889 decode_wire_avro_value(&update["input"], DEFAULT_CODEC).expect("typed update input"),
21890 arguments
21891 );
21892
21893 let handle = WorkflowHandle {
21894 client: client.clone(),
21895 workflow_id: "typed-1".to_string(),
21896 run_id: Some("run-typed-1".to_string()),
21897 workflow_type: "typed.echo".to_string(),
21898 };
21899 assert_eq!(
21900 handle
21901 .result_avro_value(WorkflowResultOptions::default())
21902 .await
21903 .expect("typed workflow result"),
21904 typed_fidelity_probe()
21905 );
21906
21907 client
21908 .complete_activity_task(
21909 "activity-typed",
21910 "attempt-typed",
21911 "rust-worker",
21912 typed_fidelity_probe(),
21913 DEFAULT_CODEC,
21914 )
21915 .await
21916 .expect("typed activity completion");
21917 assert_eq!(
21918 decode_wire_avro_value(
21919 &server.request_body("/api/worker/activity-tasks/activity-typed/complete")
21920 ["result"],
21921 DEFAULT_CODEC,
21922 )
21923 .expect("typed activity result"),
21924 typed_fidelity_probe()
21925 );
21926 client
21927 .fail_activity_task(
21928 "activity-typed",
21929 "attempt-typed",
21930 "rust-worker",
21931 "typed failure",
21932 true,
21933 )
21934 .await
21935 .expect("activity failure");
21936 }
21937
21938 #[tokio::test]
21939 async fn lifecycle_commands_support_instance_and_selected_run_targets() {
21940 let server = MockWorkerServer::start();
21941 let client = Client::builder(server.base_url())
21942 .timeout(Duration::from_secs(2))
21943 .build()
21944 .expect("client");
21945
21946 let options = WorkflowCommandOptions::new()
21947 .reason("cleanup requested")
21948 .request_id("cancel-17");
21949 let cancelled = client
21950 .cancel_workflow("wf-lifecycle", options)
21951 .await
21952 .expect("instance cancellation");
21953 assert_eq!(cancelled.command, WorkflowCommandKind::Cancel);
21954 assert_eq!(cancelled.run_id.as_deref(), Some("run-current"));
21955 assert_eq!(cancelled.outcome.as_deref(), Some("cancelled"));
21956 assert_eq!(
21957 server.request_body("/api/workflows/wf-lifecycle/cancel"),
21958 json!({"reason":"cleanup requested","request_id":"cancel-17"})
21959 );
21960
21961 let terminated = client
21962 .terminate_workflow(
21963 "wf-lifecycle",
21964 WorkflowCommandOptions::new().reason("forced stop"),
21965 )
21966 .await
21967 .expect("instance termination");
21968 assert_eq!(terminated.command, WorkflowCommandKind::Terminate);
21969 assert_eq!(terminated.outcome.as_deref(), Some("terminated"));
21970
21971 client
21972 .cancel_workflow_run(
21973 "wf-lifecycle",
21974 "run-current",
21975 WorkflowCommandOptions::default(),
21976 )
21977 .await
21978 .expect("selected run cancellation");
21979 client
21980 .terminate_workflow_run(
21981 "wf-lifecycle",
21982 "run-current",
21983 WorkflowCommandOptions::default(),
21984 )
21985 .await
21986 .expect("selected run termination");
21987
21988 for (command, error) in [
21989 (
21990 WorkflowCommandKind::Cancel,
21991 client
21992 .cancel_workflow_run(
21993 "wf-lifecycle",
21994 "run-stale",
21995 WorkflowCommandOptions::default(),
21996 )
21997 .await
21998 .expect_err("stale cancellation must be rejected"),
21999 ),
22000 (
22001 WorkflowCommandKind::Terminate,
22002 client
22003 .terminate_workflow_run(
22004 "wf-lifecycle",
22005 "run-stale",
22006 WorkflowCommandOptions::default(),
22007 )
22008 .await
22009 .expect_err("stale termination must be rejected"),
22010 ),
22011 ] {
22012 let Error::WorkflowCommandRejected(rejection) = error else {
22013 panic!("expected typed command rejection");
22014 };
22015 assert_eq!(rejection.command, command);
22016 assert_eq!(rejection.status, 409);
22017 assert_eq!(rejection.reason, "historical_run_command_rejected");
22018 assert_eq!(rejection.run_id.as_deref(), Some("run-stale"));
22019 assert_eq!(rejection.target_scope.as_deref(), Some("run"));
22020 }
22021 }
22022
22023 #[tokio::test]
22024 async fn workflow_start_options_send_server_enforced_deadlines() {
22025 let server = MockWorkerServer::start();
22026 let client = Client::builder(server.base_url())
22027 .timeout(Duration::from_secs(2))
22028 .build()
22029 .expect("client");
22030
22031 let handle = client
22032 .start_workflow_with_options(
22033 "rust.timeout",
22034 "rust-timeouts",
22035 "wf-start-options",
22036 WorkflowStartOptions::new()
22037 .execution_timeout_seconds(30)
22038 .run_timeout_seconds(1),
22039 json!([]),
22040 )
22041 .await
22042 .expect("workflow start");
22043
22044 assert_eq!(handle.run_id.as_deref(), Some("run-start-options"));
22045 let body = server.request_body("/api/workflows");
22046 assert_eq!(body["execution_timeout_seconds"], 30);
22047 assert_eq!(body["run_timeout_seconds"], 1);
22048
22049 let invalid = client
22050 .start_workflow_with_options(
22051 "rust.timeout",
22052 "rust-timeouts",
22053 "wf-invalid-options",
22054 WorkflowStartOptions::new()
22055 .execution_timeout_seconds(1)
22056 .run_timeout_seconds(2),
22057 json!([]),
22058 )
22059 .await
22060 .expect_err("invalid deadline ordering");
22061 assert!(invalid
22062 .to_string()
22063 .contains("run_timeout_seconds cannot exceed execution_timeout_seconds"));
22064 }
22065
22066 #[tokio::test]
22067 async fn workflow_result_returns_each_typed_terminal_outcome() {
22068 let server = MockWorkerServer::start();
22069 let client = Client::builder(server.base_url())
22070 .timeout(Duration::from_secs(2))
22071 .build()
22072 .expect("client");
22073 let options = WorkflowResultOptions {
22074 poll_interval: Duration::ZERO,
22075 timeout: Duration::from_secs(1),
22076 };
22077
22078 let failed = WorkflowHandle {
22079 client: client.clone(),
22080 workflow_id: "wf-failed".to_string(),
22081 run_id: Some("run-failed".to_string()),
22082 workflow_type: "failure".to_string(),
22083 }
22084 .result(options)
22085 .await
22086 .expect_err("failed outcome");
22087 let Error::WorkflowFailed(failure) = failed else {
22088 panic!("expected WorkflowFailed");
22089 };
22090 assert_eq!(failure.workflow_id, "wf-failed");
22091 assert_eq!(failure.run_id.as_deref(), Some("run-failed"));
22092 assert_eq!(failure.failure_id.as_deref(), Some("failure-17"));
22093 assert_eq!(failure.failure_category.as_deref(), Some("application"));
22094 assert_eq!(failure.exception_type.as_deref(), Some("PaymentError"));
22095 assert_eq!(
22096 failure.exception_class.as_deref(),
22097 Some("billing::PaymentError")
22098 );
22099 assert_eq!(failure.non_retryable, Some(true));
22100
22101 for (workflow_id, expected_kind, expected_reason) in [
22102 (
22103 "wf-cancelled",
22104 WorkflowTerminalKind::Cancelled,
22105 "cleanup requested",
22106 ),
22107 (
22108 "wf-terminated",
22109 WorkflowTerminalKind::Terminated,
22110 "forced stop",
22111 ),
22112 (
22113 "wf-timed-out",
22114 WorkflowTerminalKind::TimedOut,
22115 "run_timeout",
22116 ),
22117 ] {
22118 let error = WorkflowHandle {
22119 client: client.clone(),
22120 workflow_id: workflow_id.to_string(),
22121 run_id: None,
22122 workflow_type: "terminal".to_string(),
22123 }
22124 .result(options)
22125 .await
22126 .expect_err("typed terminal outcome");
22127 let outcome = match error {
22128 Error::WorkflowCancelled(outcome) => outcome,
22129 Error::WorkflowTerminated(outcome) => outcome,
22130 Error::WorkflowTimedOut(outcome) => outcome,
22131 other => panic!("unexpected terminal error: {other}"),
22132 };
22133 assert_eq!(outcome.kind, expected_kind);
22134 assert_eq!(outcome.workflow_id, workflow_id);
22135 assert_eq!(outcome.reason, expected_reason);
22136 }
22137
22138 let wait_timeout = WorkflowHandle {
22139 client,
22140 workflow_id: "wf-waiting".to_string(),
22141 run_id: Some("run-waiting".to_string()),
22142 workflow_type: "waiting".to_string(),
22143 }
22144 .result(WorkflowResultOptions {
22145 poll_interval: Duration::ZERO,
22146 timeout: Duration::ZERO,
22147 })
22148 .await
22149 .expect_err("client wait timeout");
22150 let Error::WorkflowTimedOut(timeout) = wait_timeout else {
22151 panic!("expected typed client timeout");
22152 };
22153 assert_eq!(timeout.reason, "result_wait_timeout");
22154 assert_eq!(timeout.failure_category.as_deref(), Some("client_timeout"));
22155 assert_eq!(timeout.run_id.as_deref(), Some("run-waiting"));
22156 }
22157
22158 #[tokio::test]
22159 async fn workflow_result_follows_chain_and_selected_result_preserves_history() {
22160 let server = MockWorkerServer::start();
22161 let client = Client::builder(server.base_url())
22162 .timeout(Duration::from_secs(2))
22163 .build()
22164 .expect("client");
22165
22166 let handle = WorkflowHandle {
22167 client,
22168 workflow_id: "wf-selected".to_string(),
22169 run_id: Some("run-selected".to_string()),
22170 workflow_type: "selected".to_string(),
22171 };
22172 let options = WorkflowResultOptions {
22173 poll_interval: Duration::ZERO,
22174 timeout: Duration::from_secs(1),
22175 };
22176
22177 let current = handle
22178 .result(options)
22179 .await
22180 .expect("instance result follows the current run");
22181 assert_eq!(current, json!("current run output"));
22182
22183 let error = handle
22184 .result_selected_run(options)
22185 .await
22186 .expect_err("the selected run is cancelled even though the current run completed");
22187
22188 let Error::WorkflowCancelled(outcome) = error else {
22189 panic!("expected selected run cancellation");
22190 };
22191 assert_eq!(outcome.run_id.as_deref(), Some("run-selected"));
22192 assert_eq!(outcome.reason, "selected run cancelled");
22193 assert_eq!(
22194 server.request_count("/api/workflows/wf-selected/runs/run-selected"),
22195 1
22196 );
22197 assert_eq!(server.request_count("/api/workflows/wf-selected"), 1);
22198 }
22199
22200 #[tokio::test]
22201 async fn poll_responses_decode_http_conflict_drain_as_a_stable_stop() {
22202 let server = MockWorkerServer::draining_polls();
22203 let client = Client::builder(server.base_url())
22204 .timeout(Duration::from_secs(2))
22205 .build()
22206 .expect("client");
22207
22208 let workflow = client
22209 .poll_workflow_task_response("draining-worker", "rust-workers", Duration::ZERO)
22210 .await
22211 .expect("workflow drain response");
22212 let activity = client
22213 .poll_activity_task_response("draining-worker", "rust-workers", Duration::ZERO)
22214 .await
22215 .expect("activity drain response");
22216 let query = client
22217 .poll_query_task_response("draining-worker", "rust-workers", Duration::ZERO)
22218 .await
22219 .expect("query drain response");
22220
22221 for outcome in [workflow.outcome(), activity.outcome(), query.outcome()] {
22222 assert_eq!(
22223 outcome,
22224 WorkerPollOutcome::Stop {
22225 poll_status: Some("draining".to_string()),
22226 reason: Some("worker_draining".to_string()),
22227 }
22228 );
22229 }
22230
22231 assert!(client
22232 .poll_workflow_task("draining-worker", "rust-workers", Duration::ZERO)
22233 .await
22234 .expect("compatibility poll")
22235 .is_none());
22236 }
22237
22238 #[tokio::test]
22239 async fn managed_worker_honors_drain_stop_for_every_task_family() {
22240 let server = MockWorkerServer::draining_polls();
22241 let client = Client::builder(server.base_url())
22242 .timeout(Duration::from_secs(2))
22243 .build()
22244 .expect("client");
22245
22246 let mut workflow_worker = Worker::new(client.clone(), "rust-workers")
22247 .worker_id("draining-workflow-worker")
22248 .poll_timeout(Duration::ZERO);
22249 workflow_worker.register_workflow("counter", |_ctx, _args| async { Ok(Value::Null) });
22250 workflow_worker
22251 .run()
22252 .await
22253 .expect("workflow drain is a clean stop");
22254
22255 let mut activity_worker = Worker::new(client.clone(), "rust-workers")
22256 .worker_id("draining-activity-worker")
22257 .poll_timeout(Duration::ZERO);
22258 activity_worker.register_activity("write", |_ctx, _args| async { Ok(Value::Null) });
22259 activity_worker
22260 .run()
22261 .await
22262 .expect("activity drain is a clean stop");
22263
22264 let mut query_worker = Worker::new(client, "rust-workers")
22265 .worker_id("draining-query-worker")
22266 .poll_timeout(Duration::ZERO);
22267 query_worker.register_query("counter", "current", |_ctx, _args| async {
22268 Ok(Value::Null)
22269 });
22270 query_worker
22271 .run()
22272 .await
22273 .expect("query drain is a clean stop");
22274 }
22275
22276 #[tokio::test]
22277 async fn activity_cancellation_and_late_completion_remain_machine_readable() {
22278 let server = MockWorkerServer::start();
22279 let client = Client::builder(server.base_url())
22280 .timeout(Duration::from_secs(2))
22281 .build()
22282 .expect("client");
22283
22284 let heartbeat = client
22285 .heartbeat_activity_task(
22286 "activity-cancel",
22287 "attempt-cancel",
22288 "rust-worker",
22289 typed_fidelity_probe(),
22290 )
22291 .await
22292 .expect("cancellation heartbeat");
22293 assert!(heartbeat.cancel_requested);
22294 assert!(heartbeat.should_stop());
22295 assert_eq!(heartbeat.reason.as_deref(), Some("run_cancelled"));
22296 assert_eq!(heartbeat.run_closed_reason.as_deref(), Some("cancelled"));
22297 let heartbeat_body =
22298 server.request_body("/api/worker/activity-tasks/activity-cancel/heartbeat");
22299 assert_eq!(heartbeat_body["details"]["codec"], DEFAULT_CODEC);
22300 assert_eq!(
22301 decode_wire_avro_value(&heartbeat_body["details"], DEFAULT_CODEC)
22302 .expect("typed heartbeat details"),
22303 typed_fidelity_probe()
22304 );
22305
22306 let error = client
22307 .complete_activity_task(
22308 "activity-cancel",
22309 "attempt-cancel",
22310 "rust-worker",
22311 json!({"late":true}),
22312 DEFAULT_CODEC,
22313 )
22314 .await
22315 .expect_err("late completion must be refused");
22316 assert!(activity_task_rejection_is_final(&error));
22317 let Error::ActivityTaskRejected(rejection) = error else {
22318 panic!("expected typed activity rejection");
22319 };
22320 assert_eq!(rejection.status, 409);
22321 assert_eq!(rejection.reason, "run_cancelled");
22322 assert!(rejection.cancel_requested);
22323 assert_eq!(rejection.can_continue, Some(false));
22324 }
22325
22326 #[tokio::test]
22327 async fn managed_worker_survives_late_completion_and_restart_during_cancellation() {
22328 let server = MockWorkerServer::cancelled_activity();
22329 let client = Client::builder(server.base_url())
22330 .timeout(Duration::from_secs(2))
22331 .build()
22332 .expect("client");
22333 let cancellation_observed = Arc::new(AtomicBool::new(false));
22334 let observed = Arc::clone(&cancellation_observed);
22335 let mut worker = Worker::new(client.clone(), "rust-workers")
22336 .worker_id("rust-cancel-worker")
22337 .poll_timeout(Duration::from_millis(10));
22338 worker.register_activity("cancel-aware", move |ctx, _args| {
22339 let observed = Arc::clone(&observed);
22340 async move {
22341 let heartbeat = ctx.heartbeat(json!({"stage":"running"})).await?;
22342 observed.store(heartbeat.should_stop(), Ordering::SeqCst);
22343 Ok(json!({"late":"completion"}))
22344 }
22345 });
22346
22347 assert_eq!(
22348 worker.run_once().await.expect("cancelled attempt handled"),
22349 1
22350 );
22351 assert!(cancellation_observed.load(Ordering::SeqCst));
22352 assert_eq!(
22353 server.request_count("/api/worker/activity-tasks/activity-cancel/complete"),
22354 1
22355 );
22356
22357 let mut restarted = Worker::new(client, "rust-workers")
22358 .worker_id("rust-cancel-worker-restarted")
22359 .poll_timeout(Duration::from_millis(10));
22360 restarted.register_activity("cancel-aware", |_ctx, _args| async move { Ok(Value::Null) });
22361 assert_eq!(
22362 restarted
22363 .run_once()
22364 .await
22365 .expect("replacement worker continues polling"),
22366 0
22367 );
22368 }
22369
22370 #[tokio::test]
22371 async fn managed_worker_absorbs_selected_run_terminal_timeout_completion_race() {
22372 let response = r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"outcome":"completed","recorded":false,"run_id":"run-selected-timeout","run_status":"failed","created_task_ids":[],"reason":"run_timed_out"}"#;
22373 let server = MockWorkerServer::workflow_completion("409 Conflict", response);
22374 let client = Client::builder(server.base_url())
22375 .timeout(Duration::from_secs(2))
22376 .build()
22377 .expect("client");
22378
22379 let direct_error = client
22380 .complete_workflow_task(
22381 "workflow-timeout-task",
22382 "timeout-worker",
22383 3,
22384 vec![json!({
22385 "type": "complete_workflow",
22386 "result": fixture_envelope(Value::Null)
22387 })],
22388 )
22389 .await
22390 .expect_err("the low-level client preserves the completion rejection");
22391 let Error::Http { status, body } = direct_error else {
22392 panic!("expected the original HTTP completion rejection");
22393 };
22394 assert_eq!(status, reqwest::StatusCode::CONFLICT);
22395 assert_eq!(
22396 serde_json::from_str::<Value>(&body).expect("response body")["reason"],
22397 "run_timed_out"
22398 );
22399
22400 let mut worker = Worker::new(client, "rust-workers")
22401 .worker_id("timeout-worker")
22402 .poll_timeout(Duration::from_millis(10));
22403 worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
22404 Ok(json!({"late": "result"}))
22405 });
22406
22407 assert_eq!(
22408 worker
22409 .run_once()
22410 .await
22411 .expect("authoritative selected-run timeout settles the tick"),
22412 1
22413 );
22414 assert_eq!(
22415 server.request_count("/api/worker/workflow-tasks/workflow-timeout-task/complete"),
22416 2,
22417 "both the direct client proof and managed worker must see the rejection"
22418 );
22419 }
22420
22421 #[tokio::test]
22422 async fn managed_worker_does_not_swallow_nearby_completion_errors() {
22423 for (name, status, response) in [
22424 ("bare conflict", "409 Conflict", r#"{"message":"conflict"}"#),
22425 (
22426 "command was recorded",
22427 "409 Conflict",
22428 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":true,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22429 ),
22430 (
22431 "lease conflict",
22432 "409 Conflict",
22433 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"lease_expired"}"#,
22434 ),
22435 (
22436 "nonterminal run",
22437 "409 Conflict",
22438 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"waiting","reason":"run_timed_out"}"#,
22439 ),
22440 (
22441 "different selected run",
22442 "409 Conflict",
22443 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-reused-workflow-current","run_status":"failed","reason":"run_timed_out"}"#,
22444 ),
22445 (
22446 "different task attempt",
22447 "409 Conflict",
22448 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":4,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22449 ),
22450 (
22451 "authentication failure",
22452 "401 Unauthorized",
22453 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22454 ),
22455 (
22456 "authorization failure",
22457 "403 Forbidden",
22458 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22459 ),
22460 (
22461 "protocol failure",
22462 "400 Bad Request",
22463 r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.2","requested_version":"1.3"}"#,
22464 ),
22465 (
22466 "malformed command",
22467 "422 Unprocessable Entity",
22468 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22469 ),
22470 (
22471 "transient server failure",
22472 "503 Service Unavailable",
22473 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22474 ),
22475 ] {
22476 let server = MockWorkerServer::workflow_completion(status, response);
22477 let client = Client::builder(server.base_url())
22478 .timeout(Duration::from_secs(2))
22479 .build()
22480 .expect("client");
22481 let mut worker = Worker::new(client, "rust-workers")
22482 .worker_id("timeout-worker")
22483 .poll_timeout(Duration::from_millis(10));
22484 worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
22485 Ok(json!({"late": "result"}))
22486 });
22487
22488 let error = worker
22489 .run_once()
22490 .await
22491 .expect_err(&format!("{name} must remain an error"));
22492 assert!(
22493 matches!(error, Error::Http { .. } | Error::Protocol(_)),
22494 "{name} returned an unexpected error variant: {error}"
22495 );
22496 }
22497 }
22498
22499 #[tokio::test]
22500 async fn worker_deregistration_uses_worker_plane_method_path_headers_and_result() {
22501 let server = MockWorkerServer::start();
22502 let client = Client::builder(server.base_url())
22503 .worker_token(Some("worker-secret".to_string()))
22504 .namespace("orders")
22505 .timeout(Duration::from_secs(2))
22506 .build()
22507 .expect("client");
22508 let path = "/api/worker/registrations/worker%2F%CE%B1%20space";
22509
22510 let result = client
22511 .deregister_worker_registration("worker/α space")
22512 .await
22513 .expect("deregister worker registration");
22514
22515 assert_eq!(server.method_for(path).as_deref(), Some("DELETE"));
22516 assert_eq!(
22517 server.worker_protocol_for(path).as_deref(),
22518 Some(WORKER_PROTOCOL_VERSION)
22519 );
22520 assert_eq!(server.control_protocol_for(path), None);
22521 assert_eq!(server.namespace_for(path).as_deref(), Some("orders"));
22522 assert_eq!(
22523 server.authorization_for(path).as_deref(),
22524 Some("Bearer worker-secret")
22525 );
22526 assert_eq!(
22527 result,
22528 WorkerDeregistrationEnvelope {
22529 worker_id: "deregistered-worker".to_string(),
22530 outcome: "deregistered".to_string(),
22531 recovered_workflow_task_count: 2,
22532 }
22533 );
22534 }
22535
22536 #[tokio::test]
22537 async fn low_level_registration_rejects_update_validators_before_transport() {
22538 let server = MockWorkerServer::start();
22539 let client = Client::builder(server.base_url())
22540 .timeout(Duration::from_secs(2))
22541 .build()
22542 .expect("client");
22543
22544 for update_validators in [json!(["approve"]), json!("approve")] {
22545 let error = client
22546 .register_worker_with_command_contracts(
22547 "validator-claiming-worker",
22548 "rust-workers",
22549 vec!["orders".to_string()],
22550 vec![],
22551 1,
22552 1,
22553 vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
22554 json!({
22555 "orders": {
22556 "queries": ["current"],
22557 "updates": ["approve"],
22558 "update_validators": update_validators,
22559 },
22560 }),
22561 )
22562 .await
22563 .expect_err("unsupported validator claims must fail before registration");
22564
22565 let Error::UnsupportedUpdateValidators { workflow_type } = error else {
22566 panic!("expected typed unsupported-validator failure");
22567 };
22568 assert_eq!(workflow_type, "orders");
22569 }
22570 assert_eq!(server.request_count("/api/worker/register"), 0);
22571 }
22572
22573 #[tokio::test]
22574 async fn low_level_registration_preserves_query_and_update_contracts() {
22575 let server = MockWorkerServer::start();
22576 let client = Client::builder(server.base_url())
22577 .timeout(Duration::from_secs(2))
22578 .build()
22579 .expect("client");
22580 let contracts = json!({
22581 "orders": {
22582 "queries": ["current"],
22583 "updates": ["approve"],
22584 "update_validators": [],
22585 },
22586 "payments": {
22587 "queries": ["status"],
22588 "updates": ["capture"],
22589 },
22590 });
22591
22592 client
22593 .register_worker_with_command_contracts(
22594 "command-worker",
22595 "rust-workers",
22596 vec!["orders".to_string(), "payments".to_string()],
22597 vec![],
22598 1,
22599 1,
22600 vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
22601 contracts.clone(),
22602 )
22603 .await
22604 .expect("query and update contracts must remain supported");
22605
22606 assert_eq!(
22607 server.request_body("/api/worker/register")["workflow_command_contracts"],
22608 contracts
22609 );
22610 }
22611
22612 #[tokio::test]
22613 async fn role_scoped_tokens_are_never_used_for_the_opposite_plane() {
22614 let server = MockWorkerServer::start();
22615 let control_only = Client::builder(server.base_url())
22616 .control_token(Some("control-secret".to_string()))
22617 .build()
22618 .expect("control client");
22619
22620 let error = control_only
22621 .register_worker("worker", "queue", vec![], vec![], 1, 1)
22622 .await
22623 .expect_err("control token must not authorize a worker request");
22624 assert!(matches!(
22625 error,
22626 Error::MissingRoleCredentials { role: "worker", .. }
22627 ));
22628 assert_eq!(server.request_count("/api/worker/register"), 0);
22629
22630 let worker_only = Client::builder(server.base_url())
22631 .worker_token(Some("worker-secret".to_string()))
22632 .build()
22633 .expect("worker client");
22634 let error = worker_only
22635 .health()
22636 .await
22637 .expect_err("worker token must not authorize a control request");
22638 assert!(matches!(
22639 error,
22640 Error::MissingRoleCredentials {
22641 role: "control",
22642 ..
22643 }
22644 ));
22645 assert_eq!(server.request_count("/api/health"), 0);
22646 }
22647
22648 #[tokio::test]
22649 async fn shared_token_supports_worker_and_control_planes() {
22650 let server = MockWorkerServer::start();
22651 let client = Client::builder(server.base_url())
22652 .token(Some("shared-secret".to_string()))
22653 .build()
22654 .expect("client");
22655
22656 client.health().await.expect("control request");
22657 client
22658 .register_worker("worker", "queue", vec![], vec![], 1, 1)
22659 .await
22660 .expect("worker request");
22661
22662 assert_eq!(
22663 server.authorization_for("/api/health").as_deref(),
22664 Some("Bearer shared-secret")
22665 );
22666 assert_eq!(
22667 server.control_protocol_for("/api/health").as_deref(),
22668 Some(CONTROL_PLANE_VERSION)
22669 );
22670 assert_eq!(
22671 server.authorization_for("/api/worker/register").as_deref(),
22672 Some("Bearer shared-secret")
22673 );
22674 assert_eq!(
22675 server
22676 .worker_protocol_for("/api/worker/register")
22677 .as_deref(),
22678 Some(WORKER_PROTOCOL_VERSION)
22679 );
22680 }
22681
22682 #[tokio::test]
22683 async fn baseline_worker_endpoints_send_the_baseline_protocol() {
22684 let server = MockWorkerServer::start();
22685 let client = Client::builder(server.base_url())
22686 .timeout(Duration::from_secs(2))
22687 .build()
22688 .expect("client");
22689
22690 client
22691 .register_worker("capture-worker", "capture", vec![], vec![], 1, 1)
22692 .await
22693 .expect("register");
22694 client
22695 .heartbeat_worker("capture-worker", 1, 1)
22696 .await
22697 .expect("heartbeat");
22698 client
22699 .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
22700 .await
22701 .expect("workflow poll");
22702 client
22703 .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
22704 .await
22705 .expect("activity poll");
22706
22707 for path in [
22708 "/api/worker/register",
22709 "/api/worker/heartbeat",
22710 "/api/worker/workflow-tasks/poll",
22711 "/api/worker/activity-tasks/poll",
22712 ] {
22713 assert_eq!(
22714 server.worker_protocol_for(path).as_deref(),
22715 Some(WORKER_PROTOCOL_VERSION),
22716 "unexpected protocol for {path}"
22717 );
22718 }
22719
22720 assert_eq!(
22721 server.request_body("/api/worker/workflow-tasks/poll")["timeout_seconds"],
22722 1
22723 );
22724 assert_eq!(
22725 server.request_body("/api/worker/activity-tasks/poll")["timeout_seconds"],
22726 1
22727 );
22728 assert!(
22729 server.request_body("/api/worker/workflow-tasks/poll")["poll_request_id"]
22730 .as_str()
22731 .is_some_and(|id| id.starts_with("rust-workflow-poll-"))
22732 );
22733 assert!(
22734 server.request_body("/api/worker/activity-tasks/poll")["poll_request_id"]
22735 .as_str()
22736 .is_some_and(|id| id.starts_with("rust-activity-poll-"))
22737 );
22738 }
22739
22740 #[tokio::test]
22741 async fn query_task_endpoints_send_the_query_feature_protocol() {
22742 let server = MockWorkerServer::start();
22743 let client = Client::builder(server.base_url())
22744 .timeout(Duration::from_secs(2))
22745 .build()
22746 .expect("client");
22747
22748 client
22749 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
22750 .await
22751 .expect("query poll");
22752 client
22753 .complete_query_task(
22754 "query-capture",
22755 "capture-worker",
22756 1,
22757 json!(8),
22758 DEFAULT_CODEC,
22759 )
22760 .await
22761 .expect("query complete");
22762 client
22763 .fail_query_task(
22764 "query-capture",
22765 "capture-worker",
22766 1,
22767 "failed",
22768 "query_rejected",
22769 "QueryFailed",
22770 )
22771 .await
22772 .expect("query fail");
22773
22774 for path in [
22775 "/api/worker/query-tasks/poll",
22776 "/api/worker/query-tasks/query-capture/complete",
22777 "/api/worker/query-tasks/query-capture/fail",
22778 ] {
22779 assert_eq!(
22780 server.worker_protocol_for(path).as_deref(),
22781 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
22782 "unexpected protocol for {path}"
22783 );
22784 }
22785
22786 assert_eq!(
22787 server.request_body("/api/worker/query-tasks/poll")["timeout_seconds"],
22788 1
22789 );
22790 assert!(
22791 server.request_body("/api/worker/query-tasks/poll")["poll_request_id"]
22792 .as_str()
22793 .is_some_and(|id| id.starts_with("rust-query-poll-"))
22794 );
22795 }
22796
22797 #[tokio::test]
22798 async fn disconnected_client_polls_retry_once_with_the_same_request_id() {
22799 let server = MockWorkerServer::transient_worker_failures();
22800 let client = Client::builder(server.base_url())
22801 .timeout(Duration::from_secs(2))
22802 .build()
22803 .expect("client");
22804
22805 client
22806 .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
22807 .await
22808 .expect("workflow poll retry");
22809 client
22810 .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
22811 .await
22812 .expect("activity poll retry");
22813 client
22814 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
22815 .await
22816 .expect("query poll retry");
22817
22818 for path in [
22819 "/api/worker/workflow-tasks/poll",
22820 "/api/worker/activity-tasks/poll",
22821 "/api/worker/query-tasks/poll",
22822 ] {
22823 let bodies = server.request_bodies(path);
22824 assert_eq!(bodies.len(), 2, "{path} must be retried once");
22825 assert_eq!(
22826 bodies[0]["poll_request_id"], bodies[1]["poll_request_id"],
22827 "{path} must preserve the request binding across retry"
22828 );
22829 }
22830 }
22831
22832 #[tokio::test]
22833 async fn worker_poll_retries_preserve_request_id_across_consecutive_failures() {
22834 let server = MockWorkerServer::consecutive_poll_failures(2);
22835 let client = Client::builder(server.base_url())
22836 .timeout(Duration::from_secs(2))
22837 .build()
22838 .expect("client");
22839 let mut worker = Worker::new(client, "capture")
22840 .worker_id("capture-worker")
22841 .poll_timeout(Duration::from_millis(10))
22842 .retry_policy(WorkerRetryPolicy {
22843 max_retries: 2,
22844 initial_backoff: Duration::from_millis(1),
22845 max_backoff: Duration::from_millis(1),
22846 });
22847 worker.register_workflow(
22848 "capture.workflow",
22849 |_ctx, _input| async move { Ok(Value::Null) },
22850 );
22851 worker.register_activity(
22852 "capture.activity",
22853 |_ctx, _input| async move { Ok(Value::Null) },
22854 );
22855 worker.register_query("capture.workflow", "current", |_ctx, _args| async move {
22856 Ok(Value::Null)
22857 });
22858
22859 assert_eq!(worker.run_once().await.expect("poll retries"), 0);
22860
22861 for path in [
22862 "/api/worker/workflow-tasks/poll",
22863 "/api/worker/activity-tasks/poll",
22864 "/api/worker/query-tasks/poll",
22865 ] {
22866 let bodies = server.request_bodies(path);
22867 assert_eq!(bodies.len(), 3, "{path} must use exactly two retries");
22868 assert!(
22869 bodies
22870 .iter()
22871 .all(|body| body["poll_request_id"] == bodies[0]["poll_request_id"]),
22872 "{path} must preserve one request binding across every retry"
22873 );
22874 }
22875 }
22876
22877 #[tokio::test]
22878 async fn query_protocol_rejection_from_older_server_is_typed() {
22879 let server = MockWorkerServer::reject_query_protocol();
22880 let client = Client::builder(server.base_url())
22881 .timeout(Duration::from_secs(2))
22882 .build()
22883 .expect("client");
22884
22885 let error = client
22886 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
22887 .await
22888 .expect_err("server below query protocol floor must reject");
22889 let Error::Protocol(failure) = error else {
22890 panic!("expected typed protocol failure");
22891 };
22892
22893 assert_eq!(failure.status, 400);
22894 assert_eq!(failure.reason, "unsupported_protocol_version");
22895 assert_eq!(failure.supported_version.as_deref(), Some("1.7"));
22896 assert_eq!(
22897 failure.requested_version.as_deref(),
22898 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
22899 );
22900 assert_eq!(
22901 server
22902 .worker_protocol_for("/api/worker/query-tasks/poll")
22903 .as_deref(),
22904 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
22905 );
22906 }
22907
22908 #[tokio::test]
22909 async fn run_once_without_query_handlers_keeps_pre_query_server_compatibility() {
22910 let server = MockWorkerServer::reject_query_protocol();
22911 let client = Client::builder(server.base_url())
22912 .timeout(Duration::from_secs(2))
22913 .build()
22914 .expect("client");
22915 let mut worker = Worker::new(client, "rust-workers")
22916 .worker_id("baseline-worker")
22917 .poll_timeout(Duration::from_millis(10));
22918
22919 worker.register_workflow("baseline.workflow", |_ctx, _input| async move {
22920 Ok(Value::Null)
22921 });
22922
22923 assert_eq!(worker.run_once().await.expect("baseline run once"), 0);
22924 assert_eq!(
22925 server
22926 .worker_protocol_for("/api/worker/workflow-tasks/poll")
22927 .as_deref(),
22928 Some(WORKER_PROTOCOL_VERSION)
22929 );
22930 assert_eq!(
22931 server.worker_protocol_for("/api/worker/query-tasks/poll"),
22932 None,
22933 "a worker without query handlers must not use the query-task endpoint"
22934 );
22935 }
22936
22937 #[tokio::test]
22938 async fn completion_time_query_rejection_is_typed_without_stopping_worker() {
22939 let server = MockWorkerServer::reject_query_completion();
22940 let client = Client::builder(server.base_url())
22941 .timeout(Duration::from_secs(2))
22942 .build()
22943 .expect("client");
22944
22945 let error = client
22946 .complete_query_task("query-late", "late-worker", 1, json!(8), DEFAULT_CODEC)
22947 .await
22948 .expect_err("expired completion must be rejected");
22949 let Error::QueryFailed(failure) = error else {
22950 panic!("expected typed query failure");
22951 };
22952 assert_eq!(failure.status, 409);
22953 assert_eq!(failure.reason, "query_task_timed_out");
22954
22955 let mut worker = Worker::new(client, "rust-workers")
22956 .worker_id("late-worker")
22957 .poll_timeout(Duration::from_millis(10));
22958 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
22959 worker.register_query(
22960 "counter",
22961 "current",
22962 |_ctx, _args| async move { Ok(json!(8)) },
22963 );
22964
22965 assert_eq!(worker.run_once().await.expect("late task is handled"), 1);
22966 assert_eq!(
22967 worker
22968 .run_once()
22969 .await
22970 .expect("worker continues after late completion"),
22971 0
22972 );
22973 assert_eq!(
22974 server.request_count("/api/worker/query-tasks/query-late/complete"),
22975 2
22976 );
22977 assert_eq!(
22978 server.request_count("/api/worker/query-tasks/query-late/fail"),
22979 0,
22980 "a server completion rejection must not be reported as an encoding failure"
22981 );
22982 }
22983
22984 #[tokio::test]
22985 async fn normal_shutdown_joins_pollers_and_deregisters_once() {
22986 let server = MockWorkerServer::start();
22987 let client = Client::builder(server.base_url())
22988 .timeout(Duration::from_secs(2))
22989 .build()
22990 .expect("client");
22991 let mut worker = Worker::new(client, "rust-workers")
22992 .worker_id("joined-worker")
22993 .poll_timeout(Duration::from_millis(10));
22994 worker.register_workflow(
22995 "joined.workflow",
22996 |_ctx, _input| async move { Ok(Value::Null) },
22997 );
22998 worker.register_activity(
22999 "joined.activity",
23000 |_ctx, _input| async move { Ok(Value::Null) },
23001 );
23002 worker.register_query("joined.workflow", "state", |_ctx, _input| async move {
23003 Ok(Value::Null)
23004 });
23005
23006 worker
23007 .run_until(tokio::time::sleep(Duration::from_millis(20)))
23008 .await
23009 .expect("normal shutdown");
23010
23011 let deregistration_path = "/api/worker/registrations/mock-worker";
23012 assert_eq!(server.request_count(deregistration_path), 1);
23013 for poll_path in [
23014 "/api/worker/workflow-tasks/poll",
23015 "/api/worker/activity-tasks/poll",
23016 "/api/worker/query-tasks/poll",
23017 ] {
23018 assert!(server.request_count(poll_path) > 0, "missing {poll_path}");
23019 }
23020 assert_eq!(
23021 server.captured_paths().last().map(String::as_str),
23022 Some(deregistration_path),
23023 "deregistration must start only after every poller has joined"
23024 );
23025 }
23026
23027 #[tokio::test]
23028 async fn registration_failure_does_not_deregister() {
23029 let server = MockWorkerServer::rejected_registration();
23030 let client = Client::builder(server.base_url())
23031 .timeout(Duration::from_secs(2))
23032 .build()
23033 .expect("client");
23034 let worker = Worker::new(client, "rust-workers").worker_id("never-registered");
23035
23036 let error = worker
23037 .run_until(async {})
23038 .await
23039 .expect_err("registration must fail");
23040 assert!(matches!(
23041 error,
23042 Error::Http {
23043 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23044 ..
23045 }
23046 ));
23047 assert!(server
23048 .captured_paths()
23049 .iter()
23050 .all(|path| !path.starts_with("/api/worker/registrations/")));
23051 }
23052
23053 #[tokio::test]
23054 async fn protocol_116_server_rejects_occurrence_identity_worker_registration() {
23055 let server = MockWorkerServer::rejected_registration_protocol();
23056 let client = Client::builder(server.base_url())
23057 .timeout(Duration::from_secs(2))
23058 .build()
23059 .expect("client");
23060 let worker = Worker::new(client, "rust-workers").worker_id("protocol-117-worker");
23061
23062 let error = worker
23063 .run_until(async {})
23064 .await
23065 .expect_err("a protocol 1.16 server must reject this worker");
23066 let Error::Protocol(failure) = error else {
23067 panic!("expected typed protocol rejection");
23068 };
23069 assert_eq!(failure.reason, "unsupported_protocol_version");
23070 assert_eq!(failure.supported_version.as_deref(), Some("1.16"));
23071 assert_eq!(failure.requested_version.as_deref(), Some("1.17"));
23072 assert_eq!(
23073 server
23074 .worker_protocol_for("/api/worker/register")
23075 .as_deref(),
23076 Some(WORKER_PROTOCOL_VERSION)
23077 );
23078 }
23079
23080 #[tokio::test]
23081 async fn declined_registration_does_not_deregister() {
23082 let server = MockWorkerServer::declined_registration();
23083 let client = Client::builder(server.base_url())
23084 .timeout(Duration::from_secs(2))
23085 .build()
23086 .expect("client");
23087 let worker = Worker::new(client, "rust-workers").worker_id("declined-worker");
23088
23089 let error = worker
23090 .run_until(async {})
23091 .await
23092 .expect_err("declined registration must fail");
23093 assert!(matches!(error, Error::WorkerLoop(_)));
23094 assert!(error.to_string().contains("was not accepted"));
23095 assert!(server
23096 .captured_paths()
23097 .iter()
23098 .all(|path| !path.starts_with("/api/worker/registrations/")));
23099 }
23100
23101 #[tokio::test]
23102 async fn deregistration_http_failure_is_returned_after_normal_shutdown() {
23103 let server = MockWorkerServer::rejected_deregistration();
23104 let client = Client::builder(server.base_url())
23105 .timeout(Duration::from_secs(2))
23106 .build()
23107 .expect("client");
23108 let worker = Worker::new(client, "rust-workers").worker_id("forbidden-cleanup");
23109
23110 let error = worker
23111 .run_until(async {})
23112 .await
23113 .expect_err("deregistration must fail");
23114 assert!(matches!(
23115 error,
23116 Error::Http {
23117 status: reqwest::StatusCode::FORBIDDEN,
23118 ..
23119 }
23120 ));
23121 assert_eq!(
23122 server.request_count("/api/worker/registrations/mock-worker"),
23123 1
23124 );
23125 }
23126
23127 #[tokio::test]
23128 async fn deregistration_protocol_failure_is_returned_after_normal_shutdown() {
23129 let server = MockWorkerServer::rejected_deregistration_protocol();
23130 let client = Client::builder(server.base_url())
23131 .timeout(Duration::from_secs(2))
23132 .build()
23133 .expect("client");
23134 let worker = Worker::new(client, "rust-workers").worker_id("protocol-cleanup");
23135
23136 let error = worker
23137 .run_until(async {})
23138 .await
23139 .expect_err("protocol rejection must fail shutdown");
23140 let Error::Protocol(failure) = error else {
23141 panic!("expected typed protocol failure");
23142 };
23143 assert_eq!(failure.reason, "unsupported_protocol_version");
23144 assert_eq!(
23145 failure.requested_version.as_deref(),
23146 Some(WORKER_PROTOCOL_VERSION)
23147 );
23148 assert_eq!(
23149 server.request_count("/api/worker/registrations/mock-worker"),
23150 1
23151 );
23152 }
23153
23154 #[tokio::test]
23155 async fn primary_poller_error_retains_deregistration_failure_context() {
23156 let server = MockWorkerServer::unauthorized_polls_and_rejected_deregistration();
23157 let client = Client::builder(server.base_url())
23158 .timeout(Duration::from_secs(2))
23159 .build()
23160 .expect("client");
23161 let mut worker = Worker::new(client, "rust-workers")
23162 .worker_id("combined-failure")
23163 .poll_timeout(Duration::from_millis(10));
23164 worker.register_workflow("combined.workflow", |_ctx, _input| async move {
23165 Ok(Value::Null)
23166 });
23167
23168 let error = worker
23169 .run()
23170 .await
23171 .expect_err("worker and cleanup must fail");
23172 let summary = error.to_string();
23173 assert!(summary.contains("authentication_failed"));
23174 assert!(summary.contains("worker cannot deregister"));
23175 let Error::WorkerShutdown {
23176 primary,
23177 deregistration,
23178 } = error
23179 else {
23180 panic!("expected combined worker shutdown error");
23181 };
23182 assert!(matches!(
23183 *primary,
23184 Error::Http {
23185 status: reqwest::StatusCode::UNAUTHORIZED,
23186 ..
23187 }
23188 ));
23189 assert!(matches!(
23190 *deregistration,
23191 Error::Http {
23192 status: reqwest::StatusCode::FORBIDDEN,
23193 ..
23194 }
23195 ));
23196 assert_eq!(
23197 server.request_count("/api/worker/registrations/mock-worker"),
23198 1
23199 );
23200 }
23201
23202 #[tokio::test]
23203 async fn activity_only_worker_can_shutdown_without_workflow_poller() {
23204 let server = MockWorkerServer::start();
23205 let client = Client::builder(server.base_url())
23206 .timeout(Duration::from_secs(2))
23207 .build()
23208 .expect("client");
23209 let mut worker = Worker::new(client, "rust-workers")
23210 .worker_id("activity-only-worker")
23211 .poll_timeout(Duration::from_millis(10));
23212
23213 worker.register_activity(
23214 "activity.only",
23215 |_ctx, _args| async move { Ok(Value::Null) },
23216 );
23217
23218 worker.run_until(async {}).await.expect("run worker");
23219 }
23220
23221 #[tokio::test]
23222 async fn workflow_only_worker_can_shutdown_without_activity_poller() {
23223 let server = MockWorkerServer::start();
23224 let client = Client::builder(server.base_url())
23225 .timeout(Duration::from_secs(2))
23226 .build()
23227 .expect("client");
23228 let mut worker = Worker::new(client, "rust-workers")
23229 .worker_id("workflow-only-worker")
23230 .poll_timeout(Duration::from_millis(10));
23231
23232 worker.register_workflow(
23233 "workflow.only",
23234 |_ctx, _input| async move { Ok(Value::Null) },
23235 );
23236
23237 worker.run_until(async {}).await.expect("run worker");
23238 }
23239
23240 #[tokio::test]
23241 async fn worker_heartbeat_observer_receives_server_acknowledgements() {
23242 let server = MockWorkerServer::start();
23243 let client = Client::builder(server.base_url())
23244 .timeout(Duration::from_secs(2))
23245 .build()
23246 .expect("client");
23247 let observations = Arc::new(Mutex::new(Vec::new()));
23248 let observed = Arc::clone(&observations);
23249 let mut worker = Worker::new(client, "rust-workers")
23250 .worker_id("observed-heartbeat-worker")
23251 .poll_timeout(Duration::from_millis(10))
23252 .on_worker_heartbeat(move |observation| {
23253 observed
23254 .lock()
23255 .expect("heartbeat observations")
23256 .push(observation.clone());
23257 });
23258
23259 worker.register_workflow("workflow.observed", |_ctx, _input| async move {
23260 Ok(Value::Null)
23261 });
23262 let acknowledged = Arc::clone(&observations);
23263 worker
23264 .run_until(async move {
23265 tokio::time::timeout(Duration::from_secs(2), async move {
23266 loop {
23267 if !acknowledged
23268 .lock()
23269 .expect("heartbeat observations")
23270 .is_empty()
23271 {
23272 break;
23273 }
23274 tokio::time::sleep(Duration::from_millis(1)).await;
23275 }
23276 })
23277 .await
23278 .expect("heartbeat acknowledgement within timeout");
23279 })
23280 .await
23281 .expect("run worker");
23282
23283 let observations = observations.lock().expect("heartbeat observations");
23284 let first = observations.first().expect("heartbeat acknowledgement");
23285 assert_eq!(first.worker_id, "observed-heartbeat-worker");
23286 assert_eq!(first.task_queue, "rust-workers");
23287 assert!(first.acknowledged_at_unix_millis > 0);
23288 assert_eq!(first.acknowledgement, json!({}));
23289 }
23290
23291 #[tokio::test]
23292 async fn delayed_worker_heartbeat_keeps_cadence_and_pollers_live() {
23293 let server = MockWorkerServer::delayed_heartbeat_worker();
23294 let client = Client::builder(server.base_url())
23295 .timeout(Duration::from_secs(3))
23296 .build()
23297 .expect("client");
23298 let observations = Arc::new(Mutex::new(Vec::new()));
23299 let observed = Arc::clone(&observations);
23300 let mut worker = Worker::new(client, "rust-snapshot-workers")
23301 .worker_id("rust-snapshot-worker")
23302 .poll_timeout(Duration::from_millis(10))
23303 .on_worker_heartbeat(move |observation| {
23304 observed
23305 .lock()
23306 .expect("heartbeat observations")
23307 .push(observation.clone());
23308 });
23309
23310 worker.register_workflow("snapshot", |ctx, _input| async move {
23311 ctx.wait_signal("finish").await?;
23312 Ok(json!({"status": "finished"}))
23313 });
23314 worker.register_query("snapshot", "current", |ctx, _args| async move {
23315 Ok(json!(ctx
23316 .signals("increment")
23317 .iter()
23318 .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
23319 .sum::<i64>()))
23320 });
23321 worker.register_activity("cancel-aware", |_ctx, _args| async move {
23322 Ok(json!({"late": "completion"}))
23323 });
23324
23325 worker
23326 .run_until(tokio::time::sleep(Duration::from_millis(3_800)))
23327 .await
23328 .expect("delayed heartbeat must allow a clean worker shutdown");
23329
23330 let observations = observations.lock().expect("heartbeat observations");
23331 assert!(
23332 observations.len() >= 3,
23333 "the immediate heartbeat, delayed acknowledgement, and next cadence heartbeat must complete"
23334 );
23335 assert!(
23336 observations.windows(2).all(|pair| {
23337 pair[1].acknowledged_at_unix_millis
23338 .saturating_sub(pair[0].acknowledged_at_unix_millis)
23339 >= 850
23340 }),
23341 "successful acknowledgements must not catch up faster than the advertised one-second cadence: {observations:?}"
23342 );
23343 drop(observations);
23344
23345 let heartbeat_times = server.request_times("/api/worker/heartbeat");
23346 let delayed_request_at = *heartbeat_times
23347 .get(1)
23348 .expect("intentionally delayed heartbeat request");
23349 let delay_window_start = delayed_request_at + Duration::from_millis(100);
23350 let delay_window_end = delayed_request_at + Duration::from_millis(1_400);
23351 for path in [
23352 "/api/worker/workflow-tasks/poll",
23353 "/api/worker/activity-tasks/poll",
23354 "/api/worker/query-tasks/poll",
23355 ] {
23356 assert!(
23357 server
23358 .request_times(path)
23359 .iter()
23360 .any(|received_at| *received_at >= delay_window_start
23361 && *received_at <= delay_window_end),
23362 "{path} must keep polling while a heartbeat acknowledgement is delayed"
23363 );
23364 }
23365 assert!(
23366 server.request_count("/api/worker/workflow-tasks/snapshot-wait-3/fail") >= 1,
23367 "workflow work must be settled"
23368 );
23369 assert!(
23370 server.request_count("/api/worker/activity-tasks/activity-cancel/complete") >= 1,
23371 "activity work must be settled"
23372 );
23373 assert!(
23374 server.request_count("/api/worker/query-tasks/snapshot-current/complete") >= 1,
23375 "query work must be settled"
23376 );
23377 }
23378
23379 #[tokio::test]
23380 async fn retried_worker_heartbeat_restarts_the_advertised_cadence() {
23381 let server = MockWorkerServer::heartbeat_retry_worker();
23382 let client = Client::builder(server.base_url())
23383 .timeout(Duration::from_secs(2))
23384 .build()
23385 .expect("client");
23386 let observations = Arc::new(Mutex::new(Vec::new()));
23387 let observed = Arc::clone(&observations);
23388 let worker = Worker::new(client, "rust-workers")
23389 .worker_id("heartbeat-retry-worker")
23390 .retry_policy(WorkerRetryPolicy {
23391 max_retries: 1,
23392 initial_backoff: Duration::from_millis(300),
23393 max_backoff: Duration::from_millis(300),
23394 })
23395 .on_worker_heartbeat(move |observation| {
23396 observed
23397 .lock()
23398 .expect("heartbeat observations")
23399 .push(observation.clone());
23400 });
23401
23402 worker
23403 .run_until(tokio::time::sleep(Duration::from_millis(2_700)))
23404 .await
23405 .expect("retryable heartbeat failure must remain bounded and recover");
23406
23407 let observations = observations.lock().expect("heartbeat observations");
23408 assert!(observations.len() >= 3, "heartbeat retry must recover");
23409 assert!(
23410 observations.windows(2).all(|pair| {
23411 pair[1]
23412 .acknowledged_at_unix_millis
23413 .saturating_sub(pair[0].acknowledged_at_unix_millis)
23414 >= 850
23415 }),
23416 "a successful retry must start a fresh advertised cadence: {observations:?}"
23417 );
23418 assert_eq!(
23419 server.request_count("/api/worker/heartbeat"),
23420 observations.len() + 1,
23421 "one retryable failure must add exactly one bounded request"
23422 );
23423 }
23424
23425 #[tokio::test]
23426 async fn query_enabled_worker_ignores_unmatched_signals_then_completes_once() {
23427 let server = MockWorkerServer::waiting_query_worker();
23428 let client = Client::builder(server.base_url())
23429 .timeout(Duration::from_secs(2))
23430 .build()
23431 .expect("client");
23432 let observations = Arc::new(Mutex::new(Vec::new()));
23433 let observed = Arc::clone(&observations);
23434 let mut worker = Worker::new(client, "rust-snapshot-workers")
23435 .worker_id("rust-snapshot-worker")
23436 .poll_timeout(Duration::from_millis(10))
23437 .on_worker_heartbeat(move |observation| {
23438 observed
23439 .lock()
23440 .expect("heartbeat observations")
23441 .push(observation.clone());
23442 });
23443
23444 worker.register_workflow("snapshot", |ctx, _input| async move {
23445 ctx.wait_signal("finish").await?;
23446 Ok(json!({"status": "finished"}))
23447 });
23448 worker.register_query("snapshot", "current", |ctx, _args| async move {
23449 let current = ctx
23450 .signals("increment")
23451 .iter()
23452 .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
23453 .sum::<i64>();
23454 Ok(json!(current))
23455 });
23456 worker.register_update("snapshot", "replace", |_ctx, args| async move { Ok(args) });
23457
23458 worker
23459 .run_until(tokio::time::sleep(Duration::from_millis(3_200)))
23460 .await
23461 .expect("pending workflow and query poller must remain live until shutdown");
23462
23463 assert!(
23464 observations.lock().expect("heartbeat observations").len() >= 4,
23465 "the immediate heartbeat and at least three advertised one-second intervals must be acknowledged"
23466 );
23467 assert!(
23468 server.request_count("/api/worker/workflow-tasks/poll") >= 3,
23469 "workflow polling must continue after empty replay acknowledgements"
23470 );
23471 assert!(
23472 server.request_count("/api/worker/query-tasks/poll") >= 2,
23473 "query polling must continue after serving the current query"
23474 );
23475 assert_eq!(
23476 server.request_body("/api/worker/register")["capabilities"],
23477 json!([
23478 CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY,
23479 DURABLE_SELECTION_CAPABILITY,
23480 MEMO_UPSERTS_CAPABILITY,
23481 TYPED_SEARCH_ATTRIBUTES_CAPABILITY,
23482 QUERY_TASKS_CAPABILITY,
23483 WORKFLOW_UPDATES_CAPABILITY,
23484 MESSAGE_STREAMS_CAPABILITY
23485 ])
23486 );
23487 assert_eq!(
23488 server.request_body("/api/worker/register")["workflow_command_contracts"]["snapshot"],
23489 json!({
23490 "queries": ["current"],
23491 "query_contracts": [],
23492 "signals": [],
23493 "signal_contracts": [],
23494 "updates": ["replace"],
23495 "update_contracts": [],
23496 "update_validators": [],
23497 })
23498 );
23499
23500 let opened = server.request_body("/api/worker/workflow-tasks/snapshot-open/complete");
23501 assert_eq!(
23502 opened["commands"],
23503 json!([{
23504 "type": "open_signal_wait",
23505 "signal_name": "finish",
23506 }])
23507 );
23508
23509 for task_id in ["snapshot-wait-3", "snapshot-wait-5"] {
23510 let fail_path = format!("/api/worker/workflow-tasks/{task_id}/fail");
23511 let completion_path = format!("/api/worker/workflow-tasks/{task_id}/complete");
23512 let failure = server.request_body(&fail_path);
23513 assert_eq!(
23514 failure["failure"]["type"],
23515 WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE
23516 );
23517 assert_eq!(server.request_count(&completion_path), 0);
23518 }
23519
23520 let query_completion =
23521 server.request_body("/api/worker/query-tasks/snapshot-current/complete");
23522 assert_eq!(query_completion["result"], json!(8));
23523
23524 let terminal_path = "/api/worker/workflow-tasks/snapshot-finish/complete";
23525 assert_eq!(
23526 server.request_count(terminal_path),
23527 1,
23528 "the matching signal must settle the workflow exactly once"
23529 );
23530 let terminal = server.request_body(terminal_path);
23531 assert_eq!(terminal["commands"].as_array().map(Vec::len), Some(1));
23532 assert_eq!(terminal["commands"][0]["type"], "complete_workflow");
23533 assert_eq!(
23534 decode_wire_value(&terminal["commands"][0]["result"], DEFAULT_CODEC)
23535 .expect("terminal workflow result"),
23536 json!({"status": "finished"})
23537 );
23538 }
23539
23540 #[tokio::test]
23541 async fn worker_retries_poll_and_heartbeat_transport_failures_independently() {
23542 let server = MockWorkerServer::transient_worker_failures();
23543 let client = Client::builder(server.base_url())
23544 .timeout(Duration::from_secs(2))
23545 .build()
23546 .expect("client");
23547 let mut worker = Worker::new(client, "rust-workers")
23548 .worker_id("retry-worker")
23549 .poll_timeout(Duration::from_millis(10))
23550 .retry_policy(WorkerRetryPolicy {
23551 max_retries: 2,
23552 initial_backoff: Duration::from_millis(1),
23553 max_backoff: Duration::from_millis(1),
23554 });
23555 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
23556 worker.register_activity(
23557 "counter.activity",
23558 |_ctx, _input| async move { Ok(Value::Null) },
23559 );
23560 worker.register_query(
23561 "counter",
23562 "current",
23563 |_ctx, _args| async move { Ok(json!(8)) },
23564 );
23565
23566 worker
23567 .run_until(tokio::time::sleep(Duration::from_millis(75)))
23568 .await
23569 .expect("transient failures must not stop the worker");
23570
23571 for path in [
23572 "/api/worker/heartbeat",
23573 "/api/worker/workflow-tasks/poll",
23574 "/api/worker/activity-tasks/poll",
23575 "/api/worker/query-tasks/poll",
23576 ] {
23577 assert!(
23578 server.request_count(path) >= 2,
23579 "{path} must continue after its transient failure"
23580 );
23581 }
23582 }
23583
23584 #[tokio::test]
23585 async fn worker_continues_after_long_poll_capacity_backpressure() {
23586 let server = MockWorkerServer::capacity_limited_activity_poll();
23587 let client = Client::builder(server.base_url())
23588 .timeout(Duration::from_secs(2))
23589 .build()
23590 .expect("client");
23591 let mut worker = Worker::new(client, "rust-workers")
23592 .worker_id("capacity-worker")
23593 .poll_timeout(Duration::from_millis(10))
23594 .retry_policy(WorkerRetryPolicy {
23595 max_retries: 0,
23596 initial_backoff: Duration::from_millis(1),
23597 max_backoff: Duration::from_millis(1),
23598 });
23599 worker.register_activity("capacity.activity", |_ctx, _input| async move {
23600 Ok(json!({"handled": true}))
23601 });
23602
23603 worker
23604 .run_until(tokio::time::sleep(Duration::from_millis(50)))
23605 .await
23606 .expect("capacity backpressure must not stop the worker");
23607
23608 assert!(
23609 server.request_count("/api/worker/activity-tasks/poll") >= 2,
23610 "the activity poller must continue after capacity backpressure"
23611 );
23612 assert_eq!(
23613 server.request_count("/api/worker/activity-tasks/capacity-activity/complete"),
23614 1,
23615 "the worker must complete work returned after capacity recovers"
23616 );
23617 }
23618
23619 #[test]
23620 fn worker_poll_capacity_backpressure_requires_the_typed_retryable_contract() {
23621 let capacity = Error::Http {
23622 status: reqwest::StatusCode::TOO_MANY_REQUESTS,
23623 body: r#"{"poll_status":"long_poll_capacity_exhausted","retryable":true,"retry_after_seconds":3}"#.to_string(),
23624 };
23625 assert_eq!(
23626 worker_poll_capacity_retry_after(&capacity),
23627 Some(Duration::from_secs(3))
23628 );
23629
23630 let rejected_capacity = Error::Http {
23631 status: reqwest::StatusCode::TOO_MANY_REQUESTS,
23632 body: r#"{"reason":"long_poll_capacity_exhausted","retryable":false,"retry_after_seconds":3}"#.to_string(),
23633 };
23634 assert_eq!(worker_poll_capacity_retry_after(&rejected_capacity), None);
23635 assert!(!worker_operation_is_retryable(&rejected_capacity));
23636
23637 let ordinary_rate_limit = Error::Http {
23638 status: reqwest::StatusCode::TOO_MANY_REQUESTS,
23639 body: r#"{"reason":"rate_limited","retryable":true,"retry_after_seconds":3}"#
23640 .to_string(),
23641 };
23642 assert_eq!(worker_poll_capacity_retry_after(&ordinary_rate_limit), None);
23643 assert!(worker_operation_is_retryable(&ordinary_rate_limit));
23644 }
23645
23646 #[tokio::test]
23647 async fn worker_bounds_transport_retries() {
23648 let server = MockWorkerServer::unavailable_polls();
23649 let client = Client::builder(server.base_url())
23650 .timeout(Duration::from_secs(2))
23651 .build()
23652 .expect("client");
23653 let mut worker = Worker::new(client, "rust-workers")
23654 .worker_id("bounded-retry-worker")
23655 .poll_timeout(Duration::from_millis(10))
23656 .retry_policy(WorkerRetryPolicy {
23657 max_retries: 2,
23658 initial_backoff: Duration::from_millis(1),
23659 max_backoff: Duration::from_millis(1),
23660 });
23661 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
23662
23663 let error = worker.run().await.expect_err("retry bound must terminate");
23664 assert!(matches!(error, Error::Transport(_)));
23665 assert_eq!(
23666 server.request_count("/api/worker/workflow-tasks/poll"),
23667 3,
23668 "one initial request plus exactly two retries"
23669 );
23670 }
23671
23672 #[tokio::test]
23673 async fn worker_retry_policy_can_disable_poll_retries() {
23674 let server = MockWorkerServer::unavailable_polls();
23675 let client = Client::builder(server.base_url())
23676 .timeout(Duration::from_secs(2))
23677 .build()
23678 .expect("client");
23679 let mut worker = Worker::new(client, "rust-workers")
23680 .worker_id("no-retry-worker")
23681 .poll_timeout(Duration::from_millis(10))
23682 .retry_policy(WorkerRetryPolicy {
23683 max_retries: 0,
23684 initial_backoff: Duration::from_millis(1),
23685 max_backoff: Duration::from_millis(1),
23686 });
23687 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
23688
23689 let error = worker
23690 .run_once()
23691 .await
23692 .expect_err("disabled retries must return the first transport failure");
23693 assert!(matches!(error, Error::Transport(_)));
23694 assert_eq!(
23695 server.request_count("/api/worker/workflow-tasks/poll"),
23696 1,
23697 "max_retries=0 must send only the initial request"
23698 );
23699 }
23700
23701 #[tokio::test]
23702 async fn worker_does_not_retry_authentication_failures() {
23703 let server = MockWorkerServer::unauthorized_polls();
23704 let client = Client::builder(server.base_url())
23705 .timeout(Duration::from_secs(2))
23706 .build()
23707 .expect("client");
23708 let mut worker = Worker::new(client, "rust-workers")
23709 .worker_id("unauthorized-worker")
23710 .poll_timeout(Duration::from_millis(10));
23711 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
23712
23713 let error = worker
23714 .run()
23715 .await
23716 .expect_err("authentication must terminate");
23717 let Error::Http { status, body } = error else {
23718 panic!("expected stable HTTP authentication error");
23719 };
23720 assert_eq!(status, reqwest::StatusCode::UNAUTHORIZED);
23721 assert!(body.contains("authentication_failed"));
23722 assert_eq!(
23723 server.request_count("/api/worker/workflow-tasks/poll"),
23724 1,
23725 "authentication failures must not be retried"
23726 );
23727 }
23728
23729 #[derive(Clone, Debug)]
23730 struct CapturedRequest {
23731 method: String,
23732 path: String,
23733 authorization: Option<String>,
23734 namespace: Option<String>,
23735 worker_protocol: Option<String>,
23736 control_protocol: Option<String>,
23737 body: String,
23738 received_at: Instant,
23739 }
23740
23741 struct MockWorkerServer {
23742 addr: SocketAddr,
23743 stop: Arc<AtomicBool>,
23744 requests: Arc<Mutex<Vec<CapturedRequest>>>,
23745 thread: Option<thread::JoinHandle<()>>,
23746 }
23747
23748 #[derive(Clone, Copy, Default)]
23749 struct MockWorkerBehavior {
23750 reject_query_protocol: bool,
23751 reject_query_completion: bool,
23752 waiting_query_worker: bool,
23753 decline_registration: bool,
23754 complete_named_signal: bool,
23755 poll_failures_per_path: usize,
23756 long_poll_capacity_responses_per_path: usize,
23757 heartbeat_failures: usize,
23758 heartbeat_failure_request: Option<usize>,
23759 delayed_heartbeat_request: Option<usize>,
23760 heartbeat_response_delay: Duration,
23761 concurrent_requests: bool,
23762 unauthorized_polls: bool,
23763 reject_registration: bool,
23764 reject_registration_protocol: bool,
23765 reject_deregistration: bool,
23766 reject_deregistration_protocol: bool,
23767 cancelled_activity: bool,
23768 draining_polls: bool,
23769 invalid_task_payload_codec: Option<InvalidTaskPayloadCodec>,
23770 workflow_completion_status: Option<&'static str>,
23771 workflow_completion_body: Option<&'static str>,
23772 }
23773
23774 impl MockWorkerServer {
23775 fn start() -> Self {
23776 Self::start_with_behavior(MockWorkerBehavior::default())
23777 }
23778
23779 fn reject_query_protocol() -> Self {
23780 Self::start_with_behavior(MockWorkerBehavior {
23781 reject_query_protocol: true,
23782 ..MockWorkerBehavior::default()
23783 })
23784 }
23785
23786 fn reject_query_completion() -> Self {
23787 Self::start_with_behavior(MockWorkerBehavior {
23788 reject_query_completion: true,
23789 ..MockWorkerBehavior::default()
23790 })
23791 }
23792
23793 fn waiting_query_worker() -> Self {
23794 Self::start_with_behavior(MockWorkerBehavior {
23795 waiting_query_worker: true,
23796 complete_named_signal: true,
23797 ..MockWorkerBehavior::default()
23798 })
23799 }
23800
23801 fn transient_worker_failures() -> Self {
23802 Self::start_with_behavior(MockWorkerBehavior {
23803 poll_failures_per_path: 1,
23804 heartbeat_failures: 1,
23805 ..MockWorkerBehavior::default()
23806 })
23807 }
23808
23809 fn consecutive_poll_failures(count: usize) -> Self {
23810 Self::start_with_behavior(MockWorkerBehavior {
23811 poll_failures_per_path: count,
23812 ..MockWorkerBehavior::default()
23813 })
23814 }
23815
23816 fn capacity_limited_activity_poll() -> Self {
23817 Self::start_with_behavior(MockWorkerBehavior {
23818 long_poll_capacity_responses_per_path: 1,
23819 ..MockWorkerBehavior::default()
23820 })
23821 }
23822
23823 fn delayed_heartbeat_worker() -> Self {
23824 Self::start_with_behavior(MockWorkerBehavior {
23825 waiting_query_worker: true,
23826 delayed_heartbeat_request: Some(2),
23827 heartbeat_response_delay: Duration::from_millis(1_500),
23828 concurrent_requests: true,
23829 cancelled_activity: true,
23830 ..MockWorkerBehavior::default()
23831 })
23832 }
23833
23834 fn heartbeat_retry_worker() -> Self {
23835 Self::start_with_behavior(MockWorkerBehavior {
23836 waiting_query_worker: true,
23837 heartbeat_failure_request: Some(2),
23838 concurrent_requests: true,
23839 ..MockWorkerBehavior::default()
23840 })
23841 }
23842
23843 fn unavailable_polls() -> Self {
23844 Self::start_with_behavior(MockWorkerBehavior {
23845 poll_failures_per_path: usize::MAX,
23846 ..MockWorkerBehavior::default()
23847 })
23848 }
23849
23850 fn unauthorized_polls() -> Self {
23851 Self::start_with_behavior(MockWorkerBehavior {
23852 unauthorized_polls: true,
23853 ..MockWorkerBehavior::default()
23854 })
23855 }
23856
23857 fn rejected_registration() -> Self {
23858 Self::start_with_behavior(MockWorkerBehavior {
23859 reject_registration: true,
23860 ..MockWorkerBehavior::default()
23861 })
23862 }
23863
23864 fn rejected_registration_protocol() -> Self {
23865 Self::start_with_behavior(MockWorkerBehavior {
23866 reject_registration_protocol: true,
23867 ..MockWorkerBehavior::default()
23868 })
23869 }
23870
23871 fn declined_registration() -> Self {
23872 Self::start_with_behavior(MockWorkerBehavior {
23873 decline_registration: true,
23874 ..MockWorkerBehavior::default()
23875 })
23876 }
23877
23878 fn rejected_deregistration() -> Self {
23879 Self::start_with_behavior(MockWorkerBehavior {
23880 reject_deregistration: true,
23881 ..MockWorkerBehavior::default()
23882 })
23883 }
23884
23885 fn rejected_deregistration_protocol() -> Self {
23886 Self::start_with_behavior(MockWorkerBehavior {
23887 reject_deregistration_protocol: true,
23888 ..MockWorkerBehavior::default()
23889 })
23890 }
23891
23892 fn unauthorized_polls_and_rejected_deregistration() -> Self {
23893 Self::start_with_behavior(MockWorkerBehavior {
23894 unauthorized_polls: true,
23895 reject_deregistration: true,
23896 ..MockWorkerBehavior::default()
23897 })
23898 }
23899
23900 fn cancelled_activity() -> Self {
23901 Self::start_with_behavior(MockWorkerBehavior {
23902 cancelled_activity: true,
23903 ..MockWorkerBehavior::default()
23904 })
23905 }
23906
23907 fn draining_polls() -> Self {
23908 Self::start_with_behavior(MockWorkerBehavior {
23909 draining_polls: true,
23910 ..MockWorkerBehavior::default()
23911 })
23912 }
23913
23914 fn invalid_task_payload_codec(codec: InvalidTaskPayloadCodec) -> Self {
23915 Self::start_with_behavior(MockWorkerBehavior {
23916 invalid_task_payload_codec: Some(codec),
23917 ..MockWorkerBehavior::default()
23918 })
23919 }
23920
23921 fn workflow_completion(status: &'static str, body: &'static str) -> Self {
23922 Self::start_with_behavior(MockWorkerBehavior {
23923 workflow_completion_status: Some(status),
23924 workflow_completion_body: Some(body),
23925 ..MockWorkerBehavior::default()
23926 })
23927 }
23928
23929 fn start_with_behavior(behavior: MockWorkerBehavior) -> Self {
23930 let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock server");
23931 listener
23932 .set_nonblocking(true)
23933 .expect("configure mock listener");
23934 let addr = listener.local_addr().expect("mock server address");
23935 let stop = Arc::new(AtomicBool::new(false));
23936 let server_stop = Arc::clone(&stop);
23937 let requests = Arc::new(Mutex::new(Vec::new()));
23938 let server_requests = Arc::clone(&requests);
23939 let thread = thread::spawn(move || {
23940 let mut request_threads = Vec::new();
23941 while !server_stop.load(Ordering::SeqCst) {
23942 match listener.accept() {
23943 Ok((mut stream, _)) => {
23944 if behavior.concurrent_requests {
23945 let requests = Arc::clone(&server_requests);
23946 request_threads.push(thread::spawn(move || {
23947 handle_mock_worker_request(&mut stream, &requests, behavior)
23948 }));
23949 } else {
23950 handle_mock_worker_request(&mut stream, &server_requests, behavior);
23951 }
23952 }
23953 Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
23954 let mut index = 0;
23955 while index < request_threads.len() {
23956 if request_threads[index].is_finished() {
23957 request_threads
23958 .swap_remove(index)
23959 .join()
23960 .expect("join mock request");
23961 } else {
23962 index += 1;
23963 }
23964 }
23965 thread::sleep(Duration::from_millis(5));
23966 }
23967 Err(_) => break,
23968 }
23969 }
23970 for request_thread in request_threads {
23971 request_thread.join().expect("join mock request");
23972 }
23973 });
23974
23975 Self {
23976 addr,
23977 stop,
23978 requests,
23979 thread: Some(thread),
23980 }
23981 }
23982
23983 fn base_url(&self) -> String {
23984 format!("http://{}", self.addr)
23985 }
23986
23987 fn worker_protocol_for(&self, path: &str) -> Option<String> {
23988 self.requests
23989 .lock()
23990 .expect("captured requests")
23991 .iter()
23992 .find(|request| request.path == path)
23993 .and_then(|request| request.worker_protocol.clone())
23994 }
23995
23996 fn control_protocol_for(&self, path: &str) -> Option<String> {
23997 self.requests
23998 .lock()
23999 .expect("captured requests")
24000 .iter()
24001 .find(|request| request.path == path)
24002 .and_then(|request| request.control_protocol.clone())
24003 }
24004
24005 fn method_for(&self, path: &str) -> Option<String> {
24006 self.requests
24007 .lock()
24008 .expect("captured requests")
24009 .iter()
24010 .find(|request| request.path == path)
24011 .map(|request| request.method.clone())
24012 }
24013
24014 fn authorization_for(&self, path: &str) -> Option<String> {
24015 self.requests
24016 .lock()
24017 .expect("captured requests")
24018 .iter()
24019 .find(|request| request.path == path)
24020 .and_then(|request| request.authorization.clone())
24021 }
24022
24023 fn namespace_for(&self, path: &str) -> Option<String> {
24024 self.requests
24025 .lock()
24026 .expect("captured requests")
24027 .iter()
24028 .find(|request| request.path == path)
24029 .and_then(|request| request.namespace.clone())
24030 }
24031
24032 fn request_count(&self, path: &str) -> usize {
24033 self.requests
24034 .lock()
24035 .expect("captured requests")
24036 .iter()
24037 .filter(|request| request.path == path)
24038 .count()
24039 }
24040
24041 fn captured_paths(&self) -> Vec<String> {
24042 self.requests
24043 .lock()
24044 .expect("captured requests")
24045 .iter()
24046 .map(|request| request.path.clone())
24047 .collect()
24048 }
24049
24050 fn request_times(&self, path: &str) -> Vec<Instant> {
24051 self.requests
24052 .lock()
24053 .expect("captured requests")
24054 .iter()
24055 .filter(|request| request.path == path)
24056 .map(|request| request.received_at)
24057 .collect()
24058 }
24059
24060 fn request_body(&self, path: &str) -> Value {
24061 let requests = self.requests.lock().expect("captured requests");
24062 let body = &requests
24063 .iter()
24064 .find(|request| request.path == path)
24065 .unwrap_or_else(|| panic!("missing request for {path}"))
24066 .body;
24067 serde_json::from_str(body).unwrap_or_else(|error| {
24068 panic!("invalid JSON request body for {path}: {error}: {body:?}")
24069 })
24070 }
24071
24072 fn request_bodies(&self, path: &str) -> Vec<Value> {
24073 self.requests
24074 .lock()
24075 .expect("captured requests")
24076 .iter()
24077 .filter(|request| request.path == path)
24078 .map(|request| {
24079 serde_json::from_str(&request.body).unwrap_or_else(|error| {
24080 panic!(
24081 "invalid JSON request body for {path}: {error}: {:?}",
24082 request.body
24083 )
24084 })
24085 })
24086 .collect()
24087 }
24088 }
24089
24090 impl Drop for MockWorkerServer {
24091 fn drop(&mut self) {
24092 self.stop.store(true, Ordering::SeqCst);
24093 let _ = TcpStream::connect(self.addr);
24094
24095 if let Some(thread) = self.thread.take() {
24096 thread.join().expect("join mock server");
24097 }
24098 }
24099 }
24100
24101 fn handle_mock_worker_request(
24102 stream: &mut TcpStream,
24103 requests: &Arc<Mutex<Vec<CapturedRequest>>>,
24104 behavior: MockWorkerBehavior,
24105 ) {
24106 let _ = stream.set_read_timeout(Some(Duration::from_millis(200)));
24107 let mut buffer = [0_u8; 8192];
24108 let mut request = Vec::new();
24109
24110 loop {
24111 match stream.read(&mut buffer) {
24112 Ok(0) => break,
24113 Ok(read) => {
24114 request.extend_from_slice(&buffer[..read]);
24115 if mock_request_is_complete(&request) {
24116 break;
24117 }
24118 }
24119 Err(error)
24120 if matches!(
24121 error.kind(),
24122 std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
24123 ) =>
24124 {
24125 break;
24126 }
24127 Err(_) => return,
24128 }
24129 }
24130
24131 let request = String::from_utf8_lossy(&request);
24132 let body = request
24133 .split_once("\r\n\r\n")
24134 .map(|(_, body)| body)
24135 .unwrap_or_default();
24136 let path = request
24137 .lines()
24138 .next()
24139 .and_then(|line| line.split_whitespace().nth(1))
24140 .unwrap_or_default();
24141 let method = request
24142 .lines()
24143 .next()
24144 .and_then(|line| line.split_whitespace().next())
24145 .unwrap_or_default();
24146 let authorization = request.lines().find_map(|line| {
24147 let (name, value) = line.split_once(':')?;
24148 name.eq_ignore_ascii_case("Authorization")
24149 .then(|| value.trim().to_string())
24150 });
24151 let namespace = request.lines().find_map(|line| {
24152 let (name, value) = line.split_once(':')?;
24153 name.eq_ignore_ascii_case("X-Namespace")
24154 .then(|| value.trim().to_string())
24155 });
24156 let worker_protocol = request.lines().find_map(|line| {
24157 let (name, value) = line.split_once(':')?;
24158 name.eq_ignore_ascii_case("X-Durable-Workflow-Protocol-Version")
24159 .then(|| value.trim().to_string())
24160 });
24161 let control_protocol = request.lines().find_map(|line| {
24162 let (name, value) = line.split_once(':')?;
24163 name.eq_ignore_ascii_case("X-Durable-Workflow-Control-Plane-Version")
24164 .then(|| value.trim().to_string())
24165 });
24166 let request_number = {
24167 let mut requests = requests.lock().expect("captured requests");
24168 requests.push(CapturedRequest {
24169 method: method.to_string(),
24170 path: path.to_string(),
24171 authorization,
24172 namespace,
24173 worker_protocol: worker_protocol.clone(),
24174 control_protocol,
24175 body: body.to_string(),
24176 received_at: Instant::now(),
24177 });
24178 requests
24179 .iter()
24180 .filter(|request| request.path == path)
24181 .count()
24182 };
24183
24184 if path == "/api/worker/register" {
24185 if behavior.reject_registration_protocol {
24186 write_mock_response(
24187 stream,
24188 "400 Bad Request",
24189 r#"{"reason":"unsupported_protocol_version","message":"condition-wait occurrence identity requires worker protocol 1.17","supported_version":"1.16","requested_version":"1.17"}"#,
24190 );
24191 return;
24192 }
24193 if behavior.reject_registration {
24194 write_mock_response(
24195 stream,
24196 "503 Service Unavailable",
24197 r#"{"reason":"registration_unavailable","message":"registration failed"}"#,
24198 );
24199 return;
24200 }
24201 }
24202
24203 if path.starts_with("/api/worker/registrations/") {
24204 if behavior.reject_deregistration_protocol {
24205 write_mock_response(
24206 stream,
24207 "400 Bad Request",
24208 r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.17","requested_version":"1.19"}"#,
24209 );
24210 } else if behavior.reject_deregistration {
24211 write_mock_response(
24212 stream,
24213 "403 Forbidden",
24214 r#"{"reason":"authorization_failed","message":"worker cannot deregister"}"#,
24215 );
24216 } else {
24217 write_mock_response(
24218 stream,
24219 "200 OK",
24220 r#"{"worker_id":"deregistered-worker","outcome":"deregistered","recovered_workflow_task_count":2}"#,
24221 );
24222 }
24223 return;
24224 }
24225
24226 let is_poll = matches!(
24227 path,
24228 "/api/worker/workflow-tasks/poll"
24229 | "/api/worker/activity-tasks/poll"
24230 | "/api/worker/query-tasks/poll"
24231 );
24232 if is_poll && request_number <= behavior.long_poll_capacity_responses_per_path {
24233 write_mock_response(
24234 stream,
24235 "429 Too Many Requests",
24236 r#"{"task":null,"poll_status":"long_poll_capacity_exhausted","reason":"long_poll_capacity_exhausted","retryable":true,"retry_after_seconds":1}"#,
24237 );
24238 return;
24239 }
24240 if is_poll && request_number <= behavior.poll_failures_per_path {
24241 return;
24242 }
24243 if path == "/api/worker/heartbeat" && request_number <= behavior.heartbeat_failures {
24244 return;
24245 }
24246 if path == "/api/worker/heartbeat"
24247 && behavior.heartbeat_failure_request == Some(request_number)
24248 {
24249 return;
24250 }
24251 if path == "/api/worker/heartbeat"
24252 && behavior.delayed_heartbeat_request == Some(request_number)
24253 {
24254 thread::sleep(behavior.heartbeat_response_delay);
24255 }
24256 if behavior.unauthorized_polls && is_poll {
24257 write_mock_response(
24258 stream,
24259 "401 Unauthorized",
24260 r#"{"reason":"authentication_failed","message":"invalid worker token"}"#,
24261 );
24262 return;
24263 }
24264 if behavior.draining_polls && is_poll {
24265 write_mock_response(
24266 stream,
24267 "409 Conflict",
24268 r#"{"task":null,"poll_status":"draining","reason":"worker_draining","worker_status":"draining","drain_intent":"draining"}"#,
24269 );
24270 return;
24271 }
24272
24273 if let Some(codec_case) = behavior.invalid_task_payload_codec {
24274 if is_poll && request_number == 1 {
24275 let mut task = match path {
24276 "/api/worker/workflow-tasks/poll" => json!({
24277 "task_id": "codec-workflow",
24278 "workflow_type": "codec.workflow",
24279 "payload_codec": DEFAULT_CODEC,
24280 "workflow_task_attempt": 1,
24281 "lease_owner": "codec-worker"
24282 }),
24283 "/api/worker/activity-tasks/poll" => json!({
24284 "task_id": "codec-activity",
24285 "activity_attempt_id": "codec-activity-attempt",
24286 "activity_type": "codec.activity",
24287 "payload_codec": DEFAULT_CODEC,
24288 "attempt_number": 1,
24289 "lease_owner": "codec-worker"
24290 }),
24291 "/api/worker/query-tasks/poll" => json!({
24292 "query_task_id": "codec-query",
24293 "query_task_attempt": 1,
24294 "workflow_type": "codec.workflow",
24295 "query_name": "known",
24296 "payload_codec": DEFAULT_CODEC,
24297 "lease_owner": "codec-worker"
24298 }),
24299 _ => unreachable!("is_poll limits task codec probe paths"),
24300 };
24301 codec_case.apply(&mut task);
24302 write_mock_response(stream, "200 OK", &json!({"task": task}).to_string());
24303 return;
24304 }
24305
24306 if matches!(
24307 path,
24308 "/api/worker/workflow-tasks/codec-workflow/fail"
24309 | "/api/worker/activity-tasks/codec-activity/fail"
24310 | "/api/worker/query-tasks/codec-query/fail"
24311 ) {
24312 write_mock_response(stream, "200 OK", r#"{"outcome":"failed"}"#);
24313 return;
24314 }
24315 }
24316
24317 if behavior.reject_query_protocol && path.starts_with("/api/worker/query-tasks/") {
24318 let requested_version = worker_protocol.as_deref().unwrap_or("missing");
24319 let body = format!(
24320 r#"{{"reason":"unsupported_protocol_version","message":"Query tasks require worker protocol 1.8 or newer.","supported_version":"1.7","requested_version":"{requested_version}"}}"#
24321 );
24322 write_mock_response(stream, "400 Bad Request", &body);
24323 return;
24324 }
24325
24326 if behavior.reject_query_completion && path == "/api/worker/query-tasks/query-late/complete"
24327 {
24328 write_mock_response(
24329 stream,
24330 "409 Conflict",
24331 r#"{"reason":"query_task_timed_out","message":"query task timed out before completion"}"#,
24332 );
24333 return;
24334 }
24335
24336 if behavior.workflow_completion_status.is_some()
24337 && path == "/api/worker/workflow-tasks/poll"
24338 && request_number == 1
24339 {
24340 write_mock_response(
24341 stream,
24342 "200 OK",
24343 r#"{"task":{"task_id":"workflow-timeout-task","workflow_id":"reused-workflow-id","run_id":"run-selected-timeout","workflow_type":"timeout.workflow","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"workflow_task_attempt":3,"lease_owner":"timeout-worker"}}"#,
24344 );
24345 return;
24346 }
24347
24348 if path == "/api/worker/workflow-tasks/workflow-timeout-task/complete" {
24349 if let (Some(status), Some(body)) = (
24350 behavior.workflow_completion_status,
24351 behavior.workflow_completion_body,
24352 ) {
24353 write_mock_response(stream, status, body);
24354 return;
24355 }
24356 }
24357
24358 if behavior.waiting_query_worker {
24359 if behavior.complete_named_signal
24360 && path == "/api/worker/workflow-tasks/poll"
24361 && request_number == 1
24362 {
24363 let body = json!({
24364 "task": {
24365 "task_id": "snapshot-open",
24366 "workflow_id": "snapshot-1",
24367 "run_id": "snapshot-run-1",
24368 "workflow_type": "snapshot",
24369 "payload_codec": DEFAULT_CODEC,
24370 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
24371 .expect("Avro workflow arguments"),
24372 "history_events": [],
24373 "workflow_task_attempt": 1,
24374 "lease_owner": "rust-snapshot-worker"
24375 }
24376 })
24377 .to_string();
24378 write_mock_response(stream, "200 OK", &body);
24379 return;
24380 }
24381
24382 let signal_request = request_number - usize::from(behavior.complete_named_signal);
24383 let signal_request_limit = 2 + usize::from(behavior.complete_named_signal);
24384 if path == "/api/worker/workflow-tasks/poll"
24385 && signal_request >= 1
24386 && signal_request <= signal_request_limit
24387 {
24388 let finish = behavior.complete_named_signal && signal_request == 3;
24389 let amounts = if signal_request == 1 {
24390 vec![3]
24391 } else {
24392 vec![3, 5]
24393 };
24394 let task_id = if signal_request == 1 {
24395 "snapshot-wait-3"
24396 } else if finish {
24397 "snapshot-finish"
24398 } else {
24399 "snapshot-wait-5"
24400 };
24401 let mut history_events = std::iter::once(json!({
24402 "event_type": "SignalWaitOpened",
24403 "payload": {"sequence": 1, "signal_name": "finish"}
24404 }))
24405 .chain(amounts.iter().enumerate().map(|(index, amount)| {
24406 json!({
24407 "event_type": "SignalReceived",
24408 "payload": {
24409 "signal_id": format!("increment-{amount}"),
24410 "signal_name": "increment",
24411 "workflow_sequence": index + 2,
24412 "payload_codec": DEFAULT_CODEC,
24413 "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
24414 .expect("Avro signal envelope")
24415 }
24416 })
24417 }))
24418 .collect::<Vec<_>>();
24419 let (resume_id, resume_name, resume_arguments) = if finish {
24420 history_events.push(json!({
24421 "event_type": "SignalReceived",
24422 "payload": {
24423 "signal_id": "finish",
24424 "signal_name": "finish",
24425 "workflow_sequence": 4,
24426 "payload_codec": DEFAULT_CODEC,
24427 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
24428 .expect("Avro finish signal envelope")
24429 }
24430 }));
24431 (
24432 "finish".to_string(),
24433 "finish".to_string(),
24434 encode_value_envelope(&json!([]), DEFAULT_CODEC)
24435 .expect("Avro finish resume signal"),
24436 )
24437 } else {
24438 let amount = amounts.last().expect("amount");
24439 (
24440 format!("increment-{amount}"),
24441 "increment".to_string(),
24442 encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
24443 .expect("Avro increment resume signal"),
24444 )
24445 };
24446 let body = json!({
24447 "task": {
24448 "task_id": task_id,
24449 "workflow_id": "snapshot-1",
24450 "run_id": "snapshot-run-1",
24451 "workflow_type": "snapshot",
24452 "payload_codec": DEFAULT_CODEC,
24453 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
24454 .expect("Avro workflow arguments"),
24455 "history_events": history_events,
24456 "workflow_task_attempt": 1,
24457 "workflow_signal_id": resume_id,
24458 "signal_name": resume_name,
24459 "signal_arguments": resume_arguments,
24460 "lease_owner": "rust-snapshot-worker"
24461 }
24462 })
24463 .to_string();
24464 write_mock_response(stream, "200 OK", &body);
24465 return;
24466 }
24467
24468 if path == "/api/worker/query-tasks/poll" && request_number == 1 {
24469 let history_events = [3, 5]
24470 .into_iter()
24471 .enumerate()
24472 .map(|(index, amount)| {
24473 json!({
24474 "event_type": "SignalReceived",
24475 "payload": {
24476 "signal_id": format!("increment-{amount}"),
24477 "signal_name": "increment",
24478 "workflow_sequence": index + 2,
24479 "payload_codec": DEFAULT_CODEC,
24480 "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
24481 .expect("Avro query signal envelope")
24482 }
24483 })
24484 })
24485 .collect::<Vec<_>>();
24486 let body = json!({
24487 "task": {
24488 "query_task_id": "snapshot-current",
24489 "query_task_attempt": 1,
24490 "lease_owner": "rust-snapshot-worker",
24491 "workflow_id": "snapshot-1",
24492 "run_id": "snapshot-run-1",
24493 "workflow_type": "snapshot",
24494 "query_name": "current",
24495 "payload_codec": DEFAULT_CODEC,
24496 "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
24497 .expect("Avro workflow arguments"),
24498 "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
24499 .expect("Avro query arguments"),
24500 "history_events": history_events,
24501 "run_status": "waiting"
24502 }
24503 })
24504 .to_string();
24505 write_mock_response(stream, "200 OK", &body);
24506 return;
24507 }
24508
24509 if path == "/api/worker/workflow-tasks/snapshot-wait-3/fail"
24510 || path == "/api/worker/workflow-tasks/snapshot-wait-5/fail"
24511 {
24512 write_mock_response(
24513 stream,
24514 "200 OK",
24515 r#"{"outcome":"waiting_for_history","recorded":true}"#,
24516 );
24517 return;
24518 }
24519
24520 if path == "/api/worker/workflow-tasks/snapshot-open/complete" {
24521 write_mock_response(stream, "200 OK", r#"{"outcome":"waiting","recorded":true}"#);
24522 return;
24523 }
24524
24525 if path == "/api/worker/workflow-tasks/snapshot-finish/complete" {
24526 write_mock_response(
24527 stream,
24528 "200 OK",
24529 r#"{"outcome":"completed","run_status":"completed","recorded":true}"#,
24530 );
24531 return;
24532 }
24533
24534 if path == "/api/worker/query-tasks/snapshot-current/complete" {
24535 write_mock_response(stream, "200 OK", r#"{"outcome":"completed"}"#);
24536 return;
24537 }
24538 }
24539
24540 if matches!(
24541 path,
24542 "/api/workflows/typed-1/query/inspect" | "/api/workflows/typed-1/update/replace"
24543 ) {
24544 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
24545 .expect("typed mock result");
24546 let body = json!({
24547 "result": typed_fidelity_probe().into_json().expect("result projection"),
24548 "result_envelope": result,
24549 })
24550 .to_string();
24551 write_mock_response(stream, "200 OK", &body);
24552 return;
24553 }
24554
24555 if path == "/api/workflows/typed-1" {
24556 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
24557 .expect("typed mock result");
24558 let body = json!({
24559 "workflow_id": "typed-1",
24560 "run_id": "run-typed-1",
24561 "workflow_type": "typed.echo",
24562 "status": "completed",
24563 "output": typed_fidelity_probe().into_json().expect("output projection"),
24564 "output_envelope": result,
24565 })
24566 .to_string();
24567 write_mock_response(stream, "200 OK", &body);
24568 return;
24569 }
24570
24571 let (status, body) = match path {
24572 "/api/health" => ("200 OK", r#"{"status":"ok"}"#),
24573 "/api/workflows" => (
24574 "201 Created",
24575 r#"{"workflow_id":"wf-start-options","run_id":"run-start-options","workflow_type":"rust.timeout"}"#,
24576 ),
24577 "/api/worker/register" if behavior.decline_registration => (
24578 "200 OK",
24579 r#"{"worker_id":"declined-worker","registered":false}"#,
24580 ),
24581 "/api/worker/register" if behavior.waiting_query_worker => (
24582 "200 OK",
24583 r#"{"worker_id":"rust-snapshot-worker","registered":true,"heartbeat_interval_seconds":1}"#,
24584 ),
24585 "/api/worker/register" => (
24586 "200 OK",
24587 r#"{"worker_id":"mock-worker","registered":true,"heartbeat_interval_seconds":3600}"#,
24588 ),
24589 "/api/worker/heartbeat" => ("200 OK", "{}"),
24590 "/api/worker/activity-tasks/poll"
24591 if behavior.cancelled_activity && request_number == 1 =>
24592 {
24593 (
24594 "200 OK",
24595 r#"{"task":{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","activity_type":"cancel-aware","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"rust-cancel-worker"}}"#,
24596 )
24597 }
24598 "/api/worker/activity-tasks/poll"
24599 if behavior.long_poll_capacity_responses_per_path > 0
24600 && request_number
24601 == behavior
24602 .long_poll_capacity_responses_per_path
24603 .saturating_add(1) =>
24604 {
24605 (
24606 "200 OK",
24607 r#"{"task":{"task_id":"capacity-activity","activity_attempt_id":"capacity-attempt","activity_type":"capacity.activity","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"capacity-worker"}}"#,
24608 )
24609 }
24610 "/api/worker/activity-tasks/poll" | "/api/worker/workflow-tasks/poll" => {
24611 ("200 OK", r#"{"task":null}"#)
24612 }
24613 "/api/worker/query-tasks/poll"
24614 if behavior.reject_query_completion && request_number == 1 =>
24615 {
24616 (
24617 "200 OK",
24618 r#"{"task":{"query_task_id":"query-late","query_task_attempt":1,"lease_owner":"late-worker","workflow_id":"counter-late","run_id":"run-late","workflow_type":"counter","query_name":"current","payload_codec":"avro","workflow_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"query_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"run_status":"running"}}"#,
24619 )
24620 }
24621 "/api/worker/query-tasks/poll" => ("200 OK", r#"{"task":null}"#),
24622 "/api/worker/query-tasks/query-capture/complete"
24623 | "/api/worker/query-tasks/query-capture/fail" => ("200 OK", "{}"),
24624 "/api/worker/activity-tasks/activity-cancel/heartbeat" => (
24625 "200 OK",
24626 r#"{"activity_attempt_id":"attempt-cancel","cancel_requested":true,"can_continue":false,"reason":"run_cancelled","run_closed_reason":"cancelled","heartbeat_recorded":false}"#,
24627 ),
24628 "/api/worker/activity-tasks/activity-cancel/complete" => (
24629 "409 Conflict",
24630 r#"{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","reason":"run_cancelled","cancel_requested":true,"can_continue":false,"run_closed_reason":"cancelled"}"#,
24631 ),
24632 "/api/worker/activity-tasks/activity-typed/complete"
24633 | "/api/worker/activity-tasks/activity-typed/fail"
24634 | "/api/worker/activity-tasks/capacity-activity/complete"
24635 | "/api/workflows/typed-1/signal/changed" => ("200 OK", "{}"),
24636 "/api/workflows/counter-1/query/current" => (
24637 "200 OK",
24638 r#"{"workflow_id":"counter-1","query_name":"current","result":{"count":8},"result_envelope":{"codec":"avro","blob":"wwHioz3/VYAiNw4CCmNvdW50BBAA"}}"#,
24639 ),
24640 "/api/workflows/counter-1/query/missing" => (
24641 "404 Not Found",
24642 r#"{"workflow_id":"counter-1","query_name":"missing","reason":"rejected_unknown_query","message":"unknown query"}"#,
24643 ),
24644 "/api/workflows/wf-lifecycle/cancel" => (
24645 "200 OK",
24646 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","reason":"cleanup requested","command_status":"accepted"}"#,
24647 ),
24648 "/api/workflows/wf-lifecycle/terminate" => (
24649 "200 OK",
24650 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","reason":"forced stop","command_status":"accepted"}"#,
24651 ),
24652 "/api/workflows/wf-lifecycle/runs/run-current/cancel" => (
24653 "200 OK",
24654 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","command_status":"accepted"}"#,
24655 ),
24656 "/api/workflows/wf-lifecycle/runs/run-current/terminate" => (
24657 "200 OK",
24658 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","command_status":"accepted"}"#,
24659 ),
24660 "/api/workflows/wf-lifecycle/runs/run-stale/cancel"
24661 | "/api/workflows/wf-lifecycle/runs/run-stale/terminate" => (
24662 "409 Conflict",
24663 r#"{"workflow_id":"wf-lifecycle","run_id":"run-stale","reason":"historical_run_command_rejected","target_scope":"run","message":"Commands cannot target historical runs."}"#,
24664 ),
24665 "/api/workflows/wf-failed" | "/api/workflows/wf-failed/runs/run-failed" => (
24666 "200 OK",
24667 r#"{"workflow_id":"wf-failed","run_id":"run-failed","status":"failed","closed_reason":"failed","error":"payment failed","failure":{"message":"payment failed","failure_category":"application","exception_type":"PaymentError","exception_class":"billing::PaymentError","non_retryable":true,"exception":{"type":"PaymentError","class":"billing::PaymentError","message":"payment failed"},"failures":[{"id":"failure-17","failure_category":"application"}]}}"#,
24668 ),
24669 "/api/workflows/wf-cancelled" => (
24670 "200 OK",
24671 r#"{"workflow_id":"wf-cancelled","run_id":"run-cancelled","status":"cancelled","closed_reason":"cancelled","reason":"cleanup requested"}"#,
24672 ),
24673 "/api/workflows/wf-terminated" => (
24674 "200 OK",
24675 r#"{"workflow_id":"wf-terminated","run_id":"run-terminated","status":"terminated","closed_reason":"terminated","reason":"forced stop"}"#,
24676 ),
24677 "/api/workflows/wf-timed-out" => (
24678 "200 OK",
24679 r#"{"workflow_id":"wf-timed-out","run_id":"run-timed-out","status":"failed","closed_reason":"timed_out","reason":"run_timeout"}"#,
24680 ),
24681 "/api/workflows/wf-waiting" | "/api/workflows/wf-waiting/runs/run-waiting" => (
24682 "200 OK",
24683 r#"{"workflow_id":"wf-waiting","run_id":"run-waiting","status":"waiting"}"#,
24684 ),
24685 "/api/workflows/wf-selected" => (
24686 "200 OK",
24687 r#"{"workflow_id":"wf-selected","run_id":"run-current","status":"completed","output":"current run output"}"#,
24688 ),
24689 "/api/workflows/wf-selected/runs/run-selected" => (
24690 "200 OK",
24691 r#"{"workflow_id":"wf-selected","run_id":"run-selected","status":"cancelled","closed_reason":"cancelled","reason":"selected run cancelled"}"#,
24692 ),
24693 _ => ("404 Not Found", r#"{"message":"not found"}"#),
24694 };
24695 write_mock_response(stream, status, body);
24696 }
24697
24698 fn mock_request_is_complete(request: &[u8]) -> bool {
24699 let Some(header_end) = request
24700 .windows(4)
24701 .position(|window| window == b"\r\n\r\n")
24702 .map(|position| position + 4)
24703 else {
24704 return false;
24705 };
24706 let headers = String::from_utf8_lossy(&request[..header_end]);
24707 let content_length = headers.lines().find_map(|line| {
24708 let (name, value) = line.split_once(':')?;
24709 name.eq_ignore_ascii_case("content-length")
24710 .then(|| value.trim().parse::<usize>().ok())
24711 .flatten()
24712 });
24713
24714 request.len() >= header_end + content_length.unwrap_or(0)
24715 }
24716
24717 fn write_mock_response(stream: &mut TcpStream, status: &str, body: &str) {
24718 let response = format!(
24719 "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
24720 body.len()
24721 );
24722
24723 let _ = stream.write_all(response.as_bytes());
24724 let _ = stream.flush();
24725 }
24726}