1#![doc = include_str!("../README.md")]
2
3use std::{
4 any::{type_name, Any, TypeId},
5 collections::{BTreeMap, HashMap},
6 future::Future,
7 io::{self, Read},
8 pin::Pin,
9 sync::{
10 atomic::{AtomicBool, Ordering},
11 Arc, Mutex, OnceLock,
12 },
13 task::{Context as TaskContext, Poll},
14 time::{Duration, Instant, SystemTime, UNIX_EPOCH},
15};
16
17use apache_avro::{from_avro_datum, to_avro_datum, types::Value as AvroDatum, Schema};
18use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
19use chrono::DateTime;
20use futures_util::{future::OptionFuture, task::noop_waker_ref};
21use serde::{
22 de::DeserializeOwned,
23 ser::{SerializeMap, SerializeSeq},
24 Deserialize, Deserializer, Serialize, Serializer,
25};
26pub use serde_json::{json, Value};
27use sha2::{Digest, Sha256};
28use thiserror::Error;
29pub use uuid::Uuid;
30
31pub const WORKER_PROTOCOL_VERSION: &str = "1.19";
32pub const PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION: &str = "1.18";
34pub const CONTROL_PLANE_VERSION: &str = "2";
35pub const DEFAULT_CODEC: &str = "avro";
36pub const SDK_VERSION: &str = concat!("durable-workflow-rust/", env!("CARGO_PKG_VERSION"));
37pub const CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY: &str =
39 "condition_wait_occurrence_identity";
40pub const MEMO_UPSERTS_CAPABILITY: &str = "memo_upserts";
42pub const QUERY_TASKS_CAPABILITY: &str = "query_tasks";
44pub const TYPED_SEARCH_ATTRIBUTES_CAPABILITY: &str = "typed_search_attributes";
46pub const WORKFLOW_UPDATES_CAPABILITY: &str = "workflow_updates";
48pub const MESSAGE_STREAMS_CAPABILITY: &str = "message_streams";
50pub const DURABLE_SELECTION_CAPABILITY: &str = "durable_selection";
52pub const MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.15";
53pub const MESSAGE_STREAM_SIGNAL: &str = "__durable_workflow_message_stream";
54pub const MESSAGE_STREAM_SCHEMA: &str = "durable-workflow.v2.message-stream.message";
55pub const MESSAGE_STREAM_CURSOR_SCHEMA: &str = "durable-workflow.v2.message-stream.cursor";
56pub const MESSAGE_STREAM_MAX_BATCH: usize = 100;
57pub const QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
59pub const SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
61pub const MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.14";
63pub const TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.16";
65pub const CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.9";
67pub const CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.17";
69pub const DURABLE_SELECTION_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.19";
71
72pub fn worker_protocol_supports_message_streams(version: &str) -> bool {
73 let Some((major, minor)) = version.split_once('.') else {
74 return false;
75 };
76 major == "1" && minor.parse::<u64>().is_ok_and(|minor| minor >= 15)
77}
78
79fn validate_user_signal_name(signal_name: &str) -> Result<()> {
80 if signal_name == MESSAGE_STREAM_SIGNAL {
81 return Err(Error::Codec(format!(
82 "signal name {MESSAGE_STREAM_SIGNAL:?} is reserved by the workflow runtime"
83 )));
84 }
85 Ok(())
86}
87
88const MAX_LONG_POLL_TIMEOUT_SECONDS: u64 = 60;
89const WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE: &str =
90 "Workflow task waiting for scheduled history.";
91const WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE: &str = "WorkflowTaskWaitingForHistory";
92const MISSING_TASK_PAYLOAD_CODEC: &str = "\0missing-task-payload-codec";
93const NULL_TASK_PAYLOAD_CODEC: &str = "\0null-task-payload-codec";
94const NON_STRING_TASK_PAYLOAD_CODEC: &str = "\0non-string-task-payload-codec";
95const MAX_MEMO_ENTRIES: usize = 100;
96const MAX_MEMO_VALUE_SIZE_BYTES: usize = 10_240;
97const MAX_MEMO_TOTAL_SIZE_BYTES: usize = 65_536;
98
99const QUERY_TASK_FINAL_REJECTION_REASONS: &[&str] = &[
100 "lease_expired",
101 "query_task_not_found",
102 "query_task_not_leased",
103 "query_task_timed_out",
104];
105
106pub fn portable_worker_affinity_capability_manifest() -> Value {
108 json!({
109 "local_activities": {
110 "supported": false,
111 "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
112 "reason": "rust_worker_does_not_execute_record_local_activity",
113 },
114 "worker_sessions": {
115 "supported": false,
116 "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
117 "reason": "rust_worker_has_no_typed_session_lifecycle",
118 },
119 "sticky_execution": {
120 "supported": false,
121 "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
122 "reason": "rust_worker_uses_complete_durable_history_replay",
123 },
124 })
125}
126
127pub const AVRO_VALUE_SCHEMA_JSON: &str =
129 include_str!("../schema/durable_workflow.protocol.Value.v1.avsc");
130pub const AVRO_VALUE_SCHEMA_FINGERPRINT_HEX: &str = "e2a33dff55802237";
131pub const AVRO_VALUE_SCHEMA_FINGERPRINT: [u8; 8] = [0xe2, 0xa3, 0x3d, 0xff, 0x55, 0x80, 0x22, 0x37];
132const AVRO_SINGLE_OBJECT_MAGIC: [u8; 2] = [0xc3, 0x01];
133
134static AVRO_VALUE_SCHEMA: OnceLock<std::result::Result<Schema, String>> = OnceLock::new();
135static AVRO_VALUE_ORDERED_MAP_ENCODING_SCHEMA: OnceLock<std::result::Result<Schema, String>> =
136 OnceLock::new();
137
138#[derive(Clone, Copy)]
139enum RequestProtocol {
140 ControlPlane,
141 Worker(&'static str),
142}
143
144pub type Result<T> = std::result::Result<T, Error>;
145
146#[derive(Debug, Error)]
147pub enum Error {
148 #[error("transport error: {0}")]
149 Transport(#[from] reqwest::Error),
150 #[error(
151 "invalid Durable Workflow base URL: omit the SDK-owned /api suffix and pass the Server or Cloud runtime base URL; the SDK appends /api automatically"
152 )]
153 InvalidBaseUrl,
154 #[error("json error: {0}")]
155 Json(#[from] serde_json::Error),
156 #[error("http {status}: {body}")]
157 Http {
158 status: reqwest::StatusCode,
159 body: String,
160 },
161 #[error("codec error: {0}")]
162 Codec(String),
163 #[error(transparent)]
164 QueryFailed(QueryFailure),
165 #[error(transparent)]
166 Protocol(ProtocolFailure),
167 #[error(transparent)]
168 NonDeterministicReplay(ReplayFailure),
169 #[error(transparent)]
170 ChildWorkflowFailed(ChildWorkflowFailure),
171 #[error(transparent)]
172 ActivityFailed(ActivityFailure),
173 #[error(transparent)]
174 ParallelFailed(ParallelFailure),
175 #[error(transparent)]
176 SagaCompensationFailed(SagaCompensationFailure),
177 #[error(transparent)]
178 InvalidParallelGroup(ParallelGroupError),
179 #[error(transparent)]
180 DurableOperationCancelled(DurableOperationCancelled),
181 #[error(transparent)]
182 WorkflowCancellationRequested(WorkflowCancellationRequested),
183 #[error(transparent)]
184 WorkflowCommandRejected(WorkflowCommandRejection),
185 #[error(transparent)]
186 WorkflowFailed(WorkflowTerminalOutcome),
187 #[error(transparent)]
188 WorkflowCancelled(WorkflowTerminalOutcome),
189 #[error(transparent)]
190 WorkflowTerminated(WorkflowTerminalOutcome),
191 #[error(transparent)]
192 WorkflowTimedOut(WorkflowTerminalOutcome),
193 #[error(transparent)]
194 ActivityTaskRejected(ActivityTaskRejection),
195 #[error("workflow handler {0:?} is not registered")]
196 WorkflowNotRegistered(String),
197 #[error("activity handler {0:?} is not registered")]
198 ActivityNotRegistered(String),
199 #[error(
200 "{handler_kind} handler {handler_name:?} {value_kind} type {rust_type} is incompatible with the fixed Avro Value codec: {message}"
201 )]
202 HandlerType {
203 handler_kind: HandlerKind,
204 handler_name: String,
205 value_kind: HandlerValueKind,
206 rust_type: &'static str,
207 message: String,
208 },
209 #[error("workflow future yielded without emitting a durable command")]
210 WorkflowYieldedWithoutCommand,
211 #[error(
212 "workflow_stream_command_identity_missing: workflow stream authoring requires a non-empty server-provided workflow_command_id"
213 )]
214 MissingWorkflowCommandIdentity,
215 #[error("workflow state lock is poisoned")]
216 WorkflowStatePoisoned,
217 #[error("timer duration is too large for the worker protocol")]
218 TimerDurationOverflow,
219 #[error(transparent)]
220 InvalidConditionWaitOptions(#[from] ConditionWaitOptionsError),
221 #[error(transparent)]
222 InvalidSearchAttributeUpdate(#[from] SearchAttributeUpdateError),
223 #[error("operation timed out")]
224 Timeout,
225 #[error(
226 "missing {role}-plane credentials: configure ClientBuilder::{role}_token or ClientBuilder::token; a {opposite_role}-plane token cannot authorize this request"
227 )]
228 MissingRoleCredentials {
229 role: &'static str,
230 opposite_role: &'static str,
231 },
232 #[error("worker loop error: {0}")]
233 WorkerLoop(String),
234 #[error(
235 "workflow command contract for {workflow_type:?} declares update validators, but this Rust SDK cannot execute synchronous pre-accept update validation"
236 )]
237 UnsupportedUpdateValidators { workflow_type: String },
238 #[error("{primary}; worker deregistration also failed: {deregistration}")]
239 WorkerShutdown {
240 primary: Box<Error>,
241 deregistration: Box<Error>,
242 },
243 #[error("invalid child workflow options: {0}")]
244 InvalidChildWorkflowOptions(String),
245 #[error("invalid workflow memo update: {0}")]
246 InvalidMemoUpdate(String),
247 #[error(
248 "workflow_memo_updates_unavailable: the connected runtime did not advertise workflow memo update support"
249 )]
250 WorkflowMemoUpdatesUnavailable,
251 #[error(transparent)]
252 InvalidActivityOptions(ActivityOptionsError),
253 #[error(transparent)]
254 InvalidContinueAsNewOptions(#[from] ContinueAsNewOptionsError),
255 #[doc(hidden)]
256 #[error("workflow requested continue as new")]
257 ContinueAsNew(ContinueAsNewRequest),
258}
259
260#[derive(Clone, Debug, Error, PartialEq, Eq)]
262pub enum ConditionWaitOptionsError {
263 #[error("condition_key must be non-empty")]
264 EmptyKey,
265 #[error("condition_definition_fingerprint must be non-empty")]
266 EmptyPredicateIdentity,
267 #[error("condition timeout is too large for the worker protocol")]
268 TimeoutOverflow,
269}
270
271#[derive(Clone, Debug, PartialEq, Eq)]
278pub struct ConditionWaitOptions {
279 condition_key: String,
280 predicate_identity: String,
281 timeout: Option<Duration>,
282}
283
284impl ConditionWaitOptions {
285 pub fn new(condition_key: impl Into<String>, predicate_identity: impl Into<String>) -> Self {
286 Self {
287 condition_key: condition_key.into(),
288 predicate_identity: predicate_identity.into(),
289 timeout: None,
290 }
291 }
292
293 pub fn timeout(mut self, timeout: Duration) -> Self {
294 self.timeout = Some(timeout);
295 self
296 }
297
298 fn validate(
299 &self,
300 ) -> std::result::Result<ValidatedConditionWaitOptions, ConditionWaitOptionsError> {
301 let condition_key = self.condition_key.trim();
302 if condition_key.is_empty() {
303 return Err(ConditionWaitOptionsError::EmptyKey);
304 }
305 let predicate_identity = self.predicate_identity.trim();
306 if predicate_identity.is_empty() {
307 return Err(ConditionWaitOptionsError::EmptyPredicateIdentity);
308 }
309 let timeout_seconds = self
310 .timeout
311 .map(|timeout| {
312 timeout
313 .as_secs()
314 .checked_add(u64::from(timeout.subsec_nanos() > 0))
315 .ok_or(ConditionWaitOptionsError::TimeoutOverflow)
316 })
317 .transpose()?;
318
319 Ok(ValidatedConditionWaitOptions {
320 condition_key: condition_key.to_string(),
321 predicate_identity: predicate_identity.to_string(),
322 timeout_seconds,
323 })
324 }
325}
326
327#[derive(Clone, Debug, PartialEq, Eq)]
328struct ValidatedConditionWaitOptions {
329 condition_key: String,
330 predicate_identity: String,
331 timeout_seconds: Option<u64>,
332}
333
334const CONDITION_WAIT_OCCURRENCE_PREFIX: &str = "rust:condition-wait:";
335
336#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
338#[serde(rename_all = "snake_case")]
339pub enum ConditionWaitResult {
340 Satisfied,
341 TimedOut,
342}
343
344impl ConditionWaitResult {
345 pub fn is_satisfied(self) -> bool {
346 self == Self::Satisfied
347 }
348
349 pub fn is_timed_out(self) -> bool {
350 self == Self::TimedOut
351 }
352}
353
354#[doc(hidden)]
356pub fn __condition_definition_fingerprint(source: &str) -> String {
357 let mut digest = Sha256::new();
358 digest.update(b"durable-workflow-rust.wait-condition.v1\0");
359 digest.update(source.as_bytes());
360 format!("sha256:{:x}", digest.finalize())
361}
362
363#[macro_export]
369macro_rules! wait_condition {
370 ($ctx:expr, $key:expr, timeout: $timeout:expr, $predicate:expr $(,)?) => {{
371 $ctx.wait_condition(
372 $crate::ConditionWaitOptions::new(
373 $key,
374 $crate::__condition_definition_fingerprint(concat!(
375 module_path!(),
376 "\0",
377 stringify!($predicate)
378 )),
379 )
380 .timeout($timeout),
381 $predicate,
382 )
383 }};
384 ($ctx:expr, $key:expr, $predicate:expr $(,)?) => {{
385 $ctx.wait_condition(
386 $crate::ConditionWaitOptions::new(
387 $key,
388 $crate::__condition_definition_fingerprint(concat!(
389 module_path!(),
390 "\0",
391 stringify!($predicate)
392 )),
393 ),
394 $predicate,
395 )
396 }};
397}
398
399const MAX_SEARCH_ATTRIBUTES_PER_UPDATE: usize = 100;
400const MAX_SEARCH_ATTRIBUTE_KEY_LENGTH: usize = 64;
401const MAX_SEARCH_ATTRIBUTE_STRING_LENGTH: usize = 2_048;
402const MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH: usize = 255;
403const MAX_SEARCH_ATTRIBUTE_UPDATE_BYTES: usize = 65_536;
404
405#[derive(Clone, Debug, Error, PartialEq, Eq)]
407pub enum SearchAttributeUpdateError {
408 #[error("search-attribute update requires at least one attribute")]
409 Empty,
410 #[error("search attribute key {0:?} must be 1-64 URL-safe ASCII characters")]
411 InvalidKey(String),
412 #[error("search-attribute update exceeds the limit of 100 attributes")]
413 TooManyAttributes,
414 #[error("search attribute {key:?} {kind} value exceeds {limit} bytes")]
415 ValueTooLong {
416 key: String,
417 kind: &'static str,
418 limit: usize,
419 },
420 #[error(
421 "search attribute {0:?} must not contain an empty string value; use delete() to remove it"
422 )]
423 EmptyString(String),
424 #[error("search attribute {0:?} has a non-finite float value")]
425 NonFiniteFloat(String),
426 #[error("search attribute {0:?} must use an RFC 3339 datetime with an explicit timezone")]
427 InvalidDateTime(String),
428 #[error("search-attribute update exceeds the 65536-byte protocol limit")]
429 PayloadTooLarge,
430}
431
432#[derive(Clone, Debug, PartialEq)]
434pub enum SearchAttributeValue {
435 String(String),
436 Keyword(String),
437 KeywordList(Vec<String>),
438 Int(i64),
439 Float(f64),
440 Bool(bool),
441 DateTime(String),
442 Delete,
443}
444
445impl SearchAttributeValue {
446 fn type_name(&self) -> Option<&'static str> {
447 match self {
448 Self::String(_) => Some("string"),
449 Self::Keyword(_) => Some("keyword"),
450 Self::KeywordList(_) => Some("keyword_list"),
451 Self::Int(_) => Some("int"),
452 Self::Float(_) => Some("float"),
453 Self::Bool(_) => Some("bool"),
454 Self::DateTime(_) => Some("datetime"),
455 Self::Delete => None,
456 }
457 }
458
459 fn normalized(self, key: &str) -> std::result::Result<Self, SearchAttributeUpdateError> {
460 let normalize_string = |value: String, kind: &'static str, limit: usize| {
461 let value = value.trim().to_string();
462 if value.is_empty() {
463 return Err(SearchAttributeUpdateError::EmptyString(key.to_string()));
464 }
465 if value.len() > limit {
466 return Err(SearchAttributeUpdateError::ValueTooLong {
467 key: key.to_string(),
468 kind,
469 limit,
470 });
471 }
472 Ok(value)
473 };
474
475 match self {
476 Self::String(value) => Ok(Self::String(normalize_string(
477 value,
478 "string",
479 MAX_SEARCH_ATTRIBUTE_STRING_LENGTH,
480 )?)),
481 Self::Keyword(value) => Ok(Self::Keyword(normalize_string(
482 value,
483 "keyword",
484 MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH,
485 )?)),
486 Self::KeywordList(values) => {
487 let values = values
488 .into_iter()
489 .map(|value| {
490 let value = value.trim().to_string();
491 if value.len() > MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH {
492 return Err(SearchAttributeUpdateError::ValueTooLong {
493 key: key.to_string(),
494 kind: "keyword-list entry",
495 limit: MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH,
496 });
497 }
498 Ok(value)
499 })
500 .collect::<std::result::Result<Vec<_>, _>>()?;
501 Ok(Self::KeywordList(values))
502 }
503 Self::Float(value) if !value.is_finite() => {
504 Err(SearchAttributeUpdateError::NonFiniteFloat(key.to_string()))
505 }
506 Self::DateTime(value) => {
507 let value =
508 normalize_string(value, "datetime", MAX_SEARCH_ATTRIBUTE_STRING_LENGTH)?;
509 if DateTime::parse_from_rfc3339(&value).is_err() {
510 return Err(SearchAttributeUpdateError::InvalidDateTime(key.to_string()));
511 }
512 Ok(Self::DateTime(value))
513 }
514 value => Ok(value),
515 }
516 }
517
518 fn into_json(self) -> Value {
519 match self {
520 Self::String(value) | Self::Keyword(value) | Self::DateTime(value) => {
521 Value::String(value)
522 }
523 Self::KeywordList(values) => {
524 Value::Array(values.into_iter().map(Value::String).collect())
525 }
526 Self::Int(value) => json!(value),
527 Self::Float(value) => json!(value),
528 Self::Bool(value) => json!(value),
529 Self::Delete => Value::Null,
530 }
531 }
532}
533
534#[derive(Clone, Debug, Default, PartialEq)]
536pub struct SearchAttributeUpdate {
537 attributes: BTreeMap<String, SearchAttributeValue>,
538}
539
540impl SearchAttributeUpdate {
541 pub fn new() -> Self {
542 Self::default()
543 }
544
545 pub fn set(
546 mut self,
547 key: impl Into<String>,
548 value: SearchAttributeValue,
549 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
550 let key = key.into();
551 validate_search_attribute_key(&key)?;
552 if !self.attributes.contains_key(&key)
553 && self.attributes.len() >= MAX_SEARCH_ATTRIBUTES_PER_UPDATE
554 {
555 return Err(SearchAttributeUpdateError::TooManyAttributes);
556 }
557 self.attributes.insert(key.clone(), value.normalized(&key)?);
558 self.validate_size()?;
559 Ok(self)
560 }
561
562 pub fn string(
563 self,
564 key: impl Into<String>,
565 value: impl Into<String>,
566 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
567 self.set(key, SearchAttributeValue::String(value.into()))
568 }
569
570 pub fn keyword(
571 self,
572 key: impl Into<String>,
573 value: impl Into<String>,
574 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
575 self.set(key, SearchAttributeValue::Keyword(value.into()))
576 }
577
578 pub fn keyword_list<I, V>(
579 self,
580 key: impl Into<String>,
581 values: I,
582 ) -> std::result::Result<Self, SearchAttributeUpdateError>
583 where
584 I: IntoIterator<Item = V>,
585 V: Into<String>,
586 {
587 self.set(
588 key,
589 SearchAttributeValue::KeywordList(values.into_iter().map(Into::into).collect()),
590 )
591 }
592
593 pub fn int(
594 self,
595 key: impl Into<String>,
596 value: i64,
597 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
598 self.set(key, SearchAttributeValue::Int(value))
599 }
600
601 pub fn float(
602 self,
603 key: impl Into<String>,
604 value: f64,
605 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
606 self.set(key, SearchAttributeValue::Float(value))
607 }
608
609 pub fn bool(
610 self,
611 key: impl Into<String>,
612 value: bool,
613 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
614 self.set(key, SearchAttributeValue::Bool(value))
615 }
616
617 pub fn datetime(
618 self,
619 key: impl Into<String>,
620 value: impl Into<String>,
621 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
622 self.set(key, SearchAttributeValue::DateTime(value.into()))
623 }
624
625 pub fn delete(
626 self,
627 key: impl Into<String>,
628 ) -> std::result::Result<Self, SearchAttributeUpdateError> {
629 self.set(key, SearchAttributeValue::Delete)
630 }
631
632 fn validate_size(&self) -> std::result::Result<(), SearchAttributeUpdateError> {
633 let (attributes, _) = self.clone().into_wire_parts();
634 if serde_json::to_vec(&attributes)
635 .map(|payload| payload.len() > MAX_SEARCH_ATTRIBUTE_UPDATE_BYTES)
636 .unwrap_or(true)
637 {
638 return Err(SearchAttributeUpdateError::PayloadTooLarge);
639 }
640 Ok(())
641 }
642
643 fn into_wire_parts(self) -> (Value, BTreeMap<String, String>) {
644 let mut attributes = serde_json::Map::new();
645 let mut attribute_types = BTreeMap::new();
646 for (key, value) in self.attributes {
647 if let Some(type_name) = value.type_name() {
648 attribute_types.insert(key.clone(), type_name.to_string());
649 }
650 attributes.insert(key, value.into_json());
651 }
652 (Value::Object(attributes), attribute_types)
653 }
654
655 fn validate(&self) -> std::result::Result<(), SearchAttributeUpdateError> {
656 if self.attributes.is_empty() {
657 return Err(SearchAttributeUpdateError::Empty);
658 }
659 self.validate_size()
660 }
661}
662
663fn validate_search_attribute_key(key: &str) -> std::result::Result<(), SearchAttributeUpdateError> {
664 let valid = !key.is_empty()
665 && key.len() <= MAX_SEARCH_ATTRIBUTE_KEY_LENGTH
666 && key
667 .bytes()
668 .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-' | b':'));
669 if valid {
670 Ok(())
671 } else {
672 Err(SearchAttributeUpdateError::InvalidKey(key.to_string()))
673 }
674}
675
676#[derive(Clone, Copy, Debug, PartialEq, Eq)]
678pub enum HandlerKind {
679 Workflow,
680 Activity,
681}
682
683impl std::fmt::Display for HandlerKind {
684 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
685 formatter.write_str(match self {
686 Self::Workflow => "workflow",
687 Self::Activity => "activity",
688 })
689 }
690}
691
692#[derive(Clone, Copy, Debug, PartialEq, Eq)]
694pub enum HandlerValueKind {
695 Input,
696 Result,
697}
698
699impl std::fmt::Display for HandlerValueKind {
700 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
701 formatter.write_str(match self {
702 Self::Input => "input",
703 Self::Result => "result",
704 })
705 }
706}
707
708#[derive(Clone, Copy, Debug, PartialEq, Eq)]
710pub enum WorkflowCommandKind {
711 Cancel,
712 Terminate,
713}
714
715impl WorkflowCommandKind {
716 fn as_str(self) -> &'static str {
717 match self {
718 Self::Cancel => "cancel",
719 Self::Terminate => "terminate",
720 }
721 }
722}
723
724#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
726pub struct WorkflowCommandOptions {
727 #[serde(skip_serializing_if = "Option::is_none")]
728 pub reason: Option<String>,
729 #[serde(skip_serializing_if = "Option::is_none")]
730 pub request_id: Option<String>,
731}
732
733#[derive(Clone, Debug, PartialEq, Eq)]
740pub struct WorkflowStartOptions {
741 pub execution_timeout_seconds: u64,
742 pub run_timeout_seconds: u64,
743}
744
745impl Default for WorkflowStartOptions {
746 fn default() -> Self {
747 Self {
748 execution_timeout_seconds: 3600,
749 run_timeout_seconds: 600,
750 }
751 }
752}
753
754impl WorkflowStartOptions {
755 pub fn new() -> Self {
756 Self::default()
757 }
758
759 pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
760 self.execution_timeout_seconds = seconds;
761 self
762 }
763
764 pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
765 self.run_timeout_seconds = seconds;
766 self
767 }
768
769 fn validate(&self) -> Result<()> {
770 if self.execution_timeout_seconds == 0 {
771 return Err(Error::Codec(
772 "execution_timeout_seconds must be at least 1".to_string(),
773 ));
774 }
775 if self.run_timeout_seconds == 0 {
776 return Err(Error::Codec(
777 "run_timeout_seconds must be at least 1".to_string(),
778 ));
779 }
780 if self.run_timeout_seconds > self.execution_timeout_seconds {
781 return Err(Error::Codec(
782 "run_timeout_seconds cannot exceed execution_timeout_seconds".to_string(),
783 ));
784 }
785
786 Ok(())
787 }
788}
789
790#[derive(Clone, Debug, Default, PartialEq, Eq)]
795pub struct ContinueAsNewOptions {
796 pub workflow_type: Option<String>,
797 pub task_queue: Option<String>,
798}
799
800impl ContinueAsNewOptions {
801 pub fn new() -> Self {
802 Self::default()
803 }
804
805 pub fn workflow_type(mut self, workflow_type: impl Into<String>) -> Self {
806 self.workflow_type = Some(workflow_type.into());
807 self
808 }
809
810 pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
811 self.task_queue = Some(task_queue.into());
812 self
813 }
814
815 fn validate(&self) -> std::result::Result<(), ContinueAsNewOptionsError> {
816 for (field, value) in [
817 ("workflow_type", self.workflow_type.as_deref()),
818 ("task_queue", self.task_queue.as_deref()),
819 ] {
820 if value.is_some_and(|value| value.trim().is_empty()) {
821 return Err(ContinueAsNewOptionsError {
822 field,
823 message: format!("{field} must not be empty"),
824 });
825 }
826 }
827 Ok(())
828 }
829}
830
831#[derive(Clone, Debug, Error, PartialEq, Eq)]
833#[error("invalid continue-as-new option {field}: {message}")]
834pub struct ContinueAsNewOptionsError {
835 pub field: &'static str,
836 pub message: String,
837}
838
839#[derive(Clone, Debug, Default, PartialEq, Eq)]
841pub struct WorkflowHistoryBudget {
842 pub event_count: u64,
843 pub size_bytes: Option<u64>,
844 pub continue_as_new_recommended: bool,
845 pub pressure: Option<String>,
846}
847
848#[doc(hidden)]
849#[derive(Clone, Debug)]
850pub struct ContinueAsNewRequest {
851 arguments: AvroValue,
852 options: ContinueAsNewOptions,
853}
854
855impl WorkflowCommandOptions {
856 pub fn new() -> Self {
857 Self::default()
858 }
859
860 pub fn reason(mut self, reason: impl Into<String>) -> Self {
861 self.reason = Some(reason.into());
862 self
863 }
864
865 pub fn request_id(mut self, request_id: impl Into<String>) -> Self {
866 self.request_id = Some(request_id.into());
867 self
868 }
869}
870
871#[derive(Clone, Debug, PartialEq)]
873pub struct WorkflowCommandResult {
874 pub command: WorkflowCommandKind,
875 pub workflow_id: String,
876 pub run_id: Option<String>,
877 pub outcome: Option<String>,
878 pub reason: Option<String>,
879 pub command_status: Option<String>,
880 pub raw: Value,
881}
882
883#[derive(Clone, Debug, Error)]
885#[error("workflow {command:?} rejected ({reason}, HTTP {status}): {message}")]
886pub struct WorkflowCommandRejection {
887 pub command: WorkflowCommandKind,
888 pub status: u16,
889 pub reason: String,
890 pub message: String,
891 pub workflow_id: String,
892 pub run_id: Option<String>,
893 pub target_scope: Option<String>,
894 pub body: Value,
895}
896
897#[derive(Clone, Copy, Debug, PartialEq, Eq)]
899pub enum WorkflowTerminalKind {
900 Failed,
901 Cancelled,
902 Terminated,
903 TimedOut,
904}
905
906#[derive(Clone, Debug, Error)]
912#[error("workflow {workflow_id} run {run_id:?} ended as {kind:?} ({reason})")]
913pub struct WorkflowTerminalOutcome {
914 pub kind: WorkflowTerminalKind,
915 pub workflow_id: String,
916 pub run_id: Option<String>,
917 pub reason: String,
918 pub failure_category: Option<String>,
919 pub failure_id: Option<String>,
920 pub exception_type: Option<String>,
921 pub exception_class: Option<String>,
922 pub non_retryable: Option<bool>,
923 pub message: Option<String>,
924 pub exception: Option<Value>,
925 pub raw: Value,
926}
927
928#[derive(Clone, Debug, Error)]
930#[error("activity task {operation} rejected ({reason}, HTTP {status})")]
931pub struct ActivityTaskRejection {
932 pub operation: String,
933 pub status: u16,
934 pub reason: String,
935 pub task_id: String,
936 pub activity_attempt_id: String,
937 pub cancel_requested: bool,
938 pub can_continue: Option<bool>,
939 pub run_closed_reason: Option<String>,
940 pub body: Value,
941}
942
943#[derive(Clone, Copy, Debug, PartialEq, Eq)]
945pub enum ActivityOptionsErrorKind {
946 EmptyTaskQueue,
947 EmptyRetryPolicy,
948 InvalidMaxAttempts,
949 BackoffWithoutRetryBudget,
950 TooManyBackoffIntervals,
951 InvalidBackoffCoefficient,
952 BackoffGenerationTooLarge,
953 BackoffOverflow,
954 EmptyNonRetryableErrorType,
955 TimeoutNotPositive,
956 TimeoutOverflow,
957 TimeoutOrder,
958}
959
960#[derive(Clone, Debug, Error, PartialEq, Eq)]
962#[error("invalid activity options ({kind:?}, {field:?}): {message}")]
963pub struct ActivityOptionsError {
964 pub kind: ActivityOptionsErrorKind,
965 pub field: Option<&'static str>,
966 pub message: String,
967}
968
969impl ActivityOptionsError {
970 fn new(
971 kind: ActivityOptionsErrorKind,
972 field: Option<&'static str>,
973 message: impl Into<String>,
974 ) -> Self {
975 Self {
976 kind,
977 field,
978 message: message.into(),
979 }
980 }
981}
982
983#[derive(Clone, Copy, Debug, PartialEq, Eq)]
985pub enum ActivityFailureKind {
986 Failed,
987 Cancelled,
988 TimedOut,
989}
990
991#[derive(Clone, Debug, Error)]
996#[error("activity failed ({reason}): {message}")]
997pub struct ActivityFailure {
998 pub kind: ActivityFailureKind,
999 pub reason: String,
1000 pub message: String,
1001 pub activity_execution_id: Option<String>,
1002 pub activity_attempt_id: Option<String>,
1003 pub activity_type: Option<String>,
1004 pub activity_class: Option<String>,
1005 pub attempt_number: Option<u64>,
1006 pub failure_id: Option<String>,
1007 pub failure_category: Option<String>,
1008 pub timeout_kind: Option<String>,
1009 pub non_retryable: bool,
1010 pub exception_type: Option<String>,
1011 pub exception_class: Option<String>,
1012 pub code: Option<Value>,
1013 pub exception: Option<Value>,
1014}
1015
1016#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1018pub enum ChildWorkflowFailureKind {
1019 Failed,
1020 Cancelled,
1021 Terminated,
1022}
1023
1024#[derive(Clone, Debug, Error)]
1030#[error("child workflow failed ({reason}): {message}")]
1031pub struct ChildWorkflowFailure {
1032 pub kind: ChildWorkflowFailureKind,
1033 pub reason: String,
1034 pub message: String,
1035 pub parent_workflow_id: Option<String>,
1036 pub parent_workflow_run_id: Option<String>,
1037 pub child_workflow_id: Option<String>,
1038 pub child_workflow_run_id: Option<String>,
1039 pub child_workflow_type: Option<String>,
1040 pub failure_id: Option<String>,
1041 pub failure_category: Option<String>,
1042 pub exception_type: Option<String>,
1043 pub exception_class: Option<String>,
1044 pub non_retryable: bool,
1045 pub code: Option<Value>,
1046 pub exception: Option<Value>,
1047}
1048
1049#[derive(Clone, Debug, PartialEq, Eq)]
1051pub struct WorkflowIdentity {
1052 pub workflow_id: Option<String>,
1053 pub run_id: Option<String>,
1054}
1055
1056#[derive(Clone, Debug, PartialEq)]
1058pub struct ChildWorkflowResult {
1059 pub parent: WorkflowIdentity,
1060 pub child: WorkflowIdentity,
1061 pub child_workflow_type: Option<String>,
1062 pub result: Value,
1063}
1064
1065#[derive(Clone, Debug, PartialEq)]
1067pub struct ChildWorkflowAvroResult {
1068 pub parent: WorkflowIdentity,
1069 pub child: WorkflowIdentity,
1070 pub child_workflow_type: Option<String>,
1071 pub result: AvroValue,
1072}
1073
1074#[derive(Clone, Debug, Deserialize, Hash, PartialEq, Eq, Serialize)]
1076#[serde(untagged)]
1077pub enum SelectionKey {
1078 Index(usize),
1079 Name(String),
1080}
1081
1082impl From<usize> for SelectionKey {
1083 fn from(value: usize) -> Self {
1084 Self::Index(value)
1085 }
1086}
1087
1088impl From<String> for SelectionKey {
1089 fn from(value: String) -> Self {
1090 Self::Name(value)
1091 }
1092}
1093
1094impl From<&str> for SelectionKey {
1095 fn from(value: &str) -> Self {
1096 Self::Name(value.to_string())
1097 }
1098}
1099
1100#[derive(Clone, Debug, Error, PartialEq, Eq)]
1102#[error("selected {operation_kind} operation {operation_identity} was explicitly cancelled")]
1103pub struct DurableOperationCancelled {
1104 pub selection_group_id: String,
1105 pub member_key: SelectionKey,
1106 pub member_index: usize,
1107 pub operation_kind: String,
1108 pub operation_identity: String,
1109}
1110
1111#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
1117pub struct ParallelGroupMetadata {
1118 pub parallel_group_id: String,
1119 pub parallel_group_kind: String,
1120 pub parallel_group_base_sequence: u64,
1121 pub parallel_group_size: usize,
1122 pub parallel_group_index: usize,
1123 #[serde(default, skip_serializing_if = "Option::is_none")]
1124 pub parallel_group_mode: Option<String>,
1125 #[serde(default, skip_serializing_if = "Option::is_none")]
1126 pub selection_member_key: Option<SelectionKey>,
1127 #[serde(default, skip_serializing_if = "Option::is_none")]
1128 pub selection_member_index: Option<usize>,
1129 #[serde(default, skip_serializing_if = "Option::is_none")]
1130 pub selection_member_base_sequence: Option<u64>,
1131 #[serde(default, skip_serializing_if = "Option::is_none")]
1132 pub selection_member_size: Option<usize>,
1133 #[serde(default, skip_serializing_if = "Option::is_none")]
1134 pub selection_member_kind: Option<String>,
1135}
1136
1137#[derive(Clone, Debug, PartialEq)]
1139pub enum ParallelResult {
1140 Activity(Value),
1141 ChildWorkflow(ChildWorkflowResult),
1142 Timer,
1143 Signal(Vec<Value>),
1144 Condition(ConditionWaitResult),
1145 Group(Vec<ParallelResult>),
1146}
1147
1148#[derive(Clone, Debug, PartialEq)]
1150pub enum ParallelAvroResult {
1151 Activity(AvroValue),
1152 ChildWorkflow(ChildWorkflowAvroResult),
1153 Timer,
1154 Signal(Vec<AvroValue>),
1155 Condition(ConditionWaitResult),
1156 Group(Vec<ParallelAvroResult>),
1157}
1158
1159impl ParallelAvroResult {
1160 fn into_json_result(self) -> Result<ParallelResult> {
1161 match self {
1162 Self::Activity(value) => Ok(ParallelResult::Activity(value.into_json()?)),
1163 Self::ChildWorkflow(result) => Ok(ParallelResult::ChildWorkflow(ChildWorkflowResult {
1164 parent: result.parent,
1165 child: result.child,
1166 child_workflow_type: result.child_workflow_type,
1167 result: result.result.into_json()?,
1168 })),
1169 Self::Timer => Ok(ParallelResult::Timer),
1170 Self::Signal(values) => Ok(ParallelResult::Signal(
1171 values
1172 .into_iter()
1173 .map(AvroValue::into_json)
1174 .collect::<Result<Vec<_>>>()?,
1175 )),
1176 Self::Condition(result) => Ok(ParallelResult::Condition(result)),
1177 Self::Group(results) => Ok(ParallelResult::Group(
1178 results
1179 .into_iter()
1180 .map(Self::into_json_result)
1181 .collect::<Result<Vec<_>>>()?,
1182 )),
1183 }
1184 }
1185}
1186
1187#[derive(Clone, Debug, PartialEq)]
1189pub struct ParallelCompletion {
1190 pub member_path: Vec<usize>,
1191 pub result: ParallelResult,
1192}
1193
1194#[derive(Debug, Error)]
1202#[error("parallel group {group_id} member {member_path:?} failed: {cause}")]
1203pub struct ParallelFailure {
1204 pub group_id: String,
1205 pub member_path: Vec<usize>,
1206 pub group_path: Vec<ParallelGroupMetadata>,
1207 pub completed: Vec<ParallelCompletion>,
1208 #[source]
1209 pub cause: Box<Error>,
1210}
1211
1212#[derive(Clone, Debug, Error, PartialEq, Eq)]
1214#[error("invalid deterministic parallel group ({reason}): {message}")]
1215pub struct ParallelGroupError {
1216 pub reason: &'static str,
1217 pub member_path: Vec<usize>,
1218 pub message: String,
1219}
1220
1221#[derive(Clone, Debug, Error, PartialEq, Eq)]
1223#[error("workflow cancellation was requested")]
1224pub struct WorkflowCancellationRequested;
1225
1226#[derive(Debug, Error)]
1228#[error(
1229 "saga forward execution failed; compensation activity {compensation_activity_type} (registration {compensation_registration_order}) also failed: {compensation_failure}"
1230)]
1231pub struct SagaCompensationFailure {
1232 pub initiating_failure: Box<Error>,
1233 pub compensation_failure: Box<Error>,
1234 pub compensation_activity_type: String,
1235 pub compensation_registration_order: usize,
1236}
1237
1238#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
1240pub enum ParentClosePolicy {
1241 #[default]
1242 Abandon,
1243 RequestCancel,
1244 Terminate,
1245}
1246
1247impl ParentClosePolicy {
1248 fn as_str(self) -> &'static str {
1249 match self {
1250 Self::Abandon => "abandon",
1251 Self::RequestCancel => "request_cancel",
1252 Self::Terminate => "terminate",
1253 }
1254 }
1255}
1256
1257#[derive(Clone, Debug, Default, PartialEq, Eq)]
1259pub struct ChildWorkflowRetryPolicy {
1260 pub max_attempts: Option<u32>,
1261 pub backoff_seconds: Vec<u64>,
1262 pub non_retryable_error_types: Vec<String>,
1263}
1264
1265#[derive(Clone, Debug, PartialEq, Eq)]
1269pub struct ChildWorkflowOptions {
1270 pub task_queue: String,
1271 pub parent_close_policy: ParentClosePolicy,
1272 pub retry_policy: Option<ChildWorkflowRetryPolicy>,
1273 pub execution_timeout_seconds: Option<u64>,
1274 pub run_timeout_seconds: Option<u64>,
1275}
1276
1277impl ChildWorkflowOptions {
1278 pub fn new(task_queue: impl Into<String>) -> Self {
1279 Self {
1280 task_queue: task_queue.into(),
1281 parent_close_policy: ParentClosePolicy::Abandon,
1282 retry_policy: None,
1283 execution_timeout_seconds: None,
1284 run_timeout_seconds: None,
1285 }
1286 }
1287
1288 pub fn parent_close_policy(mut self, policy: ParentClosePolicy) -> Self {
1289 self.parent_close_policy = policy;
1290 self
1291 }
1292
1293 pub fn retry_policy(mut self, policy: ChildWorkflowRetryPolicy) -> Self {
1294 self.retry_policy = Some(policy);
1295 self
1296 }
1297
1298 pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
1299 self.execution_timeout_seconds = Some(seconds);
1300 self
1301 }
1302
1303 pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
1304 self.run_timeout_seconds = Some(seconds);
1305 self
1306 }
1307}
1308
1309#[derive(Clone, Debug, PartialEq, Eq)]
1311pub enum ActivityBackoff {
1312 Explicit(Vec<Duration>),
1315 Exponential {
1317 initial_interval: Duration,
1318 coefficient: u32,
1319 maximum_interval: Option<Duration>,
1320 },
1321}
1322
1323#[derive(Clone, Debug, Default, PartialEq, Eq)]
1325pub struct ActivityRetryPolicy {
1326 pub max_attempts: Option<u32>,
1327 pub backoff: Option<ActivityBackoff>,
1328 pub non_retryable_error_types: Vec<String>,
1329}
1330
1331impl ActivityRetryPolicy {
1332 pub fn new(max_attempts: u32) -> Self {
1334 Self {
1335 max_attempts: Some(max_attempts),
1336 ..Self::default()
1337 }
1338 }
1339
1340 pub fn backoff_intervals(mut self, intervals: impl IntoIterator<Item = Duration>) -> Self {
1341 self.backoff = Some(ActivityBackoff::Explicit(intervals.into_iter().collect()));
1342 self
1343 }
1344
1345 pub fn exponential_backoff(
1346 mut self,
1347 initial_interval: Duration,
1348 coefficient: u32,
1349 maximum_interval: Option<Duration>,
1350 ) -> Self {
1351 self.backoff = Some(ActivityBackoff::Exponential {
1352 initial_interval,
1353 coefficient,
1354 maximum_interval,
1355 });
1356 self
1357 }
1358
1359 pub fn non_retryable_error_type(mut self, error_type: impl Into<String>) -> Self {
1360 self.non_retryable_error_types.push(error_type.into());
1361 self
1362 }
1363
1364 pub fn non_retryable_error_types(
1365 mut self,
1366 error_types: impl IntoIterator<Item = impl Into<String>>,
1367 ) -> Self {
1368 self.non_retryable_error_types
1369 .extend(error_types.into_iter().map(Into::into));
1370 self
1371 }
1372}
1373
1374#[derive(Clone, Debug, Default, PartialEq, Eq)]
1379pub struct ActivityOptions {
1380 pub task_queue: Option<String>,
1381 pub retry_policy: Option<ActivityRetryPolicy>,
1382 pub start_to_close_timeout: Option<Duration>,
1383 pub schedule_to_start_timeout: Option<Duration>,
1384 pub schedule_to_close_timeout: Option<Duration>,
1385 pub heartbeat_timeout: Option<Duration>,
1386}
1387
1388impl ActivityOptions {
1389 pub fn new() -> Self {
1390 Self::default()
1391 }
1392
1393 pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
1394 self.task_queue = Some(task_queue.into());
1395 self
1396 }
1397
1398 pub fn retry_policy(mut self, policy: ActivityRetryPolicy) -> Self {
1399 self.retry_policy = Some(policy);
1400 self
1401 }
1402
1403 pub fn start_to_close_timeout(mut self, timeout: Duration) -> Self {
1404 self.start_to_close_timeout = Some(timeout);
1405 self
1406 }
1407
1408 pub fn schedule_to_start_timeout(mut self, timeout: Duration) -> Self {
1409 self.schedule_to_start_timeout = Some(timeout);
1410 self
1411 }
1412
1413 pub fn schedule_to_close_timeout(mut self, timeout: Duration) -> Self {
1414 self.schedule_to_close_timeout = Some(timeout);
1415 self
1416 }
1417
1418 pub fn heartbeat_timeout(mut self, timeout: Duration) -> Self {
1419 self.heartbeat_timeout = Some(timeout);
1420 self
1421 }
1422
1423 fn validate(&self) -> std::result::Result<ValidatedActivityOptions, ActivityOptionsError> {
1424 if self
1425 .task_queue
1426 .as_deref()
1427 .is_some_and(|queue| queue.trim().is_empty())
1428 {
1429 return Err(ActivityOptionsError::new(
1430 ActivityOptionsErrorKind::EmptyTaskQueue,
1431 Some("task_queue"),
1432 "task_queue must not be empty",
1433 ));
1434 }
1435
1436 for (field, value) in [
1437 ("start_to_close_timeout", self.start_to_close_timeout),
1438 ("schedule_to_start_timeout", self.schedule_to_start_timeout),
1439 ("schedule_to_close_timeout", self.schedule_to_close_timeout),
1440 ("heartbeat_timeout", self.heartbeat_timeout),
1441 ] {
1442 if value.is_some_and(|value| value.is_zero()) {
1443 return Err(ActivityOptionsError::new(
1444 ActivityOptionsErrorKind::TimeoutNotPositive,
1445 Some(field),
1446 format!("{field} must be positive"),
1447 ));
1448 }
1449 }
1450
1451 validate_timeout_order(
1452 "heartbeat_timeout",
1453 self.heartbeat_timeout,
1454 "start_to_close_timeout",
1455 self.start_to_close_timeout,
1456 )?;
1457 validate_timeout_order(
1458 "start_to_close_timeout",
1459 self.start_to_close_timeout,
1460 "schedule_to_close_timeout",
1461 self.schedule_to_close_timeout,
1462 )?;
1463 validate_timeout_order(
1464 "schedule_to_start_timeout",
1465 self.schedule_to_start_timeout,
1466 "schedule_to_close_timeout",
1467 self.schedule_to_close_timeout,
1468 )?;
1469
1470 Ok(ValidatedActivityOptions {
1471 task_queue: self.task_queue.clone(),
1472 retry_policy: self
1473 .retry_policy
1474 .as_ref()
1475 .map(validate_activity_retry_policy)
1476 .transpose()?,
1477 start_to_close_timeout: timeout_seconds(
1478 "start_to_close_timeout",
1479 self.start_to_close_timeout,
1480 )?,
1481 schedule_to_start_timeout: timeout_seconds(
1482 "schedule_to_start_timeout",
1483 self.schedule_to_start_timeout,
1484 )?,
1485 schedule_to_close_timeout: timeout_seconds(
1486 "schedule_to_close_timeout",
1487 self.schedule_to_close_timeout,
1488 )?,
1489 heartbeat_timeout: timeout_seconds("heartbeat_timeout", self.heartbeat_timeout)?,
1490 })
1491 }
1492}
1493
1494pub enum ParallelOperation {
1500 Activity {
1501 activity_type: String,
1502 options: ActivityOptions,
1503 arguments: Result<AvroValue>,
1504 },
1505 ChildWorkflow {
1506 workflow_type: String,
1507 options: ChildWorkflowOptions,
1508 arguments: Result<AvroValue>,
1509 },
1510 Timer(Duration),
1511 Signal(String),
1512 Condition {
1513 options: ConditionWaitOptions,
1514 predicate: Box<dyn Fn() -> Result<bool> + Send + 'static>,
1515 },
1516 Group(Vec<ParallelOperation>),
1517}
1518
1519impl ParallelOperation {
1520 pub fn activity<T: Serialize>(activity_type: impl Into<String>, args: T) -> Self {
1521 Self::activity_with_options(activity_type, ActivityOptions::new(), args)
1522 }
1523
1524 pub fn activity_with_options<T: Serialize>(
1525 activity_type: impl Into<String>,
1526 options: ActivityOptions,
1527 args: T,
1528 ) -> Self {
1529 Self::Activity {
1530 activity_type: activity_type.into(),
1531 options,
1532 arguments: AvroValue::from_serialize(&args),
1533 }
1534 }
1535
1536 pub fn child_workflow<T: Serialize>(
1537 workflow_type: impl Into<String>,
1538 options: ChildWorkflowOptions,
1539 args: T,
1540 ) -> Self {
1541 Self::ChildWorkflow {
1542 workflow_type: workflow_type.into(),
1543 options,
1544 arguments: AvroValue::from_serialize(&args),
1545 }
1546 }
1547
1548 pub fn timer(duration: Duration) -> Self {
1549 Self::Timer(duration)
1550 }
1551
1552 pub fn signal(signal_name: impl Into<String>) -> Self {
1553 Self::Signal(signal_name.into())
1554 }
1555
1556 pub fn condition<F>(options: ConditionWaitOptions, predicate: F) -> Self
1557 where
1558 F: Fn() -> Result<bool> + Send + 'static,
1559 {
1560 Self::Condition {
1561 options,
1562 predicate: Box::new(predicate),
1563 }
1564 }
1565
1566 pub fn group(operations: Vec<ParallelOperation>) -> Self {
1567 Self::Group(operations)
1568 }
1569}
1570
1571#[derive(Clone, Debug)]
1572struct ValidatedActivityOptions {
1573 task_queue: Option<String>,
1574 retry_policy: Option<Value>,
1575 start_to_close_timeout: Option<u64>,
1576 schedule_to_start_timeout: Option<u64>,
1577 schedule_to_close_timeout: Option<u64>,
1578 heartbeat_timeout: Option<u64>,
1579}
1580
1581fn validate_timeout_order(
1582 smaller_name: &'static str,
1583 smaller: Option<Duration>,
1584 larger_name: &'static str,
1585 larger: Option<Duration>,
1586) -> std::result::Result<(), ActivityOptionsError> {
1587 if matches!((smaller, larger), (Some(smaller), Some(larger)) if smaller > larger) {
1588 return Err(ActivityOptionsError::new(
1589 ActivityOptionsErrorKind::TimeoutOrder,
1590 Some(smaller_name),
1591 format!("{smaller_name} must be <= {larger_name}"),
1592 ));
1593 }
1594 Ok(())
1595}
1596
1597fn timeout_seconds(
1598 field: &'static str,
1599 value: Option<Duration>,
1600) -> std::result::Result<Option<u64>, ActivityOptionsError> {
1601 value
1602 .map(|value| {
1603 activity_protocol_seconds(value).ok_or_else(|| {
1604 ActivityOptionsError::new(
1605 ActivityOptionsErrorKind::TimeoutOverflow,
1606 Some(field),
1607 format!("{field} is too large for the worker protocol"),
1608 )
1609 })
1610 })
1611 .transpose()
1612}
1613
1614fn duration_seconds_ceil(value: Duration) -> Option<u64> {
1615 value
1616 .as_secs()
1617 .checked_add(u64::from(value.subsec_nanos() > 0))
1618}
1619
1620fn activity_protocol_seconds(value: Duration) -> Option<u64> {
1621 duration_seconds_ceil(value).filter(|seconds| *seconds <= i64::MAX as u64)
1622}
1623
1624fn validate_activity_retry_policy(
1625 policy: &ActivityRetryPolicy,
1626) -> std::result::Result<Value, ActivityOptionsError> {
1627 if policy.max_attempts.is_none()
1628 && policy.backoff.is_none()
1629 && policy.non_retryable_error_types.is_empty()
1630 {
1631 return Err(ActivityOptionsError::new(
1632 ActivityOptionsErrorKind::EmptyRetryPolicy,
1633 Some("retry_policy"),
1634 "retry_policy must configure at least one field",
1635 ));
1636 }
1637 if policy.max_attempts == Some(0) {
1638 return Err(ActivityOptionsError::new(
1639 ActivityOptionsErrorKind::InvalidMaxAttempts,
1640 Some("retry_policy.max_attempts"),
1641 "max_attempts must be >= 1",
1642 ));
1643 }
1644 if policy
1645 .non_retryable_error_types
1646 .iter()
1647 .any(|error_type| error_type.trim().is_empty())
1648 {
1649 return Err(ActivityOptionsError::new(
1650 ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
1651 Some("retry_policy.non_retryable_error_types"),
1652 "non_retryable_error_types must not contain empty values",
1653 ));
1654 }
1655
1656 let backoff_seconds = match &policy.backoff {
1657 None => None,
1658 Some(backoff) => {
1659 let max_attempts = policy.max_attempts.ok_or_else(|| {
1660 ActivityOptionsError::new(
1661 ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
1662 Some("retry_policy.backoff"),
1663 "backoff requires max_attempts",
1664 )
1665 })?;
1666 let retry_count = max_attempts.saturating_sub(1) as usize;
1667 let intervals = match backoff {
1668 ActivityBackoff::Explicit(intervals) => {
1669 if intervals.len() > retry_count {
1670 return Err(ActivityOptionsError::new(
1671 ActivityOptionsErrorKind::TooManyBackoffIntervals,
1672 Some("retry_policy.backoff"),
1673 "backoff interval count must not exceed max_attempts - 1",
1674 ));
1675 }
1676 intervals.clone()
1677 }
1678 ActivityBackoff::Exponential {
1679 initial_interval,
1680 coefficient,
1681 maximum_interval,
1682 } => {
1683 if *coefficient < 1 {
1684 return Err(ActivityOptionsError::new(
1685 ActivityOptionsErrorKind::InvalidBackoffCoefficient,
1686 Some("retry_policy.backoff.coefficient"),
1687 "backoff coefficient must be >= 1",
1688 ));
1689 }
1690 if retry_count > 10_000 {
1691 return Err(ActivityOptionsError::new(
1692 ActivityOptionsErrorKind::BackoffGenerationTooLarge,
1693 Some("retry_policy.max_attempts"),
1694 "generated backoff supports at most 10000 retry intervals",
1695 ));
1696 }
1697 let mut current = *initial_interval;
1698 let mut intervals = Vec::with_capacity(retry_count);
1699 for _ in 0..retry_count {
1700 let interval = maximum_interval
1701 .map(|maximum| current.min(maximum))
1702 .unwrap_or(current);
1703 intervals.push(interval);
1704 if maximum_interval.is_some_and(|maximum| interval == maximum) {
1705 break;
1706 }
1707 current = current.checked_mul(*coefficient).ok_or_else(|| {
1708 ActivityOptionsError::new(
1709 ActivityOptionsErrorKind::BackoffOverflow,
1710 Some("retry_policy.backoff"),
1711 "generated backoff interval overflowed",
1712 )
1713 })?;
1714 }
1715 intervals
1716 }
1717 };
1718 Some(
1719 intervals
1720 .into_iter()
1721 .map(|interval| {
1722 activity_protocol_seconds(interval).ok_or_else(|| {
1723 ActivityOptionsError::new(
1724 ActivityOptionsErrorKind::BackoffOverflow,
1725 Some("retry_policy.backoff"),
1726 "backoff interval is too large for the worker protocol",
1727 )
1728 })
1729 })
1730 .collect::<std::result::Result<Vec<_>, _>>()?,
1731 )
1732 }
1733 };
1734
1735 let mut encoded = serde_json::Map::new();
1736 if let Some(max_attempts) = policy.max_attempts {
1737 encoded.insert("max_attempts".to_string(), json!(max_attempts));
1738 }
1739 if let Some(backoff_seconds) = backoff_seconds {
1740 encoded.insert("backoff_seconds".to_string(), json!(backoff_seconds));
1741 }
1742 if !policy.non_retryable_error_types.is_empty() {
1743 let mut canonical_error_types = Vec::new();
1744 for error_type in policy
1745 .non_retryable_error_types
1746 .iter()
1747 .map(|error_type| error_type.trim())
1748 {
1749 if !canonical_error_types.contains(&error_type) {
1750 canonical_error_types.push(error_type);
1751 }
1752 }
1753 encoded.insert(
1754 "non_retryable_error_types".to_string(),
1755 json!(canonical_error_types),
1756 );
1757 }
1758 Ok(Value::Object(encoded))
1759}
1760
1761#[derive(Clone, Debug, Error)]
1764#[error("non-deterministic workflow replay ({reason}) at sequence {sequence:?}: {message}")]
1765pub struct ReplayFailure {
1766 pub reason: String,
1767 pub sequence: Option<u64>,
1768 pub expected: Option<String>,
1769 pub actual: Option<String>,
1770 pub message: String,
1771}
1772
1773impl ReplayFailure {
1774 fn new(
1775 reason: impl Into<String>,
1776 sequence: Option<u64>,
1777 expected: Option<String>,
1778 actual: Option<String>,
1779 message: impl Into<String>,
1780 ) -> Self {
1781 Self {
1782 reason: reason.into(),
1783 sequence,
1784 expected,
1785 actual,
1786 message: message.into(),
1787 }
1788 }
1789}
1790
1791#[derive(Clone, Debug, Error)]
1793#[error("query failed ({reason}, HTTP {status}): {message}")]
1794pub struct QueryFailure {
1795 pub status: u16,
1796 pub reason: String,
1797 pub message: String,
1798 pub body: Value,
1799}
1800
1801#[derive(Clone, Debug, Error)]
1803#[error("protocol rejected ({reason}, HTTP {status}): {message}")]
1804pub struct ProtocolFailure {
1805 pub status: u16,
1806 pub reason: String,
1807 pub message: String,
1808 pub supported_version: Option<String>,
1809 pub requested_version: Option<String>,
1810 pub body: Value,
1811}
1812
1813#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
1814pub struct PayloadEnvelope {
1815 pub codec: String,
1816 pub blob: String,
1817}
1818
1819impl PayloadEnvelope {
1820 pub fn avro<T: Serialize>(value: &T) -> Result<Self> {
1821 encode_payload(value, DEFAULT_CODEC)
1822 }
1823
1824 pub fn avro_value(value: &AvroValue) -> Result<Self> {
1827 encode_avro_value(value)
1828 }
1829}
1830
1831#[derive(Clone, Debug)]
1833pub enum AvroValue {
1834 Null,
1835 Boolean(bool),
1836 Long(i64),
1837 Double(f64),
1838 Bytes(Vec<u8>),
1839 String(String),
1840 Array(Vec<AvroValue>),
1841 Map(BTreeMap<String, AvroValue>),
1842}
1843
1844impl PartialEq for AvroValue {
1845 fn eq(&self, other: &Self) -> bool {
1846 match (self, other) {
1847 (Self::Null, Self::Null) => true,
1848 (Self::Boolean(left), Self::Boolean(right)) => left == right,
1849 (Self::Long(left), Self::Long(right)) => left == right,
1850 (Self::Double(left), Self::Double(right)) => left.to_bits() == right.to_bits(),
1851 (Self::Bytes(left), Self::Bytes(right)) => left == right,
1852 (Self::String(left), Self::String(right)) => left == right,
1853 (Self::Array(left), Self::Array(right)) => left == right,
1854 (Self::Map(left), Self::Map(right)) => left == right,
1855 _ => false,
1856 }
1857 }
1858}
1859
1860impl AvroValue {
1861 fn from_serialize<T: Serialize>(value: &T) -> Result<Self> {
1862 Self::from_serde_value(
1863 serde_value::to_value(value).map_err(|error| {
1864 Error::Codec(format!("could not adapt value for Avro: {error}"))
1865 })?,
1866 )
1867 }
1868
1869 fn from_serde_value(value: serde_value::Value) -> Result<Self> {
1870 use serde_value::Value as SerdeValue;
1871
1872 match value {
1873 SerdeValue::Unit => Ok(Self::Null),
1874 SerdeValue::Bool(value) => Ok(Self::Boolean(value)),
1875 SerdeValue::I8(value) => Ok(Self::Long(i64::from(value))),
1876 SerdeValue::I16(value) => Ok(Self::Long(i64::from(value))),
1877 SerdeValue::I32(value) => Ok(Self::Long(i64::from(value))),
1878 SerdeValue::I64(value) => Ok(Self::Long(value)),
1879 SerdeValue::U8(value) => Ok(Self::Long(i64::from(value))),
1880 SerdeValue::U16(value) => Ok(Self::Long(i64::from(value))),
1881 SerdeValue::U32(value) => Ok(Self::Long(i64::from(value))),
1882 SerdeValue::U64(value) => i64::try_from(value).map(Self::Long).map_err(|_| {
1883 Error::Codec(
1884 "integer_overflow: Avro Value long must be within signed 64-bit range"
1885 .to_string(),
1886 )
1887 }),
1888 SerdeValue::F32(value) => Self::finite_double(f64::from(value)),
1889 SerdeValue::F64(value) => Self::finite_double(value),
1890 SerdeValue::Char(value) => Ok(Self::String(value.to_string())),
1891 SerdeValue::String(value) => Ok(Self::String(value)),
1892 SerdeValue::Bytes(value) => Ok(Self::Bytes(value)),
1893 SerdeValue::Option(None) => Ok(Self::Null),
1894 SerdeValue::Option(Some(value)) | SerdeValue::Newtype(value) => {
1895 Self::from_serde_value(*value)
1896 }
1897 SerdeValue::Seq(values) => values
1898 .into_iter()
1899 .map(Self::from_serde_value)
1900 .collect::<Result<Vec<_>>>()
1901 .map(Self::Array),
1902 SerdeValue::Map(values) => values
1903 .into_iter()
1904 .map(|(key, value)| {
1905 let SerdeValue::String(key) = key else {
1906 return Err(Error::Codec(
1907 "invalid_map_key: Avro Value map keys must be strings".to_string(),
1908 ));
1909 };
1910
1911 Ok((key, Self::from_serde_value(value)?))
1912 })
1913 .collect::<Result<BTreeMap<_, _>>>()
1914 .map(Self::Map),
1915 }
1916 }
1917
1918 fn finite_double(value: f64) -> Result<Self> {
1919 if !value.is_finite() {
1920 return Err(Error::Codec(
1921 "non_finite_float: Avro Value doubles must be finite".to_string(),
1922 ));
1923 }
1924
1925 Ok(Self::Double(value))
1926 }
1927
1928 fn into_json(self) -> Result<Value> {
1929 match self {
1930 Self::Null => Ok(Value::Null),
1931 Self::Boolean(value) => Ok(Value::Bool(value)),
1932 Self::Long(value) => Ok(Value::Number(value.into())),
1933 Self::Double(value) => serde_json::Number::from_f64(value)
1934 .map(Value::Number)
1935 .ok_or_else(|| {
1936 Error::Codec(
1937 "non_finite_float: decoded Avro Value double is not finite".to_string(),
1938 )
1939 }),
1940 Self::Bytes(value) => Ok(json!({
1941 "$type": "bytes",
1942 "base64": BASE64.encode(value),
1943 })),
1944 Self::String(value) => Ok(Value::String(value)),
1945 Self::Array(values) => values
1946 .into_iter()
1947 .map(Self::into_json)
1948 .collect::<Result<Vec<_>>>()
1949 .map(Value::Array),
1950 Self::Map(values) => values
1951 .into_iter()
1952 .map(|(key, value)| Ok((key, value.into_json()?)))
1953 .collect::<Result<serde_json::Map<_, _>>>()
1954 .map(Value::Object),
1955 }
1956 }
1957
1958 fn into_serde_value(self) -> serde_value::Value {
1959 use serde_value::Value as SerdeValue;
1960
1961 match self {
1962 Self::Null => SerdeValue::Unit,
1963 Self::Boolean(value) => SerdeValue::Bool(value),
1964 Self::Long(value) => SerdeValue::I64(value),
1965 Self::Double(value) => SerdeValue::F64(value),
1966 Self::Bytes(value) => SerdeValue::Bytes(value),
1967 Self::String(value) => SerdeValue::String(value),
1968 Self::Array(values) => {
1969 SerdeValue::Seq(values.into_iter().map(Self::into_serde_value).collect())
1970 }
1971 Self::Map(values) => SerdeValue::Map(
1972 values
1973 .into_iter()
1974 .map(|(key, value)| (SerdeValue::String(key), value.into_serde_value()))
1975 .collect(),
1976 ),
1977 }
1978 }
1979
1980 pub fn deserialize<T: DeserializeOwned>(self) -> Result<T> {
1981 self.into_serde_value().deserialize_into().map_err(|error| {
1982 Error::Codec(format!(
1983 "avro_value_type_mismatch: could not adapt decoded value: {error}"
1984 ))
1985 })
1986 }
1987}
1988
1989impl Serialize for AvroValue {
1990 fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
1991 where
1992 S: Serializer,
1993 {
1994 match self {
1995 Self::Null => serializer.serialize_unit(),
1996 Self::Boolean(value) => serializer.serialize_bool(*value),
1997 Self::Long(value) => serializer.serialize_i64(*value),
1998 Self::Double(value) => serializer.serialize_f64(*value),
1999 Self::Bytes(value) => serializer.serialize_bytes(value),
2000 Self::String(value) => serializer.serialize_str(value),
2001 Self::Array(values) => {
2002 let mut sequence = serializer.serialize_seq(Some(values.len()))?;
2003 for value in values {
2004 sequence.serialize_element(value)?;
2005 }
2006 sequence.end()
2007 }
2008 Self::Map(values) => {
2009 let mut map = serializer.serialize_map(Some(values.len()))?;
2010 for (key, value) in values {
2011 map.serialize_entry(key, value)?;
2012 }
2013 map.end()
2014 }
2015 }
2016 }
2017}
2018
2019pub fn encode_avro_value(value: &AvroValue) -> Result<PayloadEnvelope> {
2020 let datum = avro_value_to_datum(value)?;
2021 let datum = to_avro_datum(avro_value_ordered_map_encoding_schema()?, datum)
2022 .map_err(|err| Error::Codec(format!("avro_value_encode_failed: {err}")))?;
2023 let mut bytes = Vec::with_capacity(datum.len() + 10);
2024 bytes.extend_from_slice(&AVRO_SINGLE_OBJECT_MAGIC);
2025 bytes.extend_from_slice(&AVRO_VALUE_SCHEMA_FINGERPRINT);
2026 bytes.extend_from_slice(&datum);
2027 Ok(PayloadEnvelope {
2028 codec: DEFAULT_CODEC.to_string(),
2029 blob: BASE64.encode(bytes),
2030 })
2031}
2032
2033pub fn decode_avro_value(envelope: &PayloadEnvelope) -> Result<AvroValue> {
2034 if envelope.codec != DEFAULT_CODEC {
2035 return Err(unsupported_payload_codec(&envelope.codec));
2036 }
2037 decode_avro_value_blob(&envelope.blob)
2038}
2039
2040pub fn encode_payload<T: Serialize>(value: &T, codec: &str) -> Result<PayloadEnvelope> {
2041 let blob = match codec {
2042 DEFAULT_CODEC => encode_avro_value(&AvroValue::from_serialize(value)?)?.blob,
2043 other => return Err(unsupported_payload_codec(other)),
2044 };
2045
2046 Ok(PayloadEnvelope {
2047 codec: codec.to_string(),
2048 blob,
2049 })
2050}
2051
2052pub fn decode_payload<T: DeserializeOwned>(envelope: &PayloadEnvelope) -> Result<T> {
2053 match envelope.codec.as_str() {
2054 DEFAULT_CODEC => decode_avro_value(envelope)?.deserialize(),
2055 other => Err(unsupported_payload_codec(other)),
2056 }
2057}
2058
2059fn handler_type_error<T>(
2060 handler_kind: HandlerKind,
2061 handler_name: &str,
2062 value_kind: HandlerValueKind,
2063 message: impl Into<String>,
2064) -> Error {
2065 Error::HandlerType {
2066 handler_kind,
2067 handler_name: handler_name.to_string(),
2068 value_kind,
2069 rust_type: type_name::<T>(),
2070 message: message.into(),
2071 }
2072}
2073
2074fn decode_handler_input<T: DeserializeOwned>(
2075 arguments: AvroValue,
2076 handler_kind: HandlerKind,
2077 handler_name: &str,
2078) -> Result<T> {
2079 let argument = match arguments {
2080 AvroValue::Array(mut arguments) if arguments.len() == 1 => {
2081 arguments.pop().expect("one typed handler argument")
2082 }
2083 AvroValue::Array(arguments) if arguments.is_empty() => AvroValue::Null,
2084 AvroValue::Array(arguments) => {
2085 return Err(handler_type_error::<T>(
2086 handler_kind,
2087 handler_name,
2088 HandlerValueKind::Input,
2089 format!(
2090 "typed handlers accept one request value, but the task carried {} arguments",
2091 arguments.len()
2092 ),
2093 ));
2094 }
2095 argument => argument,
2096 };
2097
2098 argument.deserialize().map_err(|error| {
2099 handler_type_error::<T>(
2100 handler_kind,
2101 handler_name,
2102 HandlerValueKind::Input,
2103 error.to_string(),
2104 )
2105 })
2106}
2107
2108fn encode_handler_result<T: Serialize>(
2109 result: &T,
2110 handler_kind: HandlerKind,
2111 handler_name: &str,
2112) -> Result<AvroValue> {
2113 AvroValue::from_serialize(result).map_err(|error| {
2114 handler_type_error::<T>(
2115 handler_kind,
2116 handler_name,
2117 HandlerValueKind::Result,
2118 error.to_string(),
2119 )
2120 })
2121}
2122
2123fn decode_handler_result<T: DeserializeOwned>(
2124 result: AvroValue,
2125 handler_kind: HandlerKind,
2126 handler_name: &str,
2127) -> Result<T> {
2128 result.deserialize().map_err(|error| {
2129 handler_type_error::<T>(
2130 handler_kind,
2131 handler_name,
2132 HandlerValueKind::Result,
2133 error.to_string(),
2134 )
2135 })
2136}
2137
2138#[cfg(test)]
2139fn encode_value_envelope(value: &Value, codec: &str) -> Result<Value> {
2140 Ok(serde_json::to_value(encode_payload(value, codec)?)?)
2141}
2142
2143fn decode_wire_value(value: &Value, fallback_codec: &str) -> Result<Value> {
2144 validate_payload_codec(fallback_codec)?;
2145
2146 if value.is_null() {
2147 return Ok(Value::Null);
2148 }
2149
2150 if let Some((codec, blob)) = payload_envelope_parts(value)? {
2151 return decode_blob(blob, codec);
2152 }
2153
2154 if let Some(blob) = value.as_str() {
2155 return decode_blob(blob, fallback_codec);
2156 }
2157
2158 Err(untagged_payload_value())
2159}
2160
2161fn encode_typed_envelope(value: &AvroValue, codec: &str) -> Result<Value> {
2162 let envelope = match codec {
2163 DEFAULT_CODEC => encode_avro_value(value)?,
2164 other => return Err(unsupported_payload_codec(other)),
2165 };
2166 Ok(serde_json::to_value(envelope)?)
2167}
2168
2169fn decode_wire_avro_value(value: &Value, fallback_codec: &str) -> Result<AvroValue> {
2170 validate_payload_codec(fallback_codec)?;
2171
2172 if value.is_null() {
2173 return Ok(AvroValue::Null);
2174 }
2175
2176 if let Some((codec, blob)) = payload_envelope_parts(value)? {
2177 validate_payload_codec(codec)?;
2178 return decode_avro_value_blob(blob);
2179 }
2180
2181 if let Some(blob) = value.as_str() {
2182 return match fallback_codec {
2183 DEFAULT_CODEC => decode_avro_value_blob(blob),
2184 other => Err(unsupported_payload_codec(other)),
2185 };
2186 }
2187
2188 Err(untagged_payload_value())
2189}
2190
2191fn normalize_avro_arguments(value: AvroValue) -> AvroValue {
2192 match value {
2193 AvroValue::Null => AvroValue::Array(Vec::new()),
2194 AvroValue::Array(_) => value,
2195 other => AvroValue::Array(vec![other]),
2196 }
2197}
2198
2199fn decode_blob(blob: &str, codec: &str) -> Result<Value> {
2200 match codec {
2201 DEFAULT_CODEC => decode_avro_value_blob(blob)?.into_json(),
2202 other => Err(unsupported_payload_codec(other)),
2203 }
2204}
2205
2206fn validate_payload_codec(codec: &str) -> Result<()> {
2207 match codec {
2208 DEFAULT_CODEC => Ok(()),
2209 MISSING_TASK_PAYLOAD_CODEC => {
2210 Err(invalid_task_payload_codec("task payload_codec is missing"))
2211 }
2212 NULL_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec("task payload_codec is null")),
2213 NON_STRING_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec(
2214 "task payload_codec must be a string",
2215 )),
2216 other => Err(unsupported_payload_codec(other)),
2217 }
2218}
2219
2220fn invalid_task_payload_codec(reason: &str) -> Error {
2221 Error::Codec(format!(
2222 "unsupported_payload_codec: {reason}; Durable Workflow 2.0 requires an explicit string payload_codec=\"avro\" before worker task execution"
2223 ))
2224}
2225
2226fn payload_envelope_parts(value: &Value) -> Result<Option<(&str, &str)>> {
2227 let Some(object) = value.as_object() else {
2228 return Ok(None);
2229 };
2230 if !object.contains_key("codec") && !object.contains_key("blob") {
2231 return Ok(None);
2232 }
2233
2234 let codec = object
2235 .get("codec")
2236 .and_then(Value::as_str)
2237 .ok_or_else(invalid_payload_envelope)?;
2238 validate_payload_codec(codec)?;
2239 let blob = object
2240 .get("blob")
2241 .and_then(Value::as_str)
2242 .ok_or_else(invalid_payload_envelope)?;
2243 Ok(Some((codec, blob)))
2244}
2245
2246fn invalid_payload_envelope() -> Error {
2247 Error::Codec(
2248 "invalid_payload_envelope: durable payloads must use an object with string codec=\"avro\" and blob fields"
2249 .to_string(),
2250 )
2251}
2252
2253fn validate_workflow_task_commands(commands: &[Value]) -> Result<()> {
2254 for command in commands {
2255 let Some(command) = command.as_object() else {
2256 continue;
2257 };
2258 let Some(command_type) = command.get("type").and_then(Value::as_str) else {
2259 continue;
2260 };
2261 let Some(payload_field) = workflow_command_payload_field(command_type) else {
2262 continue;
2263 };
2264
2265 if let Some(codec) = command.get("payload_codec") {
2266 let codec = codec.as_str().ok_or_else(invalid_payload_envelope)?;
2267 validate_payload_codec(codec)?;
2268 }
2269
2270 let payload = command
2271 .get(payload_field)
2272 .ok_or_else(invalid_payload_envelope)?;
2273 validate_outbound_payload_envelope(payload)?;
2274 }
2275 Ok(())
2276}
2277
2278fn workflow_completion_protocol_version(commands: &[Value]) -> &'static str {
2279 if commands.iter().any(|command| {
2280 command.get("type").and_then(Value::as_str) == Some("open_condition_wait")
2281 && command
2282 .get("condition_wait_occurrence_id")
2283 .and_then(Value::as_str)
2284 .is_some_and(|occurrence_id| !occurrence_id.is_empty())
2285 }) {
2286 CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
2287 } else if commands.iter().any(|command| {
2288 command.get("type").and_then(Value::as_str) == Some("upsert_search_attributes")
2289 && command.get("attribute_types").is_some()
2290 }) {
2291 TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION
2292 } else if commands
2293 .iter()
2294 .any(|command| command.get("type").and_then(Value::as_str) == Some("upsert_memo"))
2295 {
2296 MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION
2297 } else if commands
2298 .iter()
2299 .any(|command| command.get("type").and_then(Value::as_str) == Some("open_condition_wait"))
2300 {
2301 CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION
2302 } else if commands.iter().any(|command| {
2303 command.get("type").and_then(Value::as_str) == Some("upsert_search_attributes")
2304 }) {
2305 SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION
2306 } else {
2307 WORKER_PROTOCOL_VERSION
2308 }
2309}
2310
2311fn workflow_completion_protocol_version_with_message_streams(
2312 commands: &[Value],
2313 has_message_stream_metadata: bool,
2314) -> &'static str {
2315 let command_protocol = workflow_completion_protocol_version(commands);
2316 if has_message_stream_metadata && !worker_protocol_supports_message_streams(command_protocol) {
2317 MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION
2318 } else {
2319 command_protocol
2320 }
2321}
2322
2323fn workflow_command_payload_field(command_type: &str) -> Option<&'static str> {
2324 match command_type {
2325 "complete_workflow" | "complete_update" | "record_side_effect" => Some("result"),
2326 "schedule_activity" | "start_child_workflow" | "continue_as_new" => Some("arguments"),
2327 "start_service_operation" => Some("request_payload"),
2328 "upsert_memo" => Some("entries"),
2329 _ => None,
2330 }
2331}
2332
2333fn validate_outbound_payload_envelope(value: &Value) -> Result<()> {
2334 let Some((codec, blob)) = payload_envelope_parts(value)? else {
2335 return Err(untagged_payload_value());
2336 };
2337 validate_payload_codec(codec)?;
2338 decode_avro_value_blob(blob)?;
2339 Ok(())
2340}
2341
2342fn unsupported_payload_codec(codec: &str) -> Error {
2343 Error::Codec(format!(
2344 "unsupported_payload_codec: workflow payload codec {codec:?} is not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
2345 ))
2346}
2347
2348fn untagged_payload_value() -> Error {
2349 Error::Codec(
2350 "unsupported_payload_codec: untagged durable payload values are not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
2351 .to_string(),
2352 )
2353}
2354
2355fn decode_avro_value_blob(blob: &str) -> Result<AvroValue> {
2356 let bytes = BASE64.decode(blob).map_err(|err| {
2357 Error::Codec(format!(
2358 "invalid_payload_framing: expected strict base64 Avro single-object bytes: {err}"
2359 ))
2360 })?;
2361
2362 if serde_json::from_slice::<Value>(&bytes).is_ok() {
2363 return Err(unsupported_payload_codec("json"));
2364 }
2365
2366 if bytes.len() < 10 || bytes[..2] != AVRO_SINGLE_OBJECT_MAGIC {
2367 return Err(Error::Codec(
2368 "invalid_payload_framing: expected Avro single-object magic c301".to_string(),
2369 ));
2370 }
2371
2372 let fingerprint: [u8; 8] = bytes[2..10]
2373 .try_into()
2374 .map_err(|_| Error::Codec("invalid Avro fingerprint length".to_string()))?;
2375 if fingerprint != AVRO_VALUE_SCHEMA_FINGERPRINT {
2376 return Err(Error::Codec(format!(
2377 "unsupported_payload_schema: unknown CRC-64-AVRO fingerprint {}",
2378 fingerprint
2379 .iter()
2380 .map(|byte| format!("{byte:02x}"))
2381 .collect::<String>()
2382 )));
2383 }
2384
2385 let mut datum_reader = StrictAvroDatumReader::new(&bytes[10..]);
2386 let datum = from_avro_datum(avro_value_schema()?, &mut datum_reader, None);
2390 if datum_reader.truncated {
2391 return Err(Error::Codec(
2392 "invalid_payload_framing: truncated Avro Value datum".to_string(),
2393 ));
2394 }
2395 let datum = datum.map_err(|err| {
2396 Error::Codec(format!(
2397 "invalid_payload_framing: malformed Avro Value datum: {err}"
2398 ))
2399 })?;
2400 if datum_reader.remaining() != 0 {
2401 return Err(Error::Codec(format!(
2402 "invalid_payload_framing: {} trailing bytes after Avro Value datum",
2403 datum_reader.remaining()
2404 )));
2405 }
2406 avro_value_from_datum(datum)
2407}
2408
2409struct StrictAvroDatumReader<'a> {
2410 bytes: &'a [u8],
2411 offset: usize,
2412 truncated: bool,
2413}
2414
2415impl<'a> StrictAvroDatumReader<'a> {
2416 fn new(bytes: &'a [u8]) -> Self {
2417 Self {
2418 bytes,
2419 offset: 0,
2420 truncated: false,
2421 }
2422 }
2423
2424 fn remaining(&self) -> usize {
2425 self.bytes.len() - self.offset
2426 }
2427}
2428
2429impl Read for StrictAvroDatumReader<'_> {
2430 fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
2431 let count = buffer.len().min(self.remaining());
2432 buffer[..count].copy_from_slice(&self.bytes[self.offset..self.offset + count]);
2433 self.offset += count;
2434 if count < buffer.len() {
2435 self.truncated = true;
2436 }
2437
2438 Ok(count)
2439 }
2440}
2441
2442fn avro_value_to_datum(value: &AvroValue) -> Result<AvroDatum> {
2443 let branch = match value {
2444 AvroValue::Null => AvroDatum::Union(0, Box::new(AvroDatum::Null)),
2445 AvroValue::Boolean(value) => AvroDatum::Union(
2446 1,
2447 Box::new(AvroDatum::Record(vec![(
2448 "boolean".to_string(),
2449 AvroDatum::Boolean(*value),
2450 )])),
2451 ),
2452 AvroValue::Long(value) => AvroDatum::Union(
2453 2,
2454 Box::new(AvroDatum::Record(vec![(
2455 "long".to_string(),
2456 AvroDatum::Long(*value),
2457 )])),
2458 ),
2459 AvroValue::Double(value) => {
2460 if !value.is_finite() {
2461 return Err(Error::Codec(
2462 "non_finite_float: Avro Value doubles must be finite".to_string(),
2463 ));
2464 }
2465 AvroDatum::Union(
2466 3,
2467 Box::new(AvroDatum::Record(vec![(
2468 "double".to_string(),
2469 AvroDatum::Double(*value),
2470 )])),
2471 )
2472 }
2473 AvroValue::Bytes(value) => AvroDatum::Union(
2474 4,
2475 Box::new(AvroDatum::Record(vec![(
2476 "bytes".to_string(),
2477 AvroDatum::Bytes(value.clone()),
2478 )])),
2479 ),
2480 AvroValue::String(value) => AvroDatum::Union(
2481 5,
2482 Box::new(AvroDatum::Record(vec![(
2483 "string".to_string(),
2484 AvroDatum::String(value.clone()),
2485 )])),
2486 ),
2487 AvroValue::Array(values) => AvroDatum::Union(
2488 6,
2489 Box::new(AvroDatum::Record(vec![(
2490 "items".to_string(),
2491 AvroDatum::Array(
2492 values
2493 .iter()
2494 .map(avro_value_to_datum)
2495 .collect::<Result<Vec<_>>>()?,
2496 ),
2497 )])),
2498 ),
2499 AvroValue::Map(values) => AvroDatum::Union(
2500 7,
2501 Box::new(AvroDatum::Record(vec![(
2502 "entries".to_string(),
2503 AvroDatum::Array(
2504 values
2505 .iter()
2506 .map(|(key, value)| {
2507 Ok(AvroDatum::Record(vec![
2508 ("key".to_string(), AvroDatum::String(key.clone())),
2509 ("value".to_string(), avro_value_to_datum(value)?),
2510 ]))
2511 })
2512 .collect::<Result<Vec<_>>>()?,
2513 ),
2514 )])),
2515 ),
2516 };
2517 Ok(AvroDatum::Record(vec![("value".to_string(), branch)]))
2518}
2519
2520fn avro_value_from_datum(datum: AvroDatum) -> Result<AvroValue> {
2521 let AvroDatum::Record(mut outer) = datum else {
2522 return Err(Error::Codec(
2523 "invalid_payload_framing: datum is not a Value record".to_string(),
2524 ));
2525 };
2526 let (_, branch) = outer
2527 .pop()
2528 .filter(|(name, _)| name == "value")
2529 .ok_or_else(|| Error::Codec("invalid_payload_framing: Value field missing".to_string()))?;
2530 let AvroDatum::Union(_, branch) = branch else {
2531 return Err(Error::Codec(
2532 "invalid_payload_framing: invalid Value union".to_string(),
2533 ));
2534 };
2535 match *branch {
2536 AvroDatum::Null => Ok(AvroValue::Null),
2537 AvroDatum::Record(mut fields) => {
2538 let (name, value) = fields.pop().ok_or_else(|| {
2539 Error::Codec("invalid_payload_framing: empty Value branch".to_string())
2540 })?;
2541 match (name.as_str(), value) {
2542 ("boolean", AvroDatum::Boolean(value)) => Ok(AvroValue::Boolean(value)),
2543 ("long", AvroDatum::Long(value)) => Ok(AvroValue::Long(value)),
2544 ("double", AvroDatum::Double(value)) if value.is_finite() => {
2545 Ok(AvroValue::Double(value))
2546 }
2547 ("bytes", AvroDatum::Bytes(value)) => Ok(AvroValue::Bytes(value)),
2548 ("string", AvroDatum::String(value)) => Ok(AvroValue::String(value)),
2549 ("items", AvroDatum::Array(values)) => values
2550 .into_iter()
2551 .map(avro_value_from_datum)
2552 .collect::<Result<Vec<_>>>()
2553 .map(AvroValue::Array),
2554 ("entries", AvroDatum::Map(values)) => values
2555 .into_iter()
2556 .map(|(key, value)| Ok((key, avro_value_from_datum(value)?)))
2557 .collect::<Result<BTreeMap<_, _>>>()
2558 .map(AvroValue::Map),
2559 _ => Err(Error::Codec(
2560 "invalid_payload_framing: unknown Value branch".to_string(),
2561 )),
2562 }
2563 }
2564 _ => Err(Error::Codec(
2565 "invalid_payload_framing: invalid Value branch".to_string(),
2566 )),
2567 }
2568}
2569
2570fn avro_value_schema() -> Result<&'static Schema> {
2571 match AVRO_VALUE_SCHEMA.get_or_init(|| {
2572 Schema::parse_str(AVRO_VALUE_SCHEMA_JSON)
2573 .map_err(|err| format!("could not parse Avro Value schema: {err}"))
2574 }) {
2575 Ok(schema) => Ok(schema),
2576 Err(message) => Err(Error::Codec(message.clone())),
2577 }
2578}
2579
2580fn avro_value_ordered_map_encoding_schema() -> Result<&'static Schema> {
2581 match AVRO_VALUE_ORDERED_MAP_ENCODING_SCHEMA.get_or_init(|| {
2582 let mut schema: Value = serde_json::from_str(AVRO_VALUE_SCHEMA_JSON)
2587 .map_err(|err| format!("could not read packaged Avro Value schema: {err}"))?;
2588 let entries_schema = schema
2589 .pointer_mut("/fields/0/type/7/fields/0/type")
2590 .ok_or_else(|| "packaged Avro Value map schema is missing".to_string())?;
2591 if *entries_schema != json!({"type": "map", "values": "Value"}) {
2592 return Err("packaged Avro Value map schema changed unexpectedly".to_string());
2593 }
2594 *entries_schema = json!({
2595 "type": "array",
2596 "items": {
2597 "type": "record",
2598 "name": "MapEntry",
2599 "fields": [
2600 {"name": "key", "type": "string"},
2601 {"name": "value", "type": "Value"}
2602 ]
2603 }
2604 });
2605 Schema::parse_str(&schema.to_string())
2606 .map_err(|err| format!("could not parse ordered-map Avro Value schema: {err}"))
2607 }) {
2608 Ok(schema) => Ok(schema),
2609 Err(message) => Err(Error::Codec(message.clone())),
2610 }
2611}
2612
2613#[derive(Clone, Debug)]
2614pub struct Client {
2615 http: reqwest::Client,
2616 base_url: String,
2617 token: Option<String>,
2618 control_token: Option<String>,
2619 worker_token: Option<String>,
2620 namespace: String,
2621 worker_storage_admission: Option<WorkerStorageAdmission>,
2622}
2623
2624impl Client {
2625 pub fn new(base_url: impl Into<String>) -> Result<Self> {
2626 Self::builder(base_url).build()
2627 }
2628
2629 pub fn builder(base_url: impl Into<String>) -> ClientBuilder {
2630 ClientBuilder {
2631 base_url: base_url.into(),
2632 token: None,
2633 control_token: None,
2634 worker_token: None,
2635 namespace: "default".to_string(),
2636 timeout: Duration::from_secs(60),
2637 }
2638 }
2639
2640 pub async fn health(&self) -> Result<Value> {
2641 self.request_json(
2642 reqwest::Method::GET,
2643 "/health",
2644 RequestProtocol::ControlPlane,
2645 Option::<&Value>::None,
2646 )
2647 .await
2648 }
2649
2650 pub async fn cluster_info(&self) -> Result<Value> {
2651 self.request_json(
2652 reqwest::Method::GET,
2653 "/cluster/info",
2654 RequestProtocol::ControlPlane,
2655 Option::<&Value>::None,
2656 )
2657 .await
2658 }
2659
2660 pub async fn start_workflow<T: Serialize>(
2661 &self,
2662 workflow_type: &str,
2663 task_queue: &str,
2664 workflow_id: &str,
2665 input: T,
2666 ) -> Result<WorkflowHandle> {
2667 self.start_workflow_with_options(
2668 workflow_type,
2669 task_queue,
2670 workflow_id,
2671 WorkflowStartOptions::default(),
2672 input,
2673 )
2674 .await
2675 }
2676
2677 pub async fn start_workflow_with_options<T: Serialize>(
2680 &self,
2681 workflow_type: &str,
2682 task_queue: &str,
2683 workflow_id: &str,
2684 options: WorkflowStartOptions,
2685 input: T,
2686 ) -> Result<WorkflowHandle> {
2687 options.validate()?;
2688 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2689 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2690 let body = json!({
2691 "workflow_id": workflow_id,
2692 "workflow_type": workflow_type,
2693 "task_queue": task_queue,
2694 "input": input_envelope,
2695 "execution_timeout_seconds": options.execution_timeout_seconds,
2696 "run_timeout_seconds": options.run_timeout_seconds
2697 });
2698
2699 let data: Value = self
2700 .request_json(
2701 reqwest::Method::POST,
2702 "/workflows",
2703 RequestProtocol::ControlPlane,
2704 Some(&body),
2705 )
2706 .await?;
2707
2708 Ok(WorkflowHandle {
2709 client: self.clone(),
2710 workflow_id: data
2711 .get("workflow_id")
2712 .and_then(Value::as_str)
2713 .unwrap_or(workflow_id)
2714 .to_string(),
2715 run_id: data
2716 .get("run_id")
2717 .and_then(Value::as_str)
2718 .map(str::to_string),
2719 workflow_type: data
2720 .get("workflow_type")
2721 .and_then(Value::as_str)
2722 .unwrap_or(workflow_type)
2723 .to_string(),
2724 })
2725 }
2726
2727 pub async fn signal_workflow<T: Serialize>(
2728 &self,
2729 workflow_id: &str,
2730 signal_name: &str,
2731 input: T,
2732 ) -> Result<Value> {
2733 self.signal_workflow_target(workflow_id, None, signal_name, input)
2734 .await
2735 }
2736
2737 pub async fn append_message_stream<T: Serialize>(
2739 &self,
2740 workflow_id: &str,
2741 stream_name: &str,
2742 message_id: &str,
2743 input: T,
2744 ) -> Result<Value> {
2745 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2746 let body = json!({
2747 "message_id": message_id,
2748 "input": encode_typed_envelope(&input, DEFAULT_CODEC)?
2749 });
2750 self.request_json(
2751 reqwest::Method::POST,
2752 &format!("/workflows/{workflow_id}/message-streams/{stream_name}/messages"),
2753 RequestProtocol::ControlPlane,
2754 Some(&body),
2755 )
2756 .await
2757 }
2758
2759 pub async fn signal_workflow_run<T: Serialize>(
2761 &self,
2762 workflow_id: &str,
2763 run_id: &str,
2764 signal_name: &str,
2765 input: T,
2766 ) -> Result<Value> {
2767 self.signal_workflow_target(workflow_id, Some(run_id), signal_name, input)
2768 .await
2769 }
2770
2771 async fn signal_workflow_target<T: Serialize>(
2772 &self,
2773 workflow_id: &str,
2774 run_id: Option<&str>,
2775 signal_name: &str,
2776 input: T,
2777 ) -> Result<Value> {
2778 validate_user_signal_name(signal_name)?;
2779 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2780 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2781 let body = json!({
2782 "input": input_envelope
2783 });
2784 let path = match run_id {
2785 Some(run_id) => {
2786 format!("/workflows/{workflow_id}/runs/{run_id}/signal/{signal_name}")
2787 }
2788 None => format!("/workflows/{workflow_id}/signal/{signal_name}"),
2789 };
2790 self.request_json(
2791 reqwest::Method::POST,
2792 &path,
2793 RequestProtocol::ControlPlane,
2794 Some(&body),
2795 )
2796 .await
2797 }
2798
2799 pub async fn cancel_workflow(
2801 &self,
2802 workflow_id: &str,
2803 options: WorkflowCommandOptions,
2804 ) -> Result<WorkflowCommandResult> {
2805 self.workflow_command(workflow_id, None, WorkflowCommandKind::Cancel, options)
2806 .await
2807 }
2808
2809 pub async fn cancel_workflow_run(
2811 &self,
2812 workflow_id: &str,
2813 run_id: &str,
2814 options: WorkflowCommandOptions,
2815 ) -> Result<WorkflowCommandResult> {
2816 self.workflow_command(
2817 workflow_id,
2818 Some(run_id),
2819 WorkflowCommandKind::Cancel,
2820 options,
2821 )
2822 .await
2823 }
2824
2825 pub async fn terminate_workflow(
2827 &self,
2828 workflow_id: &str,
2829 options: WorkflowCommandOptions,
2830 ) -> Result<WorkflowCommandResult> {
2831 self.workflow_command(workflow_id, None, WorkflowCommandKind::Terminate, options)
2832 .await
2833 }
2834
2835 pub async fn terminate_workflow_run(
2837 &self,
2838 workflow_id: &str,
2839 run_id: &str,
2840 options: WorkflowCommandOptions,
2841 ) -> Result<WorkflowCommandResult> {
2842 self.workflow_command(
2843 workflow_id,
2844 Some(run_id),
2845 WorkflowCommandKind::Terminate,
2846 options,
2847 )
2848 .await
2849 }
2850
2851 async fn workflow_command(
2852 &self,
2853 workflow_id: &str,
2854 run_id: Option<&str>,
2855 command: WorkflowCommandKind,
2856 options: WorkflowCommandOptions,
2857 ) -> Result<WorkflowCommandResult> {
2858 let path = match run_id {
2859 Some(run_id) => format!(
2860 "/workflows/{workflow_id}/runs/{run_id}/{}",
2861 command.as_str()
2862 ),
2863 None => format!("/workflows/{workflow_id}/{}", command.as_str()),
2864 };
2865 let data = match self
2866 .request_json(
2867 reqwest::Method::POST,
2868 &path,
2869 RequestProtocol::ControlPlane,
2870 Some(&options),
2871 )
2872 .await
2873 {
2874 Ok(data) => data,
2875 Err(Error::Http { status, body }) => {
2876 return Err(Error::WorkflowCommandRejected(workflow_command_rejection(
2877 command,
2878 status,
2879 body,
2880 workflow_id,
2881 run_id,
2882 )));
2883 }
2884 Err(error) => return Err(error),
2885 };
2886
2887 Ok(workflow_command_result(command, data, workflow_id, run_id))
2888 }
2889
2890 pub async fn query_workflow<T: Serialize>(
2896 &self,
2897 workflow_id: &str,
2898 query_name: &str,
2899 input: T,
2900 ) -> Result<Value> {
2901 self.query_workflow_target(workflow_id, None, query_name, input)
2902 .await
2903 }
2904
2905 pub async fn query_workflow_run<T: Serialize>(
2907 &self,
2908 workflow_id: &str,
2909 run_id: &str,
2910 query_name: &str,
2911 input: T,
2912 ) -> Result<Value> {
2913 self.query_workflow_target(workflow_id, Some(run_id), query_name, input)
2914 .await
2915 }
2916
2917 pub async fn query_workflow_avro_value<T: Serialize>(
2919 &self,
2920 workflow_id: &str,
2921 query_name: &str,
2922 input: T,
2923 ) -> Result<AvroValue> {
2924 self.query_workflow_avro_value_target(workflow_id, None, query_name, input)
2925 .await
2926 }
2927
2928 pub async fn query_workflow_run_avro_value<T: Serialize>(
2930 &self,
2931 workflow_id: &str,
2932 run_id: &str,
2933 query_name: &str,
2934 input: T,
2935 ) -> Result<AvroValue> {
2936 self.query_workflow_avro_value_target(workflow_id, Some(run_id), query_name, input)
2937 .await
2938 }
2939
2940 async fn query_workflow_avro_value_target<T: Serialize>(
2941 &self,
2942 workflow_id: &str,
2943 run_id: Option<&str>,
2944 query_name: &str,
2945 input: T,
2946 ) -> Result<AvroValue> {
2947 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2948 let body = json!({"input": encode_typed_envelope(&input, DEFAULT_CODEC)?});
2949 let path = match run_id {
2950 Some(run_id) => {
2951 format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
2952 }
2953 None => format!("/workflows/{workflow_id}/query/{query_name}"),
2954 };
2955 let response: Value = match self
2956 .request_json(
2957 reqwest::Method::POST,
2958 &path,
2959 RequestProtocol::ControlPlane,
2960 Some(&body),
2961 )
2962 .await
2963 {
2964 Ok(response) => response,
2965 Err(Error::Http { status, body }) => {
2966 return Err(Error::QueryFailed(query_failure(status, body)));
2967 }
2968 Err(error) => return Err(error),
2969 };
2970
2971 let envelope = response
2972 .get("result_envelope")
2973 .filter(|envelope| !envelope.is_null())
2974 .ok_or_else(|| {
2975 Error::Codec(
2976 "missing_payload_envelope: typed query result requires result_envelope"
2977 .to_string(),
2978 )
2979 })?;
2980 decode_wire_avro_value(envelope, DEFAULT_CODEC)
2981 }
2982
2983 async fn query_workflow_target<T: Serialize>(
2984 &self,
2985 workflow_id: &str,
2986 run_id: Option<&str>,
2987 query_name: &str,
2988 input: T,
2989 ) -> Result<Value> {
2990 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2991 let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2992 let body = json!({
2993 "input": input_envelope
2994 });
2995 let path = match run_id {
2996 Some(run_id) => {
2997 format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
2998 }
2999 None => format!("/workflows/{workflow_id}/query/{query_name}"),
3000 };
3001 let response: Value = match self
3002 .request_json(
3003 reqwest::Method::POST,
3004 &path,
3005 RequestProtocol::ControlPlane,
3006 Some(&body),
3007 )
3008 .await
3009 {
3010 Ok(response) => response,
3011 Err(Error::Http { status, body }) => {
3012 return Err(Error::QueryFailed(query_failure(status, body)));
3013 }
3014 Err(error) => return Err(error),
3015 };
3016
3017 if let Some(envelope) = response
3018 .get("result_envelope")
3019 .filter(|envelope| !envelope.is_null())
3020 {
3021 return decode_wire_value(envelope, DEFAULT_CODEC);
3022 }
3023
3024 Ok(response.get("result").cloned().unwrap_or(Value::Null))
3025 }
3026
3027 pub async fn update_workflow<T: Serialize>(
3029 &self,
3030 workflow_id: &str,
3031 update_name: &str,
3032 input: T,
3033 request_id: Option<&str>,
3034 ) -> Result<Value> {
3035 let response = self
3036 .update_workflow_response(workflow_id, update_name, input, request_id)
3037 .await?;
3038 if let Some(envelope) = response
3039 .get("result_envelope")
3040 .filter(|envelope| !envelope.is_null())
3041 {
3042 return decode_wire_value(envelope, DEFAULT_CODEC);
3043 }
3044 Ok(response.get("result").cloned().unwrap_or(response))
3045 }
3046
3047 pub async fn update_workflow_avro_value<T: Serialize>(
3049 &self,
3050 workflow_id: &str,
3051 update_name: &str,
3052 input: T,
3053 request_id: Option<&str>,
3054 ) -> Result<AvroValue> {
3055 let response = self
3056 .update_workflow_response(workflow_id, update_name, input, request_id)
3057 .await?;
3058 let envelope = response
3059 .get("result_envelope")
3060 .filter(|envelope| !envelope.is_null())
3061 .ok_or_else(|| {
3062 Error::Codec(
3063 "missing_payload_envelope: typed update result requires result_envelope"
3064 .to_string(),
3065 )
3066 })?;
3067 decode_wire_avro_value(envelope, DEFAULT_CODEC)
3068 }
3069
3070 async fn update_workflow_response<T: Serialize>(
3071 &self,
3072 workflow_id: &str,
3073 update_name: &str,
3074 input: T,
3075 request_id: Option<&str>,
3076 ) -> Result<Value> {
3077 let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
3078 let mut body = json!({
3079 "input": encode_typed_envelope(&input, DEFAULT_CODEC)?,
3080 "wait_for": "completed",
3081 });
3082 if let Some(request_id) = request_id {
3083 body["request_id"] = json!(request_id);
3084 }
3085 self.request_json(
3086 reqwest::Method::POST,
3087 &format!("/workflows/{workflow_id}/update/{update_name}"),
3088 RequestProtocol::ControlPlane,
3089 Some(&body),
3090 )
3091 .await
3092 }
3093
3094 pub async fn describe_workflow(&self, workflow_id: &str) -> Result<WorkflowDescription> {
3095 let path = format!("/workflows/{workflow_id}");
3096 let mut data: WorkflowDescription = self
3097 .request_json(
3098 reqwest::Method::GET,
3099 &path,
3100 RequestProtocol::ControlPlane,
3101 Option::<&Value>::None,
3102 )
3103 .await?;
3104 data.decode_payloads()?;
3105 Ok(data)
3106 }
3107
3108 pub async fn describe_workflow_run(
3110 &self,
3111 workflow_id: &str,
3112 run_id: &str,
3113 ) -> Result<WorkflowDescription> {
3114 let path = format!("/workflows/{workflow_id}/runs/{run_id}");
3115 let mut data: WorkflowDescription = self
3116 .request_json(
3117 reqwest::Method::GET,
3118 &path,
3119 RequestProtocol::ControlPlane,
3120 Option::<&Value>::None,
3121 )
3122 .await?;
3123 data.decode_payloads()?;
3124 Ok(data)
3125 }
3126
3127 fn workflow_stream_path(workflow_id: &str, run_id: &str, stream_name: Option<&str>) -> String {
3128 let mut path = format!(
3129 "/workflows/{}/runs/{}/streams",
3130 percent_encode_path_segment(workflow_id),
3131 percent_encode_path_segment(run_id),
3132 );
3133 if let Some(stream_name) = stream_name {
3134 path.push('/');
3135 path.push_str(&percent_encode_path_segment(stream_name));
3136 }
3137 path
3138 }
3139
3140 pub async fn list_workflow_streams(
3142 &self,
3143 workflow_id: &str,
3144 run_id: &str,
3145 ) -> Result<Vec<WorkflowStreamDescription>> {
3146 let response: WorkflowStreamListResponse = self
3147 .request_json(
3148 reqwest::Method::GET,
3149 &Self::workflow_stream_path(workflow_id, run_id, None),
3150 RequestProtocol::ControlPlane,
3151 Option::<&Value>::None,
3152 )
3153 .await?;
3154 Ok(response.streams)
3155 }
3156
3157 pub async fn describe_workflow_stream(
3159 &self,
3160 workflow_id: &str,
3161 run_id: &str,
3162 stream_name: &str,
3163 ) -> Result<WorkflowStreamDescription> {
3164 let response: WorkflowStreamDescriptionResponse = self
3165 .request_json(
3166 reqwest::Method::GET,
3167 &Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3168 RequestProtocol::ControlPlane,
3169 Option::<&Value>::None,
3170 )
3171 .await?;
3172 Ok(response.stream)
3173 }
3174
3175 pub async fn subscribe_workflow_stream(
3182 &self,
3183 workflow_id: &str,
3184 run_id: &str,
3185 stream_name: &str,
3186 from_offset: u64,
3187 max_items: usize,
3188 wait: Duration,
3189 ) -> Result<WorkflowStreamPage> {
3190 let max_items = max_items.clamp(1, 500);
3191 let wait_seconds = wait.as_secs().min(MAX_LONG_POLL_TIMEOUT_SECONDS);
3192 let path = format!(
3193 "{}/items?from={from_offset}&max_items={max_items}&wait_seconds={wait_seconds}",
3194 Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3195 );
3196 let response: WorkflowStreamPageResponse = self
3197 .request_json_with_timeout(
3198 reqwest::Method::GET,
3199 &path,
3200 RequestProtocol::ControlPlane,
3201 Option::<&Value>::None,
3202 Duration::from_secs(wait_seconds.saturating_add(5).max(5)),
3203 )
3204 .await?;
3205
3206 let items = response
3207 .items
3208 .into_iter()
3209 .map(|raw| {
3210 let offset = raw.get("offset").and_then(Value::as_u64).unwrap_or(0);
3211 let envelope = raw.get("payload").cloned();
3212 let payload = envelope
3213 .as_ref()
3214 .filter(|value| value.get("blob").is_some())
3215 .map(|value| decode_wire_avro_value(value, DEFAULT_CODEC))
3216 .transpose()?
3217 .map(AvroValue::into_json)
3218 .transpose()?;
3219 Ok(WorkflowStreamItem {
3220 offset,
3221 payload,
3222 payload_envelope: envelope,
3223 payload_reference: raw
3224 .get("payload_reference")
3225 .and_then(Value::as_str)
3226 .map(str::to_string),
3227 payload_codec: raw
3228 .get("payload_codec")
3229 .and_then(Value::as_str)
3230 .map(str::to_string),
3231 idempotency_key: raw
3232 .get("idempotency_key")
3233 .and_then(Value::as_str)
3234 .map(str::to_string),
3235 item_type: raw
3236 .get("item_type")
3237 .and_then(Value::as_str)
3238 .map(str::to_string),
3239 content_type: raw
3240 .get("content_type")
3241 .and_then(Value::as_str)
3242 .map(str::to_string),
3243 origin: raw
3244 .get("origin")
3245 .and_then(Value::as_str)
3246 .map(str::to_string),
3247 origin_reference: raw
3248 .get("origin_reference")
3249 .and_then(Value::as_str)
3250 .map(str::to_string),
3251 emitted_at: raw
3252 .get("emitted_at")
3253 .and_then(Value::as_str)
3254 .map(str::to_string),
3255 raw,
3256 })
3257 })
3258 .collect::<Result<Vec<_>>>()?;
3259 Ok(WorkflowStreamPage {
3260 stream: response.stream,
3261 items,
3262 next_offset: response.next_offset,
3263 terminal: response.terminal,
3264 })
3265 }
3266
3267 pub async fn append_workflow_stream(
3269 &self,
3270 workflow_id: &str,
3271 run_id: &str,
3272 stream_name: &str,
3273 items: &[WorkflowStreamAppendItem],
3274 max_pending_items: Option<u64>,
3275 ) -> Result<WorkflowStreamAppendResult> {
3276 if items.is_empty() {
3277 return Err(Error::Codec(
3278 "workflow_stream_items_empty: append requires at least one item".to_string(),
3279 ));
3280 }
3281 let mut body = json!({
3282 "items": items
3283 .iter()
3284 .map(|item| item.wire_value(None))
3285 .collect::<Vec<_>>(),
3286 });
3287 if let Some(max_pending_items) = max_pending_items {
3288 if max_pending_items == 0 {
3289 return Err(Error::Codec(
3290 "workflow_stream_pending_limit_invalid: max_pending_items must be positive"
3291 .to_string(),
3292 ));
3293 }
3294 body["max_pending_items"] = json!(max_pending_items);
3295 }
3296 let response: WorkflowStreamAppendResponse = self
3297 .request_json(
3298 reqwest::Method::POST,
3299 &format!(
3300 "{}/items",
3301 Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3302 ),
3303 RequestProtocol::ControlPlane,
3304 Some(&body),
3305 )
3306 .await?;
3307 Ok(WorkflowStreamAppendResult {
3308 stream: response.stream,
3309 accepted_offsets: response.accepted_offsets,
3310 accepted: response.accepted,
3311 deduped: response.deduped,
3312 })
3313 }
3314
3315 pub async fn close_workflow_stream(
3317 &self,
3318 workflow_id: &str,
3319 run_id: &str,
3320 stream_name: &str,
3321 error_reason: Option<&str>,
3322 retention_seconds: Option<u64>,
3323 ) -> Result<WorkflowStreamDescription> {
3324 let mut body = json!({});
3325 if let Some(error_reason) = error_reason {
3326 body["error_reason"] = json!(error_reason);
3327 }
3328 if let Some(retention_seconds) = retention_seconds {
3329 if retention_seconds == 0 {
3330 return Err(Error::Codec(
3331 "workflow_stream_retention_invalid: retention_seconds must be positive"
3332 .to_string(),
3333 ));
3334 }
3335 body["retention_seconds"] = json!(retention_seconds);
3336 }
3337 let response: WorkflowStreamDescriptionResponse = self
3338 .request_json(
3339 reqwest::Method::POST,
3340 &format!(
3341 "{}/close",
3342 Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3343 ),
3344 RequestProtocol::ControlPlane,
3345 Some(&body),
3346 )
3347 .await?;
3348 Ok(response.stream)
3349 }
3350
3351 pub async fn register_worker(
3352 &self,
3353 worker_id: &str,
3354 task_queue: &str,
3355 supported_workflow_types: Vec<String>,
3356 supported_activity_types: Vec<String>,
3357 max_concurrent_workflow_tasks: usize,
3358 max_concurrent_activity_tasks: usize,
3359 ) -> Result<RegisterWorkerResponse> {
3360 self.register_worker_with_capabilities(
3361 worker_id,
3362 task_queue,
3363 supported_workflow_types,
3364 supported_activity_types,
3365 max_concurrent_workflow_tasks,
3366 max_concurrent_activity_tasks,
3367 Vec::new(),
3368 )
3369 .await
3370 }
3371
3372 pub async fn register_worker_with_capabilities(
3374 &self,
3375 worker_id: &str,
3376 task_queue: &str,
3377 supported_workflow_types: Vec<String>,
3378 supported_activity_types: Vec<String>,
3379 max_concurrent_workflow_tasks: usize,
3380 max_concurrent_activity_tasks: usize,
3381 capabilities: Vec<String>,
3382 ) -> Result<RegisterWorkerResponse> {
3383 self.register_worker_with_command_contracts(
3384 worker_id,
3385 task_queue,
3386 supported_workflow_types,
3387 supported_activity_types,
3388 max_concurrent_workflow_tasks,
3389 max_concurrent_activity_tasks,
3390 capabilities,
3391 Value::Object(serde_json::Map::new()),
3392 )
3393 .await
3394 }
3395
3396 #[allow(clippy::too_many_arguments)]
3403 pub async fn register_worker_with_command_contracts(
3404 &self,
3405 worker_id: &str,
3406 task_queue: &str,
3407 supported_workflow_types: Vec<String>,
3408 supported_activity_types: Vec<String>,
3409 max_concurrent_workflow_tasks: usize,
3410 max_concurrent_activity_tasks: usize,
3411 capabilities: Vec<String>,
3412 workflow_command_contracts: Value,
3413 ) -> Result<RegisterWorkerResponse> {
3414 if let Some(contracts) = workflow_command_contracts.as_object() {
3415 for (workflow_type, contract) in contracts {
3416 let Some(update_validators) = contract.get("update_validators") else {
3417 continue;
3418 };
3419 if !update_validators
3420 .as_array()
3421 .is_some_and(|validators| validators.is_empty())
3422 {
3423 return Err(Error::UnsupportedUpdateValidators {
3424 workflow_type: workflow_type.clone(),
3425 });
3426 }
3427 }
3428 }
3429
3430 let mut body = json!({
3431 "worker_id": worker_id,
3432 "task_queue": task_queue,
3433 "runtime": "rust",
3434 "sdk_version": SDK_VERSION,
3435 "supported_workflow_types": supported_workflow_types,
3436 "supported_activity_types": supported_activity_types,
3437 "capabilities": capabilities,
3438 "capability_manifest": portable_worker_affinity_capability_manifest(),
3439 "max_concurrent_workflow_tasks": max_concurrent_workflow_tasks,
3440 "max_concurrent_activity_tasks": max_concurrent_activity_tasks
3441 });
3442 if workflow_command_contracts
3443 .as_object()
3444 .is_some_and(|contracts| !contracts.is_empty())
3445 {
3446 body["workflow_command_contracts"] = workflow_command_contracts;
3447 }
3448
3449 self.request_json(
3450 reqwest::Method::POST,
3451 "/worker/register",
3452 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3453 Some(&body),
3454 )
3455 .await
3456 }
3457
3458 pub async fn deregister_worker_registration(
3464 &self,
3465 worker_id: &str,
3466 ) -> Result<WorkerDeregistrationEnvelope> {
3467 let path = format!(
3468 "/worker/registrations/{}",
3469 percent_encode_path_segment(worker_id)
3470 );
3471 self.request_json(
3472 reqwest::Method::DELETE,
3473 &path,
3474 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3475 Option::<&Value>::None,
3476 )
3477 .await
3478 }
3479
3480 pub async fn poll_query_task(
3482 &self,
3483 worker_id: &str,
3484 task_queue: &str,
3485 timeout: Duration,
3486 ) -> Result<Option<QueryTask>> {
3487 Ok(self
3488 .poll_query_task_response(worker_id, task_queue, timeout)
3489 .await?
3490 .task)
3491 }
3492
3493 pub async fn poll_query_task_response(
3495 &self,
3496 worker_id: &str,
3497 task_queue: &str,
3498 timeout: Duration,
3499 ) -> Result<PollQueryTaskResponse> {
3500 let poll_request_id = unique_request_id("rust-query-poll");
3501 self.poll_query_task_response_with_request_id(
3502 worker_id,
3503 task_queue,
3504 timeout,
3505 &poll_request_id,
3506 1,
3507 )
3508 .await
3509 }
3510
3511 async fn poll_query_task_response_with_request_id(
3512 &self,
3513 worker_id: &str,
3514 task_queue: &str,
3515 timeout: Duration,
3516 poll_request_id: &str,
3517 transport_retries: usize,
3518 ) -> Result<PollQueryTaskResponse> {
3519 let timeout_seconds = long_poll_timeout_seconds(timeout);
3520 let body = json!({
3521 "worker_id": worker_id,
3522 "task_queue": task_queue,
3523 "poll_request_id": poll_request_id,
3524 "timeout_seconds": timeout_seconds,
3525 });
3526 self.poll_request_json(
3527 "/worker/query-tasks/poll",
3528 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3529 &body,
3530 timeout + Duration::from_secs(5),
3531 transport_retries,
3532 )
3533 .await
3534 }
3535
3536 pub async fn complete_query_task<T: Serialize>(
3538 &self,
3539 query_task_id: &str,
3540 lease_owner: &str,
3541 query_task_attempt: u64,
3542 result: T,
3543 codec: &str,
3544 ) -> Result<Value> {
3545 let typed_result = AvroValue::from_serialize(&result)?;
3546 let result_envelope = encode_typed_envelope(&typed_result, codec)?;
3547 self.complete_query_task_with_envelope(
3548 query_task_id,
3549 lease_owner,
3550 query_task_attempt,
3551 typed_result.into_json()?,
3552 result_envelope,
3553 )
3554 .await
3555 }
3556
3557 async fn complete_query_task_with_envelope(
3558 &self,
3559 query_task_id: &str,
3560 lease_owner: &str,
3561 query_task_attempt: u64,
3562 result: Value,
3563 result_envelope: Value,
3564 ) -> Result<Value> {
3565 let body = json!({
3566 "lease_owner": lease_owner,
3567 "query_task_attempt": query_task_attempt,
3568 "result": result,
3569 "result_envelope": result_envelope,
3570 });
3571 let path = format!("/worker/query-tasks/{query_task_id}/complete");
3572 let response = self
3573 .request_json(
3574 reqwest::Method::POST,
3575 &path,
3576 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3577 Some(&body),
3578 )
3579 .await;
3580 query_task_response(response)
3581 }
3582
3583 pub async fn fail_query_task(
3585 &self,
3586 query_task_id: &str,
3587 lease_owner: &str,
3588 query_task_attempt: u64,
3589 message: impl Into<String>,
3590 reason: impl Into<String>,
3591 failure_type: impl Into<String>,
3592 ) -> Result<Value> {
3593 let body = json!({
3594 "lease_owner": lease_owner,
3595 "query_task_attempt": query_task_attempt,
3596 "failure": {
3597 "message": message.into(),
3598 "reason": reason.into(),
3599 "type": failure_type.into(),
3600 }
3601 });
3602 let path = format!("/worker/query-tasks/{query_task_id}/fail");
3603 let response = self
3604 .request_json(
3605 reqwest::Method::POST,
3606 &path,
3607 RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3608 Some(&body),
3609 )
3610 .await;
3611 query_task_response(response)
3612 }
3613
3614 pub async fn heartbeat_worker(
3615 &self,
3616 worker_id: &str,
3617 workflow_available: usize,
3618 activity_available: usize,
3619 ) -> Result<Value> {
3620 let body = json!({
3621 "worker_id": worker_id,
3622 "task_slots": {
3623 "workflow_available": workflow_available,
3624 "activity_available": activity_available
3625 },
3626 "process_metrics": {
3627 "process_id": std::process::id(),
3628 "process_uptime_seconds": 0
3629 }
3630 });
3631
3632 self.request_json(
3633 reqwest::Method::POST,
3634 "/worker/heartbeat",
3635 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3636 Some(&body),
3637 )
3638 .await
3639 }
3640
3641 pub async fn poll_workflow_task(
3642 &self,
3643 worker_id: &str,
3644 task_queue: &str,
3645 timeout: Duration,
3646 ) -> Result<Option<WorkflowTask>> {
3647 Ok(self
3648 .poll_workflow_task_response(worker_id, task_queue, timeout)
3649 .await?
3650 .task)
3651 }
3652
3653 pub async fn poll_workflow_task_response(
3654 &self,
3655 worker_id: &str,
3656 task_queue: &str,
3657 timeout: Duration,
3658 ) -> Result<PollWorkflowTaskResponse> {
3659 let poll_request_id = unique_request_id("rust-workflow-poll");
3660 self.poll_workflow_task_response_with_request_id(
3661 worker_id,
3662 task_queue,
3663 timeout,
3664 &poll_request_id,
3665 1,
3666 )
3667 .await
3668 }
3669
3670 async fn poll_workflow_task_response_with_request_id(
3671 &self,
3672 worker_id: &str,
3673 task_queue: &str,
3674 timeout: Duration,
3675 poll_request_id: &str,
3676 transport_retries: usize,
3677 ) -> Result<PollWorkflowTaskResponse> {
3678 let body = json!({
3679 "worker_id": worker_id,
3680 "task_queue": task_queue,
3681 "poll_request_id": poll_request_id,
3682 "timeout_seconds": long_poll_timeout_seconds(timeout),
3683 });
3684 let mut data: PollWorkflowTaskResponse = self
3685 .poll_request_json(
3686 "/worker/workflow-tasks/poll",
3687 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3688 &body,
3689 timeout + Duration::from_secs(5),
3690 transport_retries,
3691 )
3692 .await?;
3693
3694 if let Some(task) = data.task.as_mut() {
3695 self.fetch_remaining_workflow_history(worker_id, task)
3696 .await?;
3697 }
3698
3699 Ok(data)
3700 }
3701
3702 async fn fetch_remaining_workflow_history(
3703 &self,
3704 worker_id: &str,
3705 task: &mut WorkflowTask,
3706 ) -> Result<()> {
3707 let mut next_token = task.next_history_page_token.clone();
3708
3709 while let Some(token) = next_token.take().filter(|token| !token.is_empty()) {
3710 let lease_owner = task
3711 .lease_owner
3712 .clone()
3713 .unwrap_or_else(|| worker_id.to_string());
3714 let page = self
3715 .workflow_task_history_page(
3716 &task.task_id,
3717 &lease_owner,
3718 task.workflow_task_attempt,
3719 &token,
3720 )
3721 .await?;
3722
3723 task.append_history_page(page);
3724
3725 if task.next_history_page_token.as_deref() == Some(token.as_str()) {
3726 return Err(Error::Codec(
3727 "workflow history pagination returned the same page token".to_string(),
3728 ));
3729 }
3730
3731 next_token = task.next_history_page_token.clone();
3732 }
3733
3734 Ok(())
3735 }
3736
3737 async fn workflow_task_history_page(
3738 &self,
3739 task_id: &str,
3740 lease_owner: &str,
3741 workflow_task_attempt: u64,
3742 next_history_page_token: &str,
3743 ) -> Result<WorkflowTaskHistoryPage> {
3744 let body = json!({
3745 "lease_owner": lease_owner,
3746 "workflow_task_attempt": workflow_task_attempt,
3747 "next_history_page_token": next_history_page_token
3748 });
3749 let path = format!("/worker/workflow-tasks/{task_id}/history");
3750
3751 self.request_json(
3752 reqwest::Method::POST,
3753 &path,
3754 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3755 Some(&body),
3756 )
3757 .await
3758 }
3759
3760 pub async fn complete_workflow_task(
3761 &self,
3762 task_id: &str,
3763 lease_owner: &str,
3764 workflow_task_attempt: u64,
3765 commands: Vec<Value>,
3766 ) -> Result<Value> {
3767 self.complete_workflow_task_with_message_streams(
3768 task_id,
3769 lease_owner,
3770 workflow_task_attempt,
3771 commands,
3772 Vec::new(),
3773 Vec::new(),
3774 )
3775 .await
3776 }
3777
3778 async fn complete_workflow_task_with_message_streams(
3779 &self,
3780 task_id: &str,
3781 lease_owner: &str,
3782 workflow_task_attempt: u64,
3783 commands: Vec<Value>,
3784 message_stream_cursors: Vec<Value>,
3785 message_stream_waits: Vec<Value>,
3786 ) -> Result<Value> {
3787 validate_workflow_task_commands(&commands)?;
3788 let has_message_stream_metadata =
3789 !message_stream_cursors.is_empty() || !message_stream_waits.is_empty();
3790 if has_message_stream_metadata
3791 && !worker_protocol_supports_message_streams(WORKER_PROTOCOL_VERSION)
3792 {
3793 return Err(Error::Codec(
3794 "message_streams_unavailable: message stream completion metadata requires worker protocol 1.15 or newer"
3795 .to_string(),
3796 ));
3797 }
3798 let protocol_version = workflow_completion_protocol_version_with_message_streams(
3799 &commands,
3800 has_message_stream_metadata,
3801 );
3802 let mut body = json!({
3803 "lease_owner": lease_owner,
3804 "workflow_task_attempt": workflow_task_attempt,
3805 "commands": commands
3806 });
3807 if !message_stream_cursors.is_empty() {
3808 body["message_stream_cursors"] = Value::Array(message_stream_cursors);
3809 }
3810 if !message_stream_waits.is_empty() {
3811 body["message_stream_waits"] = Value::Array(message_stream_waits);
3812 }
3813 let path = format!("/worker/workflow-tasks/{task_id}/complete");
3814 self.request_json(
3815 reqwest::Method::POST,
3816 &path,
3817 RequestProtocol::Worker(protocol_version),
3818 Some(&body),
3819 )
3820 .await
3821 }
3822
3823 pub async fn fail_workflow_task(
3824 &self,
3825 task_id: &str,
3826 lease_owner: &str,
3827 workflow_task_attempt: u64,
3828 message: impl Into<String>,
3829 ) -> Result<Value> {
3830 self.fail_workflow_task_with_type(
3831 task_id,
3832 lease_owner,
3833 workflow_task_attempt,
3834 message,
3835 "RustWorkflowTaskFailure",
3836 )
3837 .await
3838 }
3839
3840 async fn fail_workflow_task_with_type(
3841 &self,
3842 task_id: &str,
3843 lease_owner: &str,
3844 workflow_task_attempt: u64,
3845 message: impl Into<String>,
3846 failure_type: &str,
3847 ) -> Result<Value> {
3848 let body = json!({
3849 "lease_owner": lease_owner,
3850 "workflow_task_attempt": workflow_task_attempt,
3851 "failure": {
3852 "message": message.into(),
3853 "type": failure_type
3854 }
3855 });
3856 let path = format!("/worker/workflow-tasks/{task_id}/fail");
3857 self.request_json(
3858 reqwest::Method::POST,
3859 &path,
3860 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3861 Some(&body),
3862 )
3863 .await
3864 }
3865
3866 pub async fn poll_activity_task(
3867 &self,
3868 worker_id: &str,
3869 task_queue: &str,
3870 timeout: Duration,
3871 ) -> Result<Option<ActivityTask>> {
3872 Ok(self
3873 .poll_activity_task_response(worker_id, task_queue, timeout)
3874 .await?
3875 .task)
3876 }
3877
3878 pub async fn poll_activity_task_response(
3880 &self,
3881 worker_id: &str,
3882 task_queue: &str,
3883 timeout: Duration,
3884 ) -> Result<PollActivityTaskResponse> {
3885 let poll_request_id = unique_request_id("rust-activity-poll");
3886 self.poll_activity_task_response_with_request_id(
3887 worker_id,
3888 task_queue,
3889 timeout,
3890 &poll_request_id,
3891 1,
3892 )
3893 .await
3894 }
3895
3896 async fn poll_activity_task_response_with_request_id(
3897 &self,
3898 worker_id: &str,
3899 task_queue: &str,
3900 timeout: Duration,
3901 poll_request_id: &str,
3902 transport_retries: usize,
3903 ) -> Result<PollActivityTaskResponse> {
3904 let body = json!({
3905 "worker_id": worker_id,
3906 "task_queue": task_queue,
3907 "poll_request_id": poll_request_id,
3908 "timeout_seconds": long_poll_timeout_seconds(timeout),
3909 });
3910 let data: PollActivityTaskResponse = self
3911 .poll_request_json(
3912 "/worker/activity-tasks/poll",
3913 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3914 &body,
3915 timeout + Duration::from_secs(5),
3916 transport_retries,
3917 )
3918 .await?;
3919 Ok(data)
3920 }
3921
3922 pub async fn complete_activity_task<T: Serialize>(
3923 &self,
3924 task_id: &str,
3925 activity_attempt_id: &str,
3926 lease_owner: &str,
3927 result: T,
3928 codec: &str,
3929 ) -> Result<Value> {
3930 let result = encode_typed_envelope(&AvroValue::from_serialize(&result)?, codec)?;
3931 let body = json!({
3932 "activity_attempt_id": activity_attempt_id,
3933 "lease_owner": lease_owner,
3934 "result": result
3935 });
3936 let path = format!("/worker/activity-tasks/{task_id}/complete");
3937 activity_task_response(
3938 self.request_json(
3939 reqwest::Method::POST,
3940 &path,
3941 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3942 Some(&body),
3943 )
3944 .await,
3945 "complete",
3946 task_id,
3947 activity_attempt_id,
3948 )
3949 }
3950
3951 pub async fn fail_activity_task(
3952 &self,
3953 task_id: &str,
3954 activity_attempt_id: &str,
3955 lease_owner: &str,
3956 message: impl Into<String>,
3957 non_retryable: bool,
3958 ) -> Result<Value> {
3959 let body = json!({
3960 "activity_attempt_id": activity_attempt_id,
3961 "lease_owner": lease_owner,
3962 "failure": {
3963 "message": message.into(),
3964 "type": "RustActivityFailure",
3965 "non_retryable": non_retryable
3966 }
3967 });
3968 let path = format!("/worker/activity-tasks/{task_id}/fail");
3969 activity_task_response(
3970 self.request_json(
3971 reqwest::Method::POST,
3972 &path,
3973 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3974 Some(&body),
3975 )
3976 .await,
3977 "fail",
3978 task_id,
3979 activity_attempt_id,
3980 )
3981 }
3982
3983 pub async fn heartbeat_activity_task<T: Serialize>(
3984 &self,
3985 task_id: &str,
3986 activity_attempt_id: &str,
3987 lease_owner: &str,
3988 details: T,
3989 ) -> Result<ActivityHeartbeatResponse> {
3990 let details = encode_typed_envelope(&AvroValue::from_serialize(&details)?, DEFAULT_CODEC)?;
3991 let body = json!({
3992 "activity_attempt_id": activity_attempt_id,
3993 "lease_owner": lease_owner,
3994 "details": details
3995 });
3996 let path = format!("/worker/activity-tasks/{task_id}/heartbeat");
3997 activity_task_response(
3998 self.request_json(
3999 reqwest::Method::POST,
4000 &path,
4001 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
4002 Some(&body),
4003 )
4004 .await,
4005 "heartbeat",
4006 task_id,
4007 activity_attempt_id,
4008 )
4009 }
4010
4011 async fn request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
4012 &self,
4013 method: reqwest::Method,
4014 path: &str,
4015 protocol: RequestProtocol,
4016 body: Option<&B>,
4017 ) -> Result<T> {
4018 self.request_json_with_timeout(method, path, protocol, body, Duration::from_secs(60))
4019 .await
4020 }
4021
4022 async fn request_json_with_timeout<T: DeserializeOwned, B: Serialize + ?Sized>(
4023 &self,
4024 method: reqwest::Method,
4025 path: &str,
4026 protocol: RequestProtocol,
4027 body: Option<&B>,
4028 timeout: Duration,
4029 ) -> Result<T> {
4030 let auth_token = self.auth_token(protocol)?;
4031 let mut request = self
4032 .http
4033 .request(method, format!("{}/api{}", self.base_url, path))
4034 .timeout(timeout)
4035 .header(reqwest::header::ACCEPT, "application/json")
4036 .header(reqwest::header::CONTENT_TYPE, "application/json")
4037 .header("X-Namespace", &self.namespace);
4038
4039 match protocol {
4040 RequestProtocol::Worker(version) => {
4041 request = request.header("X-Durable-Workflow-Protocol-Version", version);
4042 }
4043 RequestProtocol::ControlPlane => {
4044 request = request.header(
4045 "X-Durable-Workflow-Control-Plane-Version",
4046 CONTROL_PLANE_VERSION,
4047 );
4048 }
4049 }
4050
4051 if let Some(token) = auth_token {
4052 request = request.bearer_auth(token);
4053 }
4054
4055 if let Some(body) = body {
4056 request = request.json(body);
4057 }
4058
4059 let request = request.build()?;
4060 let admission = self
4061 .worker_storage_admission
4062 .as_ref()
4063 .filter(|_| matches!(protocol, RequestProtocol::Worker(_)));
4064 let poll_request_id = path.ends_with("/poll").then(|| {
4065 request
4066 .body()
4067 .and_then(reqwest::Body::as_bytes)
4068 .and_then(|body| serde_json::from_slice::<Value>(body).ok())
4069 .and_then(|body| body.get("poll_request_id")?.as_str().map(str::to_owned))
4070 .unwrap_or_default()
4071 });
4072 let mut storage_retries = 0_usize;
4073
4074 loop {
4075 let response = self
4077 .http
4078 .execute(request.try_clone().ok_or_else(|| {
4079 Error::WorkerLoop("worker request body cannot be retried".to_string())
4080 })?)
4081 .await?;
4082 let status = response.status();
4083 let bytes = response.bytes().await?;
4084
4085 if !status.is_success() {
4086 let body = String::from_utf8_lossy(&bytes).to_string();
4087 if let Some(protocol) = protocol_failure(status, &body) {
4088 return Err(Error::Protocol(protocol));
4089 }
4090 let error = Error::Http { status, body };
4091 if let Some(admission) = admission {
4092 if let Some(advertised_delay) =
4093 worker_storage_admission_retry_after(&error, poll_request_id.as_deref())
4094 {
4095 storage_retries = storage_retries.saturating_add(1);
4096 let delay = worker_retry_delay(admission.policy, storage_retries)
4097 .max(advertised_delay)
4098 .min(admission.policy.max_backoff.max(Duration::from_millis(1)));
4099 let deadline = tokio::time::Instant::now() + delay;
4100 loop {
4101 if admission.stop.load(Ordering::SeqCst) {
4102 return Err(error);
4103 }
4104 let remaining =
4105 deadline.saturating_duration_since(tokio::time::Instant::now());
4106 if remaining.is_zero() {
4107 break;
4108 }
4109 tokio::time::sleep(remaining.min(Duration::from_millis(100))).await;
4110 }
4111 continue;
4112 }
4113 }
4114 return Err(error);
4115 }
4116
4117 if bytes.is_empty() {
4118 return Ok(serde_json::from_value(Value::Null)?);
4119 }
4120
4121 return Ok(serde_json::from_slice(&bytes)?);
4122 }
4123 }
4124
4125 async fn poll_request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
4126 &self,
4127 path: &str,
4128 protocol: RequestProtocol,
4129 body: &B,
4130 timeout: Duration,
4131 max_retries: usize,
4132 ) -> Result<T> {
4133 let mut retries = 0;
4134
4135 loop {
4136 let response = self
4137 .request_json_with_timeout(
4138 reqwest::Method::POST,
4139 path,
4140 protocol,
4141 Some(body),
4142 timeout,
4143 )
4144 .await;
4145
4146 match response {
4147 Err(Error::Transport(_)) if retries < max_retries => retries += 1,
4148 response => return worker_poll_response(response),
4149 }
4150 }
4151 }
4152
4153 fn auth_token(&self, protocol: RequestProtocol) -> Result<Option<&str>> {
4154 match protocol {
4155 RequestProtocol::Worker(_) => {
4156 if let Some(token) = self.worker_token.as_deref().or(self.token.as_deref()) {
4157 return Ok(Some(token));
4158 }
4159 if self.control_token.is_some() {
4160 return Err(Error::MissingRoleCredentials {
4161 role: "worker",
4162 opposite_role: "control",
4163 });
4164 }
4165 Ok(None)
4166 }
4167 RequestProtocol::ControlPlane => {
4168 if let Some(token) = self.control_token.as_deref().or(self.token.as_deref()) {
4169 return Ok(Some(token));
4170 }
4171 if self.worker_token.is_some() {
4172 return Err(Error::MissingRoleCredentials {
4173 role: "control",
4174 opposite_role: "worker",
4175 });
4176 }
4177 Ok(None)
4178 }
4179 }
4180 }
4181}
4182
4183fn query_failure(status: reqwest::StatusCode, raw_body: String) -> QueryFailure {
4184 let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
4185 let reason = body
4186 .get("reason")
4187 .and_then(Value::as_str)
4188 .unwrap_or("query_rejected")
4189 .to_string();
4190 let message = body
4191 .get("message")
4192 .or_else(|| body.get("error"))
4193 .and_then(Value::as_str)
4194 .unwrap_or("workflow query was rejected")
4195 .to_string();
4196
4197 QueryFailure {
4198 status: status.as_u16(),
4199 reason,
4200 message,
4201 body,
4202 }
4203}
4204
4205fn workflow_command_result(
4206 command: WorkflowCommandKind,
4207 data: Value,
4208 workflow_id: &str,
4209 run_id: Option<&str>,
4210) -> WorkflowCommandResult {
4211 WorkflowCommandResult {
4212 command,
4213 workflow_id: data
4214 .get("workflow_id")
4215 .and_then(Value::as_str)
4216 .unwrap_or(workflow_id)
4217 .to_string(),
4218 run_id: data
4219 .get("run_id")
4220 .and_then(Value::as_str)
4221 .or(run_id)
4222 .map(str::to_string),
4223 outcome: data
4224 .get("outcome")
4225 .and_then(Value::as_str)
4226 .map(str::to_string),
4227 reason: data
4228 .get("reason")
4229 .and_then(Value::as_str)
4230 .map(str::to_string),
4231 command_status: data
4232 .get("command_status")
4233 .and_then(Value::as_str)
4234 .map(str::to_string),
4235 raw: data,
4236 }
4237}
4238
4239fn workflow_command_rejection(
4240 command: WorkflowCommandKind,
4241 status: reqwest::StatusCode,
4242 raw_body: String,
4243 workflow_id: &str,
4244 run_id: Option<&str>,
4245) -> WorkflowCommandRejection {
4246 let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
4247 WorkflowCommandRejection {
4248 command,
4249 status: status.as_u16(),
4250 reason: body
4251 .get("reason")
4252 .and_then(Value::as_str)
4253 .unwrap_or("workflow_command_rejected")
4254 .to_string(),
4255 message: body
4256 .get("message")
4257 .or_else(|| body.get("error"))
4258 .and_then(Value::as_str)
4259 .unwrap_or("workflow lifecycle command was rejected")
4260 .to_string(),
4261 workflow_id: body
4262 .get("workflow_id")
4263 .and_then(Value::as_str)
4264 .unwrap_or(workflow_id)
4265 .to_string(),
4266 run_id: body
4267 .get("run_id")
4268 .and_then(Value::as_str)
4269 .or(run_id)
4270 .map(str::to_string),
4271 target_scope: body
4272 .get("target_scope")
4273 .and_then(Value::as_str)
4274 .map(str::to_string),
4275 body,
4276 }
4277}
4278
4279fn query_task_response(response: Result<Value>) -> Result<Value> {
4280 match response {
4281 Err(Error::Http { status, body }) => Err(Error::QueryFailed(query_failure(status, body))),
4282 response => response,
4283 }
4284}
4285
4286fn worker_poll_response<T: DeserializeOwned>(response: Result<T>) -> Result<T> {
4287 match response {
4288 Err(Error::Http { status, body })
4289 if status == reqwest::StatusCode::CONFLICT && worker_poll_body_is_stop(&body) =>
4290 {
4291 Ok(serde_json::from_str(&body)?)
4292 }
4293 response => response,
4294 }
4295}
4296
4297fn worker_poll_body_is_stop(body: &str) -> bool {
4298 serde_json::from_str::<Value>(body)
4299 .ok()
4300 .is_some_and(|body| {
4301 worker_poll_is_stop(
4302 body.get("poll_status").and_then(Value::as_str),
4303 body.get("reason").and_then(Value::as_str),
4304 )
4305 })
4306}
4307
4308fn worker_poll_is_stop(poll_status: Option<&str>, reason: Option<&str>) -> bool {
4309 matches!(poll_status, Some("draining" | "stopped"))
4310 || matches!(reason, Some("worker_draining" | "worker_stopped"))
4311}
4312
4313fn query_task_rejection_is_final(error: &Error) -> bool {
4314 matches!(
4315 error,
4316 Error::QueryFailed(failure)
4317 if QUERY_TASK_FINAL_REJECTION_REASONS.contains(&failure.reason.as_str())
4318 )
4319}
4320
4321fn activity_task_response<T>(
4322 response: Result<T>,
4323 operation: &str,
4324 task_id: &str,
4325 activity_attempt_id: &str,
4326) -> Result<T> {
4327 match response {
4328 Err(Error::Http { status, body }) => {
4329 let body = serde_json::from_str(&body).unwrap_or_else(|_| json!({"message": body}));
4330 Err(Error::ActivityTaskRejected(ActivityTaskRejection {
4331 operation: operation.to_string(),
4332 status: status.as_u16(),
4333 reason: body
4334 .get("reason")
4335 .and_then(Value::as_str)
4336 .unwrap_or("activity_task_rejected")
4337 .to_string(),
4338 task_id: body
4339 .get("task_id")
4340 .and_then(Value::as_str)
4341 .unwrap_or(task_id)
4342 .to_string(),
4343 activity_attempt_id: body
4344 .get("activity_attempt_id")
4345 .and_then(Value::as_str)
4346 .unwrap_or(activity_attempt_id)
4347 .to_string(),
4348 cancel_requested: body
4349 .get("cancel_requested")
4350 .and_then(Value::as_bool)
4351 .unwrap_or(false),
4352 can_continue: body.get("can_continue").and_then(Value::as_bool),
4353 run_closed_reason: body
4354 .get("run_closed_reason")
4355 .and_then(Value::as_str)
4356 .map(str::to_string),
4357 body,
4358 }))
4359 }
4360 response => response,
4361 }
4362}
4363
4364fn activity_task_rejection_is_final(error: &Error) -> bool {
4365 matches!(
4366 error,
4367 Error::ActivityTaskRejected(rejection)
4368 if matches!(
4369 rejection.reason.as_str(),
4370 "run_cancelled"
4371 | "run_terminated"
4372 | "attempt_closed"
4373 | "stale_attempt"
4374 | "activity_cancelled"
4375 | "task_cancelled"
4376 | "run_closed"
4377 | "activity_not_running"
4378 | "attempt_not_found"
4379 )
4380 )
4381}
4382
4383fn workflow_task_completion_is_terminal_timeout(
4384 error: &Error,
4385 task_id: &str,
4386 workflow_task_attempt: u64,
4387 run_id: Option<&str>,
4388) -> bool {
4389 let Error::Http { status, body } = error else {
4390 return false;
4391 };
4392 if *status != reqwest::StatusCode::CONFLICT {
4393 return false;
4394 }
4395
4396 let Some(run_id) = run_id else {
4397 return false;
4398 };
4399 let Ok(body) = serde_json::from_str::<Value>(body) else {
4400 return false;
4401 };
4402
4403 body.get("recorded").and_then(Value::as_bool) == Some(false)
4404 && body.get("reason").and_then(Value::as_str) == Some("run_timed_out")
4405 && body.get("run_status").and_then(Value::as_str) == Some("failed")
4406 && body.get("run_id").and_then(Value::as_str) == Some(run_id)
4407 && body.get("task_id").and_then(Value::as_str) == Some(task_id)
4408 && body.get("workflow_task_attempt").and_then(Value::as_u64) == Some(workflow_task_attempt)
4409}
4410
4411fn protocol_failure(status: reqwest::StatusCode, raw_body: &str) -> Option<ProtocolFailure> {
4412 let body: Value = serde_json::from_str(raw_body).ok()?;
4413 let reason = body.get("reason")?.as_str()?;
4414 if !matches!(
4415 reason,
4416 "missing_protocol_version"
4417 | "unsupported_protocol_version"
4418 | "missing_control_plane_version"
4419 | "unsupported_control_plane_version"
4420 ) {
4421 return None;
4422 }
4423
4424 Some(ProtocolFailure {
4425 status: status.as_u16(),
4426 reason: reason.to_string(),
4427 message: body
4428 .get("message")
4429 .or_else(|| body.get("error"))
4430 .and_then(Value::as_str)
4431 .unwrap_or("protocol version rejected")
4432 .to_string(),
4433 supported_version: body
4434 .get("supported_version")
4435 .and_then(Value::as_str)
4436 .map(str::to_string),
4437 requested_version: body
4438 .get("requested_version")
4439 .and_then(Value::as_str)
4440 .map(str::to_string),
4441 body,
4442 })
4443}
4444
4445fn long_poll_timeout_seconds(timeout: Duration) -> u64 {
4446 timeout
4447 .as_secs()
4448 .saturating_add(u64::from(timeout.subsec_nanos() > 0))
4449 .min(MAX_LONG_POLL_TIMEOUT_SECONDS)
4450}
4451
4452fn worker_operation_is_retryable(error: &Error) -> bool {
4453 if worker_poll_capacity_retry_after(error).is_some()
4454 || worker_storage_admission_body(error).is_some()
4455 || worker_operation_is_explicitly_non_retryable(error)
4456 {
4457 return false;
4458 }
4459
4460 match error {
4461 Error::Transport(error) => {
4462 error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()
4463 }
4464 Error::Http { status, .. } => {
4465 matches!(
4466 *status,
4467 reqwest::StatusCode::REQUEST_TIMEOUT | reqwest::StatusCode::TOO_MANY_REQUESTS
4468 ) || status.is_server_error()
4469 }
4470 _ => false,
4471 }
4472}
4473
4474fn worker_storage_admission_body(error: &Error) -> Option<Value> {
4475 let body: Value = match error {
4476 Error::Http { body, .. } => serde_json::from_str(body).ok()?,
4477 Error::ActivityTaskRejected(rejection) => rejection.body.clone(),
4478 _ => return None,
4479 };
4480 matches!(
4481 body.get("reason").and_then(Value::as_str),
4482 Some("storage_pressure" | "storage_admission_unavailable")
4483 )
4484 .then_some(body)
4485}
4486
4487fn worker_storage_admission_retry_after(
4488 error: &Error,
4489 poll_request_id: Option<&str>,
4490) -> Option<Duration> {
4491 let Error::Http { status, .. } = error else {
4492 return None;
4493 };
4494 let body = worker_storage_admission_body(error)?;
4495 let delay = body.get("retry_after_seconds")?.as_u64()?;
4496 if *status != reqwest::StatusCode::SERVICE_UNAVAILABLE
4497 || delay == 0
4498 || body.get("retryable") != Some(&Value::Bool(true))
4499 || !matches!(body.get("storage_state")?.as_str()?, "draining" | "fenced")
4500 || (body["reason"] == "storage_admission_unavailable" && body["storage_state"] != "fenced")
4501 || body
4502 .get("request_admitted")
4503 .is_some_and(|admitted| admitted != &Value::Bool(false))
4504 {
4505 return None;
4506 }
4507 match poll_request_id {
4508 Some(id) => {
4509 if id.is_empty()
4510 || body.get("task") != Some(&Value::Null)
4511 || body.get("poll_request_id").and_then(Value::as_str) != Some(id)
4512 || body.get("poll_status") != body.get("reason")
4513 || body.get("retry_same_poll_request_id") != Some(&Value::Bool(true))
4514 || body.get("claim_admitted") != Some(&Value::Bool(false))
4515 {
4516 return None;
4517 }
4518 }
4519 None if body.get("request_admitted") != Some(&Value::Bool(false)) => return None,
4520 None => {}
4521 }
4522 Some(Duration::from_secs(delay))
4523}
4524
4525fn worker_operation_is_explicitly_non_retryable(error: &Error) -> bool {
4526 let Error::Http { body, .. } = error else {
4527 return false;
4528 };
4529
4530 serde_json::from_str::<Value>(body)
4531 .ok()
4532 .and_then(|body| body.get("retryable").and_then(Value::as_bool))
4533 == Some(false)
4534}
4535
4536fn worker_poll_capacity_retry_after(error: &Error) -> Option<Duration> {
4537 let Error::Http { status, body } = error else {
4538 return None;
4539 };
4540 if *status != reqwest::StatusCode::TOO_MANY_REQUESTS {
4541 return None;
4542 }
4543
4544 let body = serde_json::from_str::<Value>(body).ok()?;
4545 let capacity_exhausted = body.get("poll_status").and_then(Value::as_str)
4546 == Some("long_poll_capacity_exhausted")
4547 || body.get("reason").and_then(Value::as_str) == Some("long_poll_capacity_exhausted");
4548 if !capacity_exhausted || body.get("retryable").and_then(Value::as_bool) != Some(true) {
4549 return None;
4550 }
4551
4552 Some(Duration::from_secs(
4553 body.get("retry_after_seconds")
4554 .and_then(Value::as_u64)
4555 .unwrap_or_default(),
4556 ))
4557}
4558
4559fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration {
4560 let exponent = retry.saturating_sub(1).min(31) as u32;
4561 policy
4562 .initial_backoff
4563 .saturating_mul(1_u32 << exponent)
4564 .min(policy.max_backoff)
4565}
4566
4567#[derive(Debug)]
4568pub struct ClientBuilder {
4569 base_url: String,
4570 token: Option<String>,
4571 control_token: Option<String>,
4572 worker_token: Option<String>,
4573 namespace: String,
4574 timeout: Duration,
4575}
4576
4577impl ClientBuilder {
4578 pub fn token(mut self, token: Option<String>) -> Self {
4579 self.token = token;
4580 self
4581 }
4582
4583 pub fn control_token(mut self, token: Option<String>) -> Self {
4584 self.control_token = token;
4585 self
4586 }
4587
4588 pub fn worker_token(mut self, token: Option<String>) -> Self {
4589 self.worker_token = token;
4590 self
4591 }
4592
4593 pub fn namespace(mut self, namespace: impl Into<String>) -> Self {
4594 self.namespace = namespace.into();
4595 self
4596 }
4597
4598 pub fn timeout(mut self, timeout: Duration) -> Self {
4599 self.timeout = timeout;
4600 self
4601 }
4602
4603 pub fn build(self) -> Result<Client> {
4604 let base_url = self.base_url.trim_end_matches('/').to_string();
4605 let has_sdk_api_suffix = reqwest::Url::parse(&base_url)
4606 .map(|url| url.path().trim_end_matches('/').ends_with("/api"))
4607 .unwrap_or_else(|_| base_url.ends_with("/api"));
4608
4609 if has_sdk_api_suffix {
4610 return Err(Error::InvalidBaseUrl);
4611 }
4612
4613 Ok(Client {
4614 http: reqwest::Client::builder().timeout(self.timeout).build()?,
4615 base_url,
4616 token: self.token,
4617 control_token: self.control_token,
4618 worker_token: self.worker_token,
4619 namespace: self.namespace,
4620 worker_storage_admission: None,
4621 })
4622 }
4623}
4624
4625#[derive(Clone, Debug)]
4626pub struct WorkflowHandle {
4627 client: Client,
4628 pub workflow_id: String,
4629 pub run_id: Option<String>,
4630 pub workflow_type: String,
4631}
4632
4633impl WorkflowHandle {
4634 pub async fn describe(&self) -> Result<WorkflowDescription> {
4636 self.client.describe_workflow(&self.workflow_id).await
4637 }
4638
4639 pub async fn describe_selected_run(&self) -> Result<WorkflowDescription> {
4641 let run_id = self.run_id.as_deref().ok_or_else(|| {
4642 Error::Codec("run_id is required for selected-run description".to_string())
4643 })?;
4644 self.client
4645 .describe_workflow_run(&self.workflow_id, run_id)
4646 .await
4647 }
4648
4649 pub async fn signal<T: Serialize>(&self, signal_name: &str, input: T) -> Result<Value> {
4650 self.client
4651 .signal_workflow(&self.workflow_id, signal_name, input)
4652 .await
4653 }
4654
4655 pub async fn append_message<T: Serialize>(
4656 &self,
4657 stream_name: &str,
4658 message_id: &str,
4659 input: T,
4660 ) -> Result<Value> {
4661 self.client
4662 .append_message_stream(&self.workflow_id, stream_name, message_id, input)
4663 .await
4664 }
4665
4666 pub async fn signal_selected_run<T: Serialize>(
4668 &self,
4669 signal_name: &str,
4670 input: T,
4671 ) -> Result<Value> {
4672 let run_id = self.run_id.as_deref().ok_or_else(|| {
4673 Error::Codec("run_id is required for selected-run signaling".to_string())
4674 })?;
4675 self.client
4676 .signal_workflow_run(&self.workflow_id, run_id, signal_name, input)
4677 .await
4678 }
4679
4680 pub async fn cancel(&self, options: WorkflowCommandOptions) -> Result<WorkflowCommandResult> {
4682 self.client
4683 .cancel_workflow(&self.workflow_id, options)
4684 .await
4685 }
4686
4687 pub async fn cancel_selected_run(
4689 &self,
4690 options: WorkflowCommandOptions,
4691 ) -> Result<WorkflowCommandResult> {
4692 let run_id = self.run_id.as_deref().ok_or_else(|| {
4693 Error::Codec("run_id is required for selected-run cancellation".to_string())
4694 })?;
4695 self.client
4696 .cancel_workflow_run(&self.workflow_id, run_id, options)
4697 .await
4698 }
4699
4700 pub async fn terminate(
4702 &self,
4703 options: WorkflowCommandOptions,
4704 ) -> Result<WorkflowCommandResult> {
4705 self.client
4706 .terminate_workflow(&self.workflow_id, options)
4707 .await
4708 }
4709
4710 pub async fn terminate_selected_run(
4712 &self,
4713 options: WorkflowCommandOptions,
4714 ) -> Result<WorkflowCommandResult> {
4715 let run_id = self.run_id.as_deref().ok_or_else(|| {
4716 Error::Codec("run_id is required for selected-run termination".to_string())
4717 })?;
4718 self.client
4719 .terminate_workflow_run(&self.workflow_id, run_id, options)
4720 .await
4721 }
4722
4723 pub async fn query<T: Serialize>(&self, query_name: &str, input: T) -> Result<Value> {
4725 self.client
4726 .query_workflow(&self.workflow_id, query_name, input)
4727 .await
4728 }
4729
4730 pub async fn query_avro_value<T: Serialize>(
4731 &self,
4732 query_name: &str,
4733 input: T,
4734 ) -> Result<AvroValue> {
4735 self.client
4736 .query_workflow_avro_value(&self.workflow_id, query_name, input)
4737 .await
4738 }
4739
4740 pub async fn update<T: Serialize>(
4741 &self,
4742 update_name: &str,
4743 input: T,
4744 request_id: Option<&str>,
4745 ) -> Result<Value> {
4746 self.client
4747 .update_workflow(&self.workflow_id, update_name, input, request_id)
4748 .await
4749 }
4750
4751 pub async fn update_avro_value<T: Serialize>(
4752 &self,
4753 update_name: &str,
4754 input: T,
4755 request_id: Option<&str>,
4756 ) -> Result<AvroValue> {
4757 self.client
4758 .update_workflow_avro_value(&self.workflow_id, update_name, input, request_id)
4759 .await
4760 }
4761
4762 pub async fn query_selected_run<T: Serialize>(
4764 &self,
4765 query_name: &str,
4766 input: T,
4767 ) -> Result<Value> {
4768 let run_id = self
4769 .run_id
4770 .as_deref()
4771 .ok_or_else(|| Error::Codec("run_id is required for selected-run query".to_string()))?;
4772 self.client
4773 .query_workflow_run(&self.workflow_id, run_id, query_name, input)
4774 .await
4775 }
4776
4777 pub async fn result(&self, options: WorkflowResultOptions) -> Result<Value> {
4779 self.result_target(options, None).await
4780 }
4781
4782 pub async fn result_avro_value(&self, options: WorkflowResultOptions) -> Result<AvroValue> {
4784 self.result_avro_value_target(options, None).await
4785 }
4786
4787 pub async fn result_typed<T: DeserializeOwned>(
4789 &self,
4790 options: WorkflowResultOptions,
4791 ) -> Result<T> {
4792 let result = self.result_avro_value(options).await?;
4793 decode_handler_result(result, HandlerKind::Workflow, &self.workflow_type)
4794 }
4795
4796 pub async fn result_selected_run(&self, options: WorkflowResultOptions) -> Result<Value> {
4798 let run_id = self.run_id.as_deref().ok_or_else(|| {
4799 Error::Codec("run_id is required for selected-run result".to_string())
4800 })?;
4801 self.result_target(options, Some(run_id)).await
4802 }
4803
4804 pub async fn result_selected_run_avro_value(
4806 &self,
4807 options: WorkflowResultOptions,
4808 ) -> Result<AvroValue> {
4809 let run_id = self.run_id.as_deref().ok_or_else(|| {
4810 Error::Codec("run_id is required for selected-run result".to_string())
4811 })?;
4812 self.result_avro_value_target(options, Some(run_id)).await
4813 }
4814
4815 pub async fn result_selected_run_typed<T: DeserializeOwned>(
4817 &self,
4818 options: WorkflowResultOptions,
4819 ) -> Result<T> {
4820 let result = self.result_selected_run_avro_value(options).await?;
4821 decode_handler_result(result, HandlerKind::Workflow, &self.workflow_type)
4822 }
4823
4824 async fn result_avro_value_target(
4825 &self,
4826 options: WorkflowResultOptions,
4827 selected_run_id: Option<&str>,
4828 ) -> Result<AvroValue> {
4829 let started = Instant::now();
4830
4831 loop {
4832 let description = match selected_run_id {
4833 Some(run_id) => {
4834 self.client
4835 .describe_workflow_run(&self.workflow_id, run_id)
4836 .await?
4837 }
4838 None => self.describe().await?,
4839 };
4840 if description.is_completed() {
4841 return description.output_avro_value.ok_or_else(|| {
4842 Error::Codec(
4843 "missing_payload_envelope: typed workflow result requires output_envelope"
4844 .to_string(),
4845 )
4846 });
4847 }
4848 if description.is_terminal() {
4849 let outcome =
4850 workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
4851 return Err(match outcome.kind {
4852 WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
4853 WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
4854 WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
4855 WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
4856 });
4857 }
4858 if started.elapsed() >= options.timeout {
4859 return Err(Error::Timeout);
4860 }
4861 tokio::time::sleep(options.poll_interval).await;
4862 }
4863 }
4864
4865 async fn result_target(
4866 &self,
4867 options: WorkflowResultOptions,
4868 selected_run_id: Option<&str>,
4869 ) -> Result<Value> {
4870 let started = Instant::now();
4871
4872 loop {
4873 let description = match selected_run_id {
4874 Some(run_id) => {
4875 self.client
4876 .describe_workflow_run(&self.workflow_id, run_id)
4877 .await?
4878 }
4879 None => self.describe().await?,
4880 };
4881 if description.is_completed() {
4882 return Ok(description.output.unwrap_or(Value::Null));
4883 }
4884
4885 if description.is_terminal() {
4886 let outcome =
4887 workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
4888 return Err(match outcome.kind {
4889 WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
4890 WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
4891 WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
4892 WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
4893 });
4894 }
4895
4896 if started.elapsed() >= options.timeout {
4897 return Err(Error::WorkflowTimedOut(WorkflowTerminalOutcome {
4898 kind: WorkflowTerminalKind::TimedOut,
4899 workflow_id: description
4900 .workflow_id
4901 .clone()
4902 .unwrap_or_else(|| self.workflow_id.clone()),
4903 run_id: description
4904 .run_id
4905 .clone()
4906 .or_else(|| selected_run_id.map(str::to_string)),
4907 reason: "result_wait_timeout".to_string(),
4908 failure_category: Some("client_timeout".to_string()),
4909 failure_id: None,
4910 exception_type: None,
4911 exception_class: None,
4912 non_retryable: None,
4913 message: Some(format!(
4914 "workflow result was not terminal within {:?}",
4915 options.timeout
4916 )),
4917 exception: None,
4918 raw: description.raw_value(),
4919 }));
4920 }
4921
4922 tokio::time::sleep(options.poll_interval).await;
4923 }
4924 }
4925}
4926
4927#[derive(Clone, Copy, Debug)]
4928pub struct WorkflowResultOptions {
4929 pub poll_interval: Duration,
4930 pub timeout: Duration,
4931}
4932
4933impl Default for WorkflowResultOptions {
4934 fn default() -> Self {
4935 Self {
4936 poll_interval: Duration::from_millis(500),
4937 timeout: Duration::from_secs(30),
4938 }
4939 }
4940}
4941
4942#[derive(Clone, Debug, Deserialize)]
4943pub struct WorkflowDescription {
4944 pub workflow_id: Option<String>,
4945 pub run_id: Option<String>,
4946 pub workflow_type: Option<String>,
4947 pub status: Option<String>,
4948 #[serde(default)]
4949 pub closed_reason: Option<String>,
4950 #[serde(default)]
4951 pub error: Option<String>,
4952 #[serde(default)]
4953 pub failure: Option<Value>,
4954 #[serde(default)]
4955 pub exception: Option<Value>,
4956 #[serde(default)]
4957 pub failures: Vec<Value>,
4958 #[serde(default)]
4959 pub output: Option<Value>,
4960 #[serde(default)]
4961 pub output_envelope: Option<Value>,
4962 #[serde(skip)]
4963 pub output_avro_value: Option<AvroValue>,
4964 #[serde(flatten)]
4965 pub raw: HashMap<String, Value>,
4966}
4967
4968#[derive(Clone, Debug, Deserialize)]
4970pub struct WorkflowStreamDescription {
4971 pub stream_name: String,
4972 pub status: String,
4973 pub last_offset: i64,
4974 pub total_items: u64,
4975 pub pending_items: u64,
4976 #[serde(default)]
4977 pub opened_at: Option<String>,
4978 #[serde(default)]
4979 pub last_appended_at: Option<String>,
4980 #[serde(default)]
4981 pub closed_at: Option<String>,
4982 #[serde(default)]
4983 pub error_reason: Option<String>,
4984 #[serde(default)]
4985 pub retention_seconds: Option<u64>,
4986 #[serde(flatten)]
4987 pub raw: HashMap<String, Value>,
4988}
4989
4990impl WorkflowStreamDescription {
4991 pub fn is_terminal(&self) -> bool {
4992 matches!(self.status.as_str(), "closed" | "errored")
4993 }
4994}
4995
4996#[derive(Clone, Debug, Default)]
4998pub struct WorkflowStreamAppendItem {
4999 pub payload_envelope: Option<Value>,
5000 pub payload_reference: Option<String>,
5001 pub item_type: Option<String>,
5002 pub content_type: Option<String>,
5003 pub idempotency_key: Option<String>,
5004}
5005
5006impl WorkflowStreamAppendItem {
5007 pub fn new<T: Serialize>(payload: T) -> Result<Self> {
5009 let value = AvroValue::from_serialize(&payload)?;
5010 Ok(Self {
5011 payload_envelope: Some(encode_typed_envelope(&value, DEFAULT_CODEC)?),
5012 ..Self::default()
5013 })
5014 }
5015
5016 pub fn from_reference(reference: impl Into<String>) -> Self {
5018 Self {
5019 payload_reference: Some(reference.into()),
5020 ..Self::default()
5021 }
5022 }
5023
5024 pub fn item_type(mut self, item_type: impl Into<String>) -> Self {
5025 self.item_type = Some(item_type.into());
5026 self
5027 }
5028
5029 pub fn content_type(mut self, content_type: impl Into<String>) -> Self {
5030 self.content_type = Some(content_type.into());
5031 self
5032 }
5033
5034 pub fn idempotency_key(mut self, idempotency_key: impl Into<String>) -> Self {
5035 self.idempotency_key = Some(idempotency_key.into());
5036 self
5037 }
5038
5039 fn wire_value(&self, derived_idempotency_key: Option<String>) -> Value {
5040 let mut item = serde_json::Map::new();
5041 if let Some(payload) = &self.payload_envelope {
5042 item.insert("payload".to_string(), payload.clone());
5043 item.insert("payload_codec".to_string(), json!(DEFAULT_CODEC));
5044 }
5045 if let Some(reference) = &self.payload_reference {
5046 item.insert("payload_reference".to_string(), json!(reference));
5047 }
5048 if let Some(item_type) = &self.item_type {
5049 item.insert("item_type".to_string(), json!(item_type));
5050 }
5051 if let Some(content_type) = &self.content_type {
5052 item.insert("content_type".to_string(), json!(content_type));
5053 }
5054 if let Some(key) = derived_idempotency_key
5055 .as_ref()
5056 .or(self.idempotency_key.as_ref())
5057 {
5058 item.insert("idempotency_key".to_string(), json!(key));
5059 }
5060 Value::Object(item)
5061 }
5062}
5063
5064#[derive(Clone, Debug)]
5066pub struct WorkflowStreamItem {
5067 pub offset: u64,
5068 pub payload: Option<Value>,
5069 pub payload_envelope: Option<Value>,
5070 pub payload_reference: Option<String>,
5071 pub payload_codec: Option<String>,
5072 pub idempotency_key: Option<String>,
5073 pub item_type: Option<String>,
5074 pub content_type: Option<String>,
5075 pub origin: Option<String>,
5076 pub origin_reference: Option<String>,
5077 pub emitted_at: Option<String>,
5078 pub raw: Value,
5079}
5080
5081#[derive(Clone, Debug)]
5083pub struct WorkflowStreamPage {
5084 pub stream: WorkflowStreamDescription,
5085 pub items: Vec<WorkflowStreamItem>,
5086 pub next_offset: u64,
5087 pub terminal: bool,
5088}
5089
5090#[derive(Clone, Debug)]
5092pub struct WorkflowStreamAppendResult {
5093 pub stream: WorkflowStreamDescription,
5094 pub accepted_offsets: Vec<u64>,
5095 pub accepted: u64,
5096 pub deduped: u64,
5097}
5098
5099#[derive(Deserialize)]
5100struct WorkflowStreamListResponse {
5101 #[serde(default)]
5102 streams: Vec<WorkflowStreamDescription>,
5103}
5104
5105#[derive(Deserialize)]
5106struct WorkflowStreamDescriptionResponse {
5107 stream: WorkflowStreamDescription,
5108}
5109
5110#[derive(Deserialize)]
5111struct WorkflowStreamPageResponse {
5112 stream: WorkflowStreamDescription,
5113 #[serde(default)]
5114 items: Vec<Value>,
5115 next_offset: u64,
5116 terminal: bool,
5117}
5118
5119#[derive(Deserialize)]
5120struct WorkflowStreamAppendResponse {
5121 stream: WorkflowStreamDescription,
5122 #[serde(default)]
5123 accepted_offsets: Vec<u64>,
5124 accepted: u64,
5125 deduped: u64,
5126}
5127
5128impl WorkflowDescription {
5129 pub fn is_completed(&self) -> bool {
5130 matches!(self.status.as_deref(), Some("completed" | "Completed"))
5131 }
5132
5133 pub fn is_terminal(&self) -> bool {
5134 matches!(
5135 self.status.as_deref(),
5136 Some(
5137 "completed"
5138 | "Completed"
5139 | "failed"
5140 | "Failed"
5141 | "cancelled"
5142 | "Cancelled"
5143 | "terminated"
5144 | "Terminated"
5145 | "timed_out"
5146 | "TimedOut",
5147 )
5148 )
5149 }
5150
5151 fn decode_payloads(&mut self) -> Result<()> {
5152 if let Some(envelope) = &self.output_envelope {
5153 let value = decode_wire_avro_value(envelope, DEFAULT_CODEC)?;
5154 self.output = Some(value.clone().into_json()?);
5155 self.output_avro_value = Some(value);
5156 }
5157
5158 Ok(())
5159 }
5160
5161 fn raw_value(&self) -> Value {
5162 let mut data = self.raw.clone();
5163 data.insert(
5164 "workflow_id".to_string(),
5165 self.workflow_id
5166 .clone()
5167 .map(Value::String)
5168 .unwrap_or(Value::Null),
5169 );
5170 data.insert(
5171 "run_id".to_string(),
5172 self.run_id
5173 .clone()
5174 .map(Value::String)
5175 .unwrap_or(Value::Null),
5176 );
5177 data.insert(
5178 "workflow_type".to_string(),
5179 self.workflow_type
5180 .clone()
5181 .map(Value::String)
5182 .unwrap_or(Value::Null),
5183 );
5184 data.insert(
5185 "status".to_string(),
5186 self.status
5187 .clone()
5188 .map(Value::String)
5189 .unwrap_or(Value::Null),
5190 );
5191 data.insert(
5192 "closed_reason".to_string(),
5193 self.closed_reason
5194 .clone()
5195 .map(Value::String)
5196 .unwrap_or(Value::Null),
5197 );
5198 if let Some(failure) = &self.failure {
5199 data.insert("failure".to_string(), failure.clone());
5200 }
5201 if let Some(exception) = &self.exception {
5202 data.insert("exception".to_string(), exception.clone());
5203 }
5204 Value::Object(data.into_iter().collect())
5205 }
5206}
5207
5208fn workflow_terminal_outcome(
5209 description: &WorkflowDescription,
5210 workflow_id: &str,
5211 run_id: Option<&str>,
5212) -> WorkflowTerminalOutcome {
5213 let terminal_kind = description
5214 .closed_reason
5215 .as_deref()
5216 .or(description.status.as_deref())
5217 .unwrap_or("failed")
5218 .to_ascii_lowercase();
5219 let kind = match terminal_kind.as_str() {
5220 "cancelled" | "canceled" => WorkflowTerminalKind::Cancelled,
5221 "terminated" => WorkflowTerminalKind::Terminated,
5222 "timed_out" | "timedout" => WorkflowTerminalKind::TimedOut,
5223 _ => WorkflowTerminalKind::Failed,
5224 };
5225 let default_reason = match kind {
5226 WorkflowTerminalKind::Failed => "workflow_failed",
5227 WorkflowTerminalKind::Cancelled => "cancelled",
5228 WorkflowTerminalKind::Terminated => "terminated",
5229 WorkflowTerminalKind::TimedOut => "timed_out",
5230 };
5231 let failure = description
5232 .failure
5233 .as_ref()
5234 .filter(|value| value.is_object());
5235 let nested_failure = failure
5236 .and_then(|value| value.get("failures"))
5237 .and_then(Value::as_array)
5238 .and_then(|failures| failures.last())
5239 .or_else(|| description.failures.last());
5240 let exception = description
5241 .exception
5242 .clone()
5243 .or_else(|| failure.and_then(|value| value.get("exception")).cloned())
5244 .or_else(|| {
5245 nested_failure
5246 .and_then(|value| value.get("exception_payload"))
5247 .cloned()
5248 });
5249 let string_field = |name: &str| {
5250 failure
5251 .and_then(|value| value.get(name))
5252 .and_then(Value::as_str)
5253 .or_else(|| {
5254 nested_failure
5255 .and_then(|value| value.get(name))
5256 .and_then(Value::as_str)
5257 })
5258 .map(str::to_string)
5259 };
5260 let exception_field = |name: &str| {
5261 exception
5262 .as_ref()
5263 .and_then(|value| value.get(name))
5264 .and_then(Value::as_str)
5265 .map(str::to_string)
5266 };
5267 let message = description
5268 .error
5269 .clone()
5270 .or_else(|| string_field("message"))
5271 .or_else(|| exception_field("message"));
5272 let reason = description
5273 .raw
5274 .get("reason")
5275 .and_then(Value::as_str)
5276 .map(str::to_string)
5277 .or_else(|| {
5278 failure
5279 .and_then(|value| value.get("reason"))
5280 .and_then(Value::as_str)
5281 .map(str::to_string)
5282 })
5283 .or_else(|| description.closed_reason.clone())
5284 .unwrap_or_else(|| default_reason.to_string());
5285 let failure_id = string_field("failure_id").or_else(|| {
5286 nested_failure
5287 .and_then(|value| value.get("id"))
5288 .and_then(Value::as_str)
5289 .map(str::to_string)
5290 });
5291
5292 WorkflowTerminalOutcome {
5293 kind,
5294 workflow_id: description
5295 .workflow_id
5296 .clone()
5297 .unwrap_or_else(|| workflow_id.to_string()),
5298 run_id: description
5299 .run_id
5300 .clone()
5301 .or_else(|| run_id.map(str::to_string)),
5302 reason,
5303 failure_category: string_field("failure_category")
5304 .or_else(|| Some(default_reason.to_string())),
5305 failure_id,
5306 exception_type: string_field("exception_type").or_else(|| exception_field("type")),
5307 exception_class: string_field("exception_class").or_else(|| exception_field("class")),
5308 non_retryable: failure
5309 .and_then(|value| value.get("non_retryable"))
5310 .and_then(Value::as_bool)
5311 .or_else(|| {
5312 nested_failure
5313 .and_then(|value| value.get("non_retryable"))
5314 .and_then(Value::as_bool)
5315 }),
5316 message,
5317 exception,
5318 raw: description.raw_value(),
5319 }
5320}
5321
5322#[derive(Clone, Debug, Deserialize)]
5323pub struct RegisterWorkerResponse {
5324 pub worker_id: String,
5325 pub registered: bool,
5326 #[serde(default)]
5327 pub heartbeat_interval_seconds: Option<u64>,
5328 #[serde(default)]
5329 pub protocol_version: Option<String>,
5330 #[serde(default)]
5331 pub server_capabilities: Option<Value>,
5332}
5333
5334#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
5336pub struct WorkerDeregistrationEnvelope {
5337 pub worker_id: String,
5338 pub outcome: String,
5339 pub recovered_workflow_task_count: u64,
5340}
5341
5342#[derive(Clone, Debug, Deserialize)]
5343pub struct PollWorkflowTaskResponse {
5344 #[serde(default)]
5345 pub task: Option<WorkflowTask>,
5346 #[serde(default)]
5347 pub poll_status: Option<String>,
5348 #[serde(default)]
5349 pub reason: Option<String>,
5350 #[serde(default)]
5351 pub protocol_version: Option<String>,
5352 #[serde(default)]
5353 pub server_capabilities: Option<Value>,
5354}
5355
5356impl PollWorkflowTaskResponse {
5357 pub fn outcome(&self) -> WorkerPollOutcome {
5359 worker_poll_outcome(
5360 self.task.is_some(),
5361 self.poll_status.as_deref(),
5362 self.reason.as_deref(),
5363 )
5364 }
5365}
5366
5367fn runtime_supports_workflow_memo_updates(capabilities: Option<&Value>) -> bool {
5368 let Some(capabilities) = capabilities.and_then(Value::as_object) else {
5369 return false;
5370 };
5371 let supported = capabilities
5372 .get("workflow_memo_updates")
5373 .and_then(Value::as_object)
5374 .and_then(|memo| memo.get("supported"))
5375 .and_then(Value::as_bool)
5376 == Some(true);
5377 let command_advertised = capabilities
5378 .get("supported_workflow_task_commands")
5379 .and_then(Value::as_array)
5380 .is_some_and(|commands| {
5381 commands
5382 .iter()
5383 .any(|command| command.as_str() == Some("upsert_memo"))
5384 });
5385 supported && command_advertised
5386}
5387
5388fn commands_use_workflow_memo_updates(commands: &[Value]) -> bool {
5389 commands
5390 .iter()
5391 .any(|command| command.get("type").and_then(Value::as_str) == Some("upsert_memo"))
5392}
5393
5394#[derive(Clone, Debug, Deserialize)]
5395pub struct PollActivityTaskResponse {
5396 #[serde(default)]
5397 pub task: Option<ActivityTask>,
5398 #[serde(default)]
5399 pub poll_status: Option<String>,
5400 #[serde(default)]
5401 pub reason: Option<String>,
5402}
5403
5404impl PollActivityTaskResponse {
5405 pub fn outcome(&self) -> WorkerPollOutcome {
5407 worker_poll_outcome(
5408 self.task.is_some(),
5409 self.poll_status.as_deref(),
5410 self.reason.as_deref(),
5411 )
5412 }
5413}
5414
5415#[derive(Clone, Debug, Deserialize)]
5416pub struct PollQueryTaskResponse {
5417 #[serde(default)]
5418 pub task: Option<QueryTask>,
5419 #[serde(default)]
5420 pub poll_status: Option<String>,
5421 #[serde(default)]
5422 pub reason: Option<String>,
5423}
5424
5425impl PollQueryTaskResponse {
5426 pub fn outcome(&self) -> WorkerPollOutcome {
5428 worker_poll_outcome(
5429 self.task.is_some(),
5430 self.poll_status.as_deref(),
5431 self.reason.as_deref(),
5432 )
5433 }
5434}
5435
5436#[derive(Clone, Debug, PartialEq, Eq)]
5438pub enum WorkerPollOutcome {
5439 Task,
5441 Idle {
5443 poll_status: Option<String>,
5444 reason: Option<String>,
5445 },
5446 Stop {
5448 poll_status: Option<String>,
5449 reason: Option<String>,
5450 },
5451}
5452
5453impl WorkerPollOutcome {
5454 pub fn should_stop(&self) -> bool {
5455 matches!(self, Self::Stop { .. })
5456 }
5457}
5458
5459fn worker_poll_outcome(
5460 has_task: bool,
5461 poll_status: Option<&str>,
5462 reason: Option<&str>,
5463) -> WorkerPollOutcome {
5464 if worker_poll_is_stop(poll_status, reason) {
5465 return WorkerPollOutcome::Stop {
5466 poll_status: poll_status.map(str::to_string),
5467 reason: reason.map(str::to_string),
5468 };
5469 }
5470
5471 if has_task {
5472 WorkerPollOutcome::Task
5473 } else {
5474 WorkerPollOutcome::Idle {
5475 poll_status: poll_status.map(str::to_string),
5476 reason: reason.map(str::to_string),
5477 }
5478 }
5479}
5480
5481#[derive(Clone, Debug, Deserialize)]
5483pub struct QueryTask {
5484 pub query_task_id: String,
5485 #[serde(default = "default_workflow_task_attempt")]
5486 pub query_task_attempt: u64,
5487 #[serde(default)]
5488 pub lease_owner: Option<String>,
5489 #[serde(default)]
5490 pub workflow_id: Option<String>,
5491 #[serde(default)]
5492 pub run_id: Option<String>,
5493 pub workflow_type: String,
5494 pub query_name: String,
5495 #[serde(
5496 default = "missing_task_payload_codec",
5497 deserialize_with = "deserialize_task_payload_codec"
5498 )]
5499 pub payload_codec: String,
5500 #[serde(default)]
5501 pub workflow_arguments: Option<Value>,
5502 #[serde(default)]
5503 pub query_arguments: Option<Value>,
5504 #[serde(default)]
5505 pub history_events: Vec<HistoryEvent>,
5506 #[serde(default)]
5507 pub history_export: Option<Value>,
5508 #[serde(default)]
5509 pub run_status: Option<String>,
5510}
5511
5512#[derive(Clone, Debug, Deserialize)]
5513pub struct WorkflowTask {
5514 pub task_id: String,
5515 #[serde(default)]
5516 pub workflow_command_id: Option<String>,
5517 #[serde(default)]
5518 pub workflow_id: Option<String>,
5519 #[serde(default)]
5520 pub run_id: Option<String>,
5521 pub workflow_type: String,
5522 #[serde(default)]
5523 pub cancel_requested: bool,
5524 #[serde(
5525 default = "missing_task_payload_codec",
5526 deserialize_with = "deserialize_task_payload_codec"
5527 )]
5528 pub payload_codec: String,
5529 #[serde(default)]
5530 pub arguments: Option<Value>,
5531 #[serde(default)]
5532 pub history_events: Vec<HistoryEvent>,
5533 #[serde(default)]
5534 pub total_history_events: Option<u64>,
5535 #[serde(default)]
5536 pub history_size_bytes: Option<u64>,
5537 #[serde(default)]
5538 pub continue_as_new_recommended: Option<bool>,
5539 #[serde(default)]
5540 pub history_budget_pressure: Option<String>,
5541 #[serde(default)]
5542 pub next_history_page_token: Option<String>,
5543 #[serde(default = "default_workflow_task_attempt")]
5544 pub workflow_task_attempt: u64,
5545 #[serde(default)]
5546 pub workflow_signal_id: Option<String>,
5547 #[serde(default)]
5548 pub signal_name: Option<String>,
5549 #[serde(default)]
5550 pub signal_arguments: Option<Value>,
5551 #[serde(default)]
5552 pub workflow_update_id: Option<String>,
5553 #[serde(default)]
5554 pub update_name: Option<String>,
5555 #[serde(default)]
5556 pub lease_owner: Option<String>,
5557}
5558
5559impl WorkflowTask {
5560 fn append_history_page(&mut self, page: WorkflowTaskHistoryPage) {
5561 self.history_events.extend(page.history_events);
5562
5563 if page.total_history_events.is_some() {
5564 self.total_history_events = page.total_history_events;
5565 }
5566
5567 self.next_history_page_token = page
5568 .next_history_page_token
5569 .filter(|token| !token.is_empty());
5570 }
5571}
5572
5573#[derive(Clone, Debug, Deserialize)]
5574struct WorkflowTaskHistoryPage {
5575 #[serde(default)]
5576 history_events: Vec<HistoryEvent>,
5577 #[serde(default)]
5578 total_history_events: Option<u64>,
5579 #[serde(default)]
5580 next_history_page_token: Option<String>,
5581}
5582
5583#[derive(Clone, Debug, Deserialize)]
5584pub struct ActivityTask {
5585 pub task_id: String,
5586 #[serde(default)]
5587 pub activity_attempt_id: Option<String>,
5588 #[serde(default)]
5589 pub attempt_id: Option<String>,
5590 pub activity_type: String,
5591 #[serde(
5592 default = "missing_task_payload_codec",
5593 deserialize_with = "deserialize_task_payload_codec"
5594 )]
5595 pub payload_codec: String,
5596 #[serde(default)]
5597 pub arguments: Option<Value>,
5598 #[serde(default = "default_attempt_number")]
5599 pub attempt_number: u64,
5600 #[serde(default)]
5601 pub lease_owner: Option<String>,
5602}
5603
5604#[derive(Clone, Debug, Deserialize)]
5605pub struct HistoryEvent {
5606 #[serde(alias = "type")]
5607 pub event_type: String,
5608 #[serde(default)]
5609 pub payload: Value,
5610 #[serde(flatten)]
5611 pub raw: HashMap<String, Value>,
5612}
5613
5614#[derive(Clone, Debug, PartialEq)]
5616pub struct QuerySignal {
5617 pub id: Option<String>,
5618 pub name: String,
5619 pub arguments: Vec<Value>,
5620 avro_arguments: Vec<AvroValue>,
5621 pub workflow_sequence: Option<u64>,
5622}
5623
5624impl QuerySignal {
5625 pub fn arguments_avro_value(&self) -> &[AvroValue] {
5627 &self.avro_arguments
5628 }
5629}
5630
5631#[derive(Clone, Debug)]
5637pub struct QueryContext {
5638 pub workflow_id: Option<String>,
5639 pub run_id: Option<String>,
5640 pub workflow_type: String,
5641 pub run_status: Option<String>,
5642 workflow_input: Value,
5643 workflow_input_avro_value: AvroValue,
5644 history_events: Arc<Vec<HistoryEvent>>,
5645 signal_events: Arc<Vec<QuerySignal>>,
5646}
5647
5648impl QueryContext {
5649 pub fn workflow_input(&self) -> &Value {
5651 &self.workflow_input
5652 }
5653
5654 pub fn workflow_input_avro_value(&self) -> &AvroValue {
5656 &self.workflow_input_avro_value
5657 }
5658
5659 pub fn history_events(&self) -> &[HistoryEvent] {
5661 self.history_events.as_slice()
5662 }
5663
5664 pub fn signal_events(&self) -> &[QuerySignal] {
5666 self.signal_events.as_slice()
5667 }
5668
5669 pub fn signals(&self, signal_name: &str) -> Vec<Vec<Value>> {
5671 self.signal_events
5672 .iter()
5673 .filter(|signal| signal.name == signal_name)
5674 .map(|signal| signal.arguments.clone())
5675 .collect()
5676 }
5677
5678 pub fn signals_avro_value(&self, signal_name: &str) -> Vec<Vec<AvroValue>> {
5680 self.signal_events
5681 .iter()
5682 .filter(|signal| signal.name == signal_name)
5683 .map(|signal| signal.avro_arguments.clone())
5684 .collect()
5685 }
5686}
5687
5688#[derive(Clone, Debug, Deserialize)]
5689pub struct ActivityHeartbeatResponse {
5690 #[serde(default)]
5691 pub cancel_requested: bool,
5692 #[serde(default)]
5693 pub heartbeat_recorded: bool,
5694 #[serde(default)]
5695 pub can_continue: Option<bool>,
5696 #[serde(default)]
5697 pub reason: Option<String>,
5698 #[serde(default)]
5699 pub run_closed_reason: Option<String>,
5700 #[serde(default)]
5701 pub run_closed_at: Option<String>,
5702 #[serde(default)]
5703 pub lease_expires_at: Option<String>,
5704 #[serde(default)]
5705 pub last_heartbeat_at: Option<String>,
5706}
5707
5708impl ActivityHeartbeatResponse {
5709 pub fn should_stop(&self) -> bool {
5711 self.cancel_requested || self.can_continue == Some(false)
5712 }
5713}
5714
5715fn missing_task_payload_codec() -> String {
5716 MISSING_TASK_PAYLOAD_CODEC.to_string()
5717}
5718
5719fn deserialize_task_payload_codec<'de, D>(deserializer: D) -> std::result::Result<String, D::Error>
5720where
5721 D: Deserializer<'de>,
5722{
5723 Ok(match Value::deserialize(deserializer)? {
5724 Value::String(codec) => codec,
5725 Value::Null => NULL_TASK_PAYLOAD_CODEC.to_string(),
5726 _ => NON_STRING_TASK_PAYLOAD_CODEC.to_string(),
5727 })
5728}
5729
5730fn default_workflow_task_attempt() -> u64 {
5731 1
5732}
5733
5734fn default_attempt_number() -> u64 {
5735 1
5736}
5737
5738type WorkflowFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5739type WorkflowHandler = Arc<dyn Fn(WorkflowContext, AvroValue) -> WorkflowFuture + Send + Sync>;
5740type ErasedWorkflowState = Arc<dyn Any + Send + Sync>;
5741type WorkflowStateSnapshot = Arc<dyn Fn() -> Result<ErasedWorkflowState> + Send + Sync>;
5742type ReplayedWorkflowHandler =
5743 Arc<dyn Fn(WorkflowContext, AvroValue) -> ReplayedWorkflowInvocation + Send + Sync>;
5744type ActivityFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5745type ActivityHandler = Arc<dyn Fn(ActivityContext, AvroValue) -> ActivityFuture + Send + Sync>;
5746type QueryFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5747type QueryHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
5748type UpdateHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
5749type ReplayedQueryHandler = Arc<
5750 dyn Fn(QueryContext, ErasedWorkflowState, AvroValue) -> std::result::Result<QueryFuture, String>
5751 + Send
5752 + Sync,
5753>;
5754type WorkerHeartbeatObserver = Arc<dyn Fn(&WorkerHeartbeatObservation) + Send + Sync>;
5755
5756struct ReplayedWorkflowInvocation {
5757 future: WorkflowFuture,
5758 snapshot: WorkflowStateSnapshot,
5759}
5760
5761#[derive(Clone)]
5762struct RegisteredWorkflow {
5763 execute: WorkflowHandler,
5764 replay: Option<ReplayedWorkflowHandler>,
5765 state_type: Option<TypeId>,
5766}
5767
5768#[derive(Debug)]
5769struct WorkflowTaskDecision {
5770 commands: Vec<Value>,
5771 message_stream_cursors: Vec<Value>,
5772 message_stream_waits: Vec<Value>,
5773}
5774
5775impl WorkflowTaskDecision {
5776 fn without_message_streams(commands: Vec<Value>) -> Self {
5777 Self {
5778 commands,
5779 message_stream_cursors: Vec::new(),
5780 message_stream_waits: Vec::new(),
5781 }
5782 }
5783}
5784
5785#[derive(Clone)]
5786enum RegisteredQuery {
5787 Snapshot(QueryHandler),
5788 Replayed {
5789 state_type: TypeId,
5790 handler: ReplayedQueryHandler,
5791 },
5792}
5793
5794#[derive(Clone, Debug)]
5795pub struct WorkerHeartbeatObservation {
5796 pub worker_id: String,
5797 pub task_queue: String,
5798 pub acknowledged_at_unix_millis: u64,
5799 pub acknowledgement: Value,
5800}
5801
5802#[derive(Clone, Copy, Debug)]
5814pub struct WorkerRetryPolicy {
5815 pub max_retries: usize,
5817 pub initial_backoff: Duration,
5819 pub max_backoff: Duration,
5821}
5822
5823impl Default for WorkerRetryPolicy {
5824 fn default() -> Self {
5825 Self {
5826 max_retries: 5,
5827 initial_backoff: Duration::from_millis(100),
5828 max_backoff: Duration::from_secs(5),
5829 }
5830 }
5831}
5832
5833#[derive(Clone, Debug)]
5834struct WorkerStorageAdmission {
5835 policy: WorkerRetryPolicy,
5836 stop: Arc<AtomicBool>,
5837}
5838
5839struct StopWorkerOnDrop(Arc<AtomicBool>);
5840
5841impl Drop for StopWorkerOnDrop {
5842 fn drop(&mut self) {
5843 self.0.store(true, Ordering::SeqCst);
5844 }
5845}
5846
5847async fn wait_for_worker_stop(stop: &AtomicBool) {
5848 while !stop.load(Ordering::SeqCst) {
5849 tokio::time::sleep(Duration::from_millis(100)).await;
5850 }
5851}
5852
5853#[derive(Clone, Copy, Debug, PartialEq, Eq)]
5854enum ManagedPollOutcome {
5855 Idle,
5856 Handled,
5857 Stop,
5858}
5859
5860#[derive(Clone)]
5861pub struct Worker {
5862 client: Client,
5863 worker_id: String,
5864 task_queue: String,
5865 workflows: HashMap<String, RegisteredWorkflow>,
5866 activities: HashMap<String, ActivityHandler>,
5867 queries: HashMap<String, HashMap<String, RegisteredQuery>>,
5868 updates: HashMap<String, HashMap<String, UpdateHandler>>,
5869 max_concurrent_workflow_tasks: usize,
5870 max_concurrent_activity_tasks: usize,
5871 poll_timeout: Duration,
5872 heartbeat_interval: Duration,
5873 retry_policy: WorkerRetryPolicy,
5874 heartbeat_observer: Option<WorkerHeartbeatObserver>,
5875}
5876
5877impl Worker {
5878 pub fn new(client: Client, task_queue: impl Into<String>) -> Self {
5879 Self {
5880 client,
5881 worker_id: default_worker_id(),
5882 task_queue: task_queue.into(),
5883 workflows: HashMap::new(),
5884 activities: HashMap::new(),
5885 queries: HashMap::new(),
5886 updates: HashMap::new(),
5887 max_concurrent_workflow_tasks: 10,
5888 max_concurrent_activity_tasks: 10,
5889 poll_timeout: Duration::from_secs(30),
5890 heartbeat_interval: Duration::from_secs(60),
5891 retry_policy: WorkerRetryPolicy::default(),
5892 heartbeat_observer: None,
5893 }
5894 }
5895
5896 pub fn worker_id(mut self, worker_id: impl Into<String>) -> Self {
5897 self.worker_id = worker_id.into();
5898 self
5899 }
5900
5901 pub fn poll_timeout(mut self, timeout: Duration) -> Self {
5902 self.poll_timeout = timeout;
5903 self
5904 }
5905
5906 pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
5907 self.heartbeat_interval = interval;
5908 self
5909 }
5910
5911 pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self {
5913 self.retry_policy = policy;
5914 self
5915 }
5916
5917 pub fn on_worker_heartbeat<F>(mut self, observer: F) -> Self
5918 where
5919 F: Fn(&WorkerHeartbeatObservation) + Send + Sync + 'static,
5920 {
5921 self.heartbeat_observer = Some(Arc::new(observer));
5922 self
5923 }
5924
5925 pub fn max_concurrent_workflow_tasks(mut self, count: usize) -> Self {
5926 self.max_concurrent_workflow_tasks = count.max(1);
5927 self
5928 }
5929
5930 pub fn max_concurrent_activity_tasks(mut self, count: usize) -> Self {
5931 self.max_concurrent_activity_tasks = count.max(1);
5932 self
5933 }
5934
5935 pub fn register_workflow<F, Fut>(&mut self, workflow_type: impl Into<String>, handler: F)
5942 where
5943 F: Fn(WorkflowContext, Value) -> Fut + Send + Sync + 'static,
5944 Fut: Future<Output = Result<Value>> + Send + 'static,
5945 {
5946 let handler = Arc::new(handler);
5947 self.workflows.insert(
5948 workflow_type.into(),
5949 RegisteredWorkflow {
5950 execute: Arc::new(move |ctx, input| {
5951 let handler = Arc::clone(&handler);
5952 Box::pin(async move {
5953 let result = handler(ctx, input.into_json()?).await?;
5954 AvroValue::from_serialize(&result)
5955 })
5956 }),
5957 replay: None,
5958 state_type: None,
5959 },
5960 );
5961 }
5962
5963 pub fn register_typed_workflow<I, O, F, Fut>(
5974 &mut self,
5975 workflow_type: impl Into<String>,
5976 handler: F,
5977 ) where
5978 I: DeserializeOwned + Send + 'static,
5979 O: Serialize + Send + 'static,
5980 F: Fn(WorkflowContext, I) -> Fut + Send + Sync + 'static,
5981 Fut: Future<Output = Result<O>> + Send + 'static,
5982 {
5983 let workflow_type = workflow_type.into();
5984 let handler_name = workflow_type.clone();
5985 let handler = Arc::new(handler);
5986 self.workflows.insert(
5987 workflow_type,
5988 RegisteredWorkflow {
5989 execute: Arc::new(move |ctx, input| {
5990 let handler = Arc::clone(&handler);
5991 let handler_name = handler_name.clone();
5992 Box::pin(async move {
5993 let input =
5994 decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
5995 let result = handler(ctx, input).await?;
5996 encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
5997 })
5998 }),
5999 replay: None,
6000 state_type: None,
6001 },
6002 );
6003 }
6004
6005 pub fn register_workflow_avro_value<F, Fut>(
6007 &mut self,
6008 workflow_type: impl Into<String>,
6009 handler: F,
6010 ) where
6011 F: Fn(WorkflowContext, AvroValue) -> Fut + Send + Sync + 'static,
6012 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6013 {
6014 self.workflows.insert(
6015 workflow_type.into(),
6016 RegisteredWorkflow {
6017 execute: Arc::new(move |ctx, input| Box::pin(handler(ctx, input))),
6018 replay: None,
6019 state_type: None,
6020 },
6021 );
6022 }
6023
6024 pub fn register_replayed_workflow<S, Factory, F, Fut>(
6032 &mut self,
6033 workflow_type: impl Into<String>,
6034 state_factory: Factory,
6035 handler: F,
6036 ) where
6037 S: Clone + Send + Sync + 'static,
6038 Factory: Fn() -> S + Send + Sync + 'static,
6039 F: Fn(WorkflowContext, Value, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
6040 Fut: Future<Output = Result<Value>> + Send + 'static,
6041 {
6042 let state_factory = Arc::new(state_factory);
6043 let handler = Arc::new(handler);
6044
6045 let execute_factory = Arc::clone(&state_factory);
6046 let execute_handler = Arc::clone(&handler);
6047 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6048 let state = WorkflowInstance::new(execute_factory());
6049 let handler = Arc::clone(&execute_handler);
6050 Box::pin(async move {
6051 let result = handler(ctx, input.into_json()?, state).await?;
6052 AvroValue::from_serialize(&result)
6053 }) as WorkflowFuture
6054 });
6055
6056 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6057 let state = WorkflowInstance::new(state_factory());
6058 let snapshot_state = state.clone();
6059 let snapshot: WorkflowStateSnapshot =
6060 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6061 let replay_handler = Arc::clone(&handler);
6062 let future = async move {
6063 let result = replay_handler(ctx, input.into_json()?, state).await?;
6064 AvroValue::from_serialize(&result)
6065 };
6066 ReplayedWorkflowInvocation {
6067 future: Box::pin(future),
6068 snapshot,
6069 }
6070 });
6071
6072 self.workflows.insert(
6073 workflow_type.into(),
6074 RegisteredWorkflow {
6075 execute,
6076 replay: Some(replay),
6077 state_type: Some(TypeId::of::<S>()),
6078 },
6079 );
6080 }
6081
6082 pub fn register_typed_replayed_workflow<I, O, S, Factory, F, Fut>(
6088 &mut self,
6089 workflow_type: impl Into<String>,
6090 state_factory: Factory,
6091 handler: F,
6092 ) where
6093 I: DeserializeOwned + Send + 'static,
6094 O: Serialize + Send + 'static,
6095 S: Clone + Send + Sync + 'static,
6096 Factory: Fn() -> S + Send + Sync + 'static,
6097 F: Fn(WorkflowContext, I, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
6098 Fut: Future<Output = Result<O>> + Send + 'static,
6099 {
6100 let workflow_type = workflow_type.into();
6101 let state_factory = Arc::new(state_factory);
6102 let handler = Arc::new(handler);
6103
6104 let execute_name = workflow_type.clone();
6105 let execute_factory = Arc::clone(&state_factory);
6106 let execute_handler = Arc::clone(&handler);
6107 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6108 let state = WorkflowInstance::new(execute_factory());
6109 let handler = Arc::clone(&execute_handler);
6110 let handler_name = execute_name.clone();
6111 Box::pin(async move {
6112 let input = decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
6113 let result = handler(ctx, input, state).await?;
6114 encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
6115 }) as WorkflowFuture
6116 });
6117
6118 let replay_name = workflow_type.clone();
6119 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6120 let state = WorkflowInstance::new(state_factory());
6121 let snapshot_state = state.clone();
6122 let snapshot: WorkflowStateSnapshot =
6123 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6124 let handler = Arc::clone(&handler);
6125 let handler_name = replay_name.clone();
6126 let future = async move {
6127 let input = decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
6128 let result = handler(ctx, input, state).await?;
6129 encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
6130 };
6131 ReplayedWorkflowInvocation {
6132 future: Box::pin(future),
6133 snapshot,
6134 }
6135 });
6136
6137 self.workflows.insert(
6138 workflow_type,
6139 RegisteredWorkflow {
6140 execute,
6141 replay: Some(replay),
6142 state_type: Some(TypeId::of::<S>()),
6143 },
6144 );
6145 }
6146
6147 pub fn register_replayed_workflow_avro_value<S, Factory, F, Fut>(
6149 &mut self,
6150 workflow_type: impl Into<String>,
6151 state_factory: Factory,
6152 handler: F,
6153 ) where
6154 S: Clone + Send + Sync + 'static,
6155 Factory: Fn() -> S + Send + Sync + 'static,
6156 F: Fn(WorkflowContext, AvroValue, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
6157 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6158 {
6159 let state_factory = Arc::new(state_factory);
6160 let handler = Arc::new(handler);
6161
6162 let execute_factory = Arc::clone(&state_factory);
6163 let execute_handler = Arc::clone(&handler);
6164 let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6165 let state = WorkflowInstance::new(execute_factory());
6166 Box::pin(execute_handler(ctx, input, state)) as WorkflowFuture
6167 });
6168
6169 let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6170 let state = WorkflowInstance::new(state_factory());
6171 let snapshot_state = state.clone();
6172 let snapshot: WorkflowStateSnapshot =
6173 Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6174 ReplayedWorkflowInvocation {
6175 future: Box::pin(handler(ctx, input, state)),
6176 snapshot,
6177 }
6178 });
6179
6180 self.workflows.insert(
6181 workflow_type.into(),
6182 RegisteredWorkflow {
6183 execute,
6184 replay: Some(replay),
6185 state_type: Some(TypeId::of::<S>()),
6186 },
6187 );
6188 }
6189
6190 pub fn register_activity<F, Fut>(&mut self, activity_type: impl Into<String>, handler: F)
6191 where
6192 F: Fn(ActivityContext, Value) -> Fut + Send + Sync + 'static,
6193 Fut: Future<Output = Result<Value>> + Send + 'static,
6194 {
6195 let handler = Arc::new(handler);
6196 self.activities.insert(
6197 activity_type.into(),
6198 Arc::new(move |ctx, args| {
6199 let handler = Arc::clone(&handler);
6200 Box::pin(async move {
6201 let result = handler(ctx, args.into_json()?).await?;
6202 AvroValue::from_serialize(&result)
6203 })
6204 }),
6205 );
6206 }
6207
6208 pub fn register_typed_activity<I, O, F, Fut>(
6214 &mut self,
6215 activity_type: impl Into<String>,
6216 handler: F,
6217 ) where
6218 I: DeserializeOwned + Send + 'static,
6219 O: Serialize + Send + 'static,
6220 F: Fn(ActivityContext, I) -> Fut + Send + Sync + 'static,
6221 Fut: Future<Output = Result<O>> + Send + 'static,
6222 {
6223 let activity_type = activity_type.into();
6224 let handler_name = activity_type.clone();
6225 let handler = Arc::new(handler);
6226 self.activities.insert(
6227 activity_type,
6228 Arc::new(move |ctx, input| {
6229 let handler = Arc::clone(&handler);
6230 let handler_name = handler_name.clone();
6231 Box::pin(async move {
6232 let input =
6233 decode_handler_input::<I>(input, HandlerKind::Activity, &handler_name)?;
6234 let result = handler(ctx, input).await?;
6235 encode_handler_result(&result, HandlerKind::Activity, &handler_name)
6236 })
6237 }),
6238 );
6239 }
6240
6241 pub fn register_activity_avro_value<F, Fut>(
6243 &mut self,
6244 activity_type: impl Into<String>,
6245 handler: F,
6246 ) where
6247 F: Fn(ActivityContext, AvroValue) -> Fut + Send + Sync + 'static,
6248 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6249 {
6250 self.activities.insert(
6251 activity_type.into(),
6252 Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
6253 );
6254 }
6255
6256 pub fn register_query<F, Fut>(
6262 &mut self,
6263 workflow_type: impl Into<String>,
6264 query_name: impl Into<String>,
6265 handler: F,
6266 ) where
6267 F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
6268 Fut: Future<Output = Result<Value>> + Send + 'static,
6269 {
6270 let handler = Arc::new(handler);
6271 self.queries
6272 .entry(workflow_type.into())
6273 .or_default()
6274 .insert(
6275 query_name.into(),
6276 RegisteredQuery::Snapshot(Arc::new(move |ctx, args| {
6277 let handler = Arc::clone(&handler);
6278 Box::pin(async move {
6279 let result = handler(ctx, args.into_json()?).await?;
6280 AvroValue::from_serialize(&result)
6281 })
6282 })),
6283 );
6284 }
6285
6286 pub fn register_query_avro_value<F, Fut>(
6288 &mut self,
6289 workflow_type: impl Into<String>,
6290 query_name: impl Into<String>,
6291 handler: F,
6292 ) where
6293 F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
6294 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6295 {
6296 self.queries
6297 .entry(workflow_type.into())
6298 .or_default()
6299 .insert(
6300 query_name.into(),
6301 RegisteredQuery::Snapshot(Arc::new(move |ctx, args| Box::pin(handler(ctx, args)))),
6302 );
6303 }
6304
6305 pub fn register_replayed_query<S, F, Fut>(
6312 &mut self,
6313 workflow_type: impl Into<String>,
6314 query_name: impl Into<String>,
6315 handler: F,
6316 ) where
6317 S: Clone + Send + Sync + 'static,
6318 F: Fn(QueryContext, Arc<S>, Value) -> Fut + Send + Sync + 'static,
6319 Fut: Future<Output = Result<Value>> + Send + 'static,
6320 {
6321 let handler = Arc::new(handler);
6322 let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
6323 let state = state.downcast::<S>().map_err(|_| {
6324 "registered query state type does not match the replayed workflow state".to_string()
6325 })?;
6326 let handler = Arc::clone(&handler);
6327 Ok(Box::pin(async move {
6328 let result = handler(ctx, state, args.into_json()?).await?;
6329 AvroValue::from_serialize(&result)
6330 }))
6331 });
6332
6333 self.queries
6334 .entry(workflow_type.into())
6335 .or_default()
6336 .insert(
6337 query_name.into(),
6338 RegisteredQuery::Replayed {
6339 state_type: TypeId::of::<S>(),
6340 handler: erased_handler,
6341 },
6342 );
6343 }
6344
6345 pub fn register_replayed_query_avro_value<S, F, Fut>(
6347 &mut self,
6348 workflow_type: impl Into<String>,
6349 query_name: impl Into<String>,
6350 handler: F,
6351 ) where
6352 S: Clone + Send + Sync + 'static,
6353 F: Fn(QueryContext, Arc<S>, AvroValue) -> Fut + Send + Sync + 'static,
6354 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6355 {
6356 let handler = Arc::new(handler);
6357 let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
6358 let state = state.downcast::<S>().map_err(|_| {
6359 "registered query state type does not match the replayed workflow state".to_string()
6360 })?;
6361 Ok(Box::pin(handler(ctx, state, args)))
6362 });
6363
6364 self.queries
6365 .entry(workflow_type.into())
6366 .or_default()
6367 .insert(
6368 query_name.into(),
6369 RegisteredQuery::Replayed {
6370 state_type: TypeId::of::<S>(),
6371 handler: erased_handler,
6372 },
6373 );
6374 }
6375
6376 pub fn register_update<F, Fut>(
6378 &mut self,
6379 workflow_type: impl Into<String>,
6380 update_name: impl Into<String>,
6381 handler: F,
6382 ) where
6383 F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
6384 Fut: Future<Output = Result<Value>> + Send + 'static,
6385 {
6386 let handler = Arc::new(handler);
6387 self.updates
6388 .entry(workflow_type.into())
6389 .or_default()
6390 .insert(
6391 update_name.into(),
6392 Arc::new(move |ctx, args| {
6393 let handler = Arc::clone(&handler);
6394 Box::pin(async move {
6395 let result = handler(ctx, args.into_json()?).await?;
6396 AvroValue::from_serialize(&result)
6397 })
6398 }),
6399 );
6400 }
6401
6402 pub fn register_update_avro_value<F, Fut>(
6404 &mut self,
6405 workflow_type: impl Into<String>,
6406 update_name: impl Into<String>,
6407 handler: F,
6408 ) where
6409 F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
6410 Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6411 {
6412 self.updates
6413 .entry(workflow_type.into())
6414 .or_default()
6415 .insert(
6416 update_name.into(),
6417 Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
6418 );
6419 }
6420
6421 pub async fn register(&self) -> Result<RegisterWorkerResponse> {
6422 let mut command_contracts = serde_json::Map::new();
6423 for workflow_type in self.workflows.keys() {
6424 let mut queries = self
6425 .queries
6426 .get(workflow_type)
6427 .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
6428 .unwrap_or_default();
6429 queries.sort();
6430 let mut updates = self
6431 .updates
6432 .get(workflow_type)
6433 .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
6434 .unwrap_or_default();
6435 updates.sort();
6436 command_contracts.insert(
6437 workflow_type.clone(),
6438 json!({
6439 "queries": queries,
6440 "query_contracts": [],
6441 "signals": [],
6442 "signal_contracts": [],
6443 "updates": updates,
6444 "update_contracts": [],
6445 "update_validators": [],
6446 }),
6447 );
6448 }
6449
6450 self.client
6451 .register_worker_with_command_contracts(
6452 &self.worker_id,
6453 &self.task_queue,
6454 self.workflows.keys().cloned().collect(),
6455 self.activities.keys().cloned().collect(),
6456 self.max_concurrent_workflow_tasks,
6457 self.max_concurrent_activity_tasks,
6458 [
6459 Some(CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY.to_string()),
6460 Some(DURABLE_SELECTION_CAPABILITY.to_string()),
6461 Some(MEMO_UPSERTS_CAPABILITY.to_string()),
6462 Some(TYPED_SEARCH_ATTRIBUTES_CAPABILITY.to_string()),
6463 (!self.queries.is_empty()).then(|| QUERY_TASKS_CAPABILITY.to_string()),
6464 (!self.updates.is_empty()).then(|| WORKFLOW_UPDATES_CAPABILITY.to_string()),
6465 worker_protocol_supports_message_streams(WORKER_PROTOCOL_VERSION)
6466 .then(|| MESSAGE_STREAMS_CAPABILITY.to_string()),
6467 ]
6468 .into_iter()
6469 .flatten()
6470 .collect(),
6471 Value::Object(command_contracts),
6472 )
6473 .await
6474 }
6475
6476 pub async fn run(&self) -> Result<()> {
6482 self.run_until(std::future::pending::<()>()).await
6483 }
6484
6485 pub async fn run_until<F>(&self, shutdown: F) -> Result<()>
6489 where
6490 F: Future<Output = ()>,
6491 {
6492 let stop = Arc::new(AtomicBool::new(false));
6493 let _stop_on_drop = StopWorkerOnDrop(Arc::clone(&stop));
6494 let worker = self.with_storage_admission(Arc::clone(&stop));
6495 let run = worker.run_with_storage_admission(Arc::clone(&stop));
6496 tokio::pin!(run);
6497 tokio::pin!(shutdown);
6498 tokio::select! {
6499 result = &mut run => result,
6500 _ = &mut shutdown => {
6501 stop.store(true, Ordering::SeqCst);
6502 run.await
6503 }
6504 }
6505 }
6506
6507 fn with_storage_admission(&self, stop: Arc<AtomicBool>) -> Self {
6508 let mut worker = self.clone();
6509 worker.client.worker_storage_admission = Some(WorkerStorageAdmission {
6510 policy: self.retry_policy,
6511 stop,
6512 });
6513 worker
6514 }
6515
6516 async fn run_with_storage_admission(&self, stop: Arc<AtomicBool>) -> Result<()> {
6517 let registration = self.register().await?;
6518 if !registration.registered {
6519 return Err(Error::WorkerLoop(format!(
6520 "worker registration for {:?} was not accepted",
6521 self.worker_id
6522 )));
6523 }
6524 let registered_worker_id = registration.worker_id.clone();
6525 let primary = self.run_registered_until(stop, registration).await;
6526 let deregistration = self
6527 .client
6528 .deregister_worker_registration(®istered_worker_id)
6529 .await;
6530
6531 match (primary, deregistration) {
6532 (Ok(()), Ok(_)) => Ok(()),
6533 (Ok(()), Err(deregistration)) => Err(deregistration),
6534 (Err(primary), Ok(_)) => Err(primary),
6535 (Err(primary), Err(deregistration)) => Err(Error::WorkerShutdown {
6536 primary: Box::new(primary),
6537 deregistration: Box::new(deregistration),
6538 }),
6539 }
6540 }
6541
6542 async fn run_registered_until(
6543 &self,
6544 stop: Arc<AtomicBool>,
6545 registration: RegisterWorkerResponse,
6546 ) -> Result<()> {
6547 let heartbeat_interval = Duration::from_secs(
6548 registration
6549 .heartbeat_interval_seconds
6550 .unwrap_or(self.heartbeat_interval.as_secs().max(1)),
6551 );
6552 let heartbeat = tokio::time::sleep(Duration::ZERO);
6558 tokio::pin!(heartbeat);
6559 let mut workflow_poller = (!self.workflows.is_empty()).then(|| {
6563 let worker = self.clone();
6564 let stop = Arc::clone(&stop);
6565 tokio::spawn(async move { worker.poll_workflows_until_stopped(stop).await })
6566 });
6567 let mut activity_poller = (!self.activities.is_empty()).then(|| {
6568 let worker = self.clone();
6569 let stop = Arc::clone(&stop);
6570 tokio::spawn(async move { worker.poll_activities_until_stopped(stop).await })
6571 });
6572 let mut query_poller = (!self.queries.is_empty()).then(|| {
6573 let worker = self.clone();
6574 let stop = Arc::clone(&stop);
6575 tokio::spawn(async move { worker.poll_queries_until_stopped(stop).await })
6576 });
6577
6578 loop {
6579 tokio::select! {
6580 _ = wait_for_worker_stop(&stop) => {
6581 stop.store(true, Ordering::SeqCst);
6582 break;
6583 }
6584 _ = &mut heartbeat => {
6585 let result = self.retry_worker_operation(|| {
6586 self.client.heartbeat_worker(
6587 &self.worker_id,
6588 self.max_concurrent_workflow_tasks,
6589 self.max_concurrent_activity_tasks,
6590 )
6591 }).await;
6592 heartbeat
6593 .as_mut()
6594 .reset(tokio::time::Instant::now() + heartbeat_interval);
6595 match result {
6596 Ok(acknowledgement) => {
6597 if let Some(observer) = &self.heartbeat_observer {
6598 observer(&WorkerHeartbeatObservation {
6599 worker_id: self.worker_id.clone(),
6600 task_queue: self.task_queue.clone(),
6601 acknowledged_at_unix_millis: SystemTime::now()
6602 .duration_since(UNIX_EPOCH)
6603 .unwrap_or_default()
6604 .as_millis()
6605 .min(u64::MAX as u128)
6606 as u64,
6607 acknowledgement,
6608 });
6609 }
6610 }
6611 Err(error) => {
6612 stop.store(true, Ordering::SeqCst);
6613 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await?;
6614 return Err(error);
6615 }
6616 }
6617 }
6618 result = OptionFuture::from(workflow_poller.as_mut()), if workflow_poller.is_some() => {
6619 workflow_poller = None;
6620 let stopped_by_server = stop.load(Ordering::SeqCst);
6621 stop.store(true, Ordering::SeqCst);
6622 let poller_result = optional_poller_result("workflow", result);
6623 let join_result =
6624 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6625 poller_result?;
6626 join_result?;
6627 if stopped_by_server {
6628 return Ok(());
6629 }
6630 return Err(Error::WorkerLoop(
6631 "workflow poller stopped unexpectedly".to_string(),
6632 ));
6633 }
6634 result = OptionFuture::from(activity_poller.as_mut()), if activity_poller.is_some() => {
6635 activity_poller = None;
6636 let stopped_by_server = stop.load(Ordering::SeqCst);
6637 stop.store(true, Ordering::SeqCst);
6638 let poller_result = optional_poller_result("activity", result);
6639 let join_result =
6640 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6641 poller_result?;
6642 join_result?;
6643 if stopped_by_server {
6644 return Ok(());
6645 }
6646 return Err(Error::WorkerLoop(
6647 "activity poller stopped unexpectedly".to_string(),
6648 ));
6649 }
6650 result = OptionFuture::from(query_poller.as_mut()), if query_poller.is_some() => {
6651 query_poller = None;
6652 let stopped_by_server = stop.load(Ordering::SeqCst);
6653 stop.store(true, Ordering::SeqCst);
6654 let poller_result = optional_poller_result("query", result);
6655 let join_result =
6656 join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6657 poller_result?;
6658 join_result?;
6659 if stopped_by_server {
6660 return Ok(());
6661 }
6662 return Err(Error::WorkerLoop(
6663 "query poller stopped unexpectedly".to_string(),
6664 ));
6665 }
6666 }
6667 }
6668
6669 join_pollers(
6670 workflow_poller.take(),
6671 activity_poller.take(),
6672 query_poller.take(),
6673 )
6674 .await
6675 }
6676
6677 pub async fn run_once(&self) -> Result<usize> {
6691 let worker = self.with_storage_admission(Arc::new(AtomicBool::new(false)));
6692 let mut handled = 0;
6693 match worker.poll_workflow_once().await? {
6694 ManagedPollOutcome::Handled => handled += 1,
6695 ManagedPollOutcome::Stop => return Ok(handled),
6696 ManagedPollOutcome::Idle => {}
6697 }
6698 match worker.poll_activity_once().await? {
6699 ManagedPollOutcome::Handled => handled += 1,
6700 ManagedPollOutcome::Stop => return Ok(handled),
6701 ManagedPollOutcome::Idle => {}
6702 }
6703 if !self.queries.is_empty() {
6704 match worker.poll_query_once().await? {
6705 ManagedPollOutcome::Handled => handled += 1,
6706 ManagedPollOutcome::Stop => return Ok(handled),
6707 ManagedPollOutcome::Idle => {}
6708 }
6709 }
6710 Ok(handled)
6711 }
6712
6713 async fn poll_workflow_once(&self) -> Result<ManagedPollOutcome> {
6714 let poll_request_id = unique_request_id("rust-workflow-poll");
6715 let response = self
6716 .retry_worker_operation(|| {
6717 self.client.poll_workflow_task_response_with_request_id(
6718 &self.worker_id,
6719 &self.task_queue,
6720 self.poll_timeout,
6721 &poll_request_id,
6722 0,
6723 )
6724 })
6725 .await;
6726 let Some(response) = self.settle_worker_poll_response(response).await? else {
6727 return Ok(ManagedPollOutcome::Idle);
6728 };
6729 if response.outcome().should_stop() {
6730 return Ok(ManagedPollOutcome::Stop);
6731 }
6732 let memo_updates_supported =
6733 runtime_supports_workflow_memo_updates(response.server_capabilities.as_ref());
6734 let Some(task) = response.task else {
6735 return Ok(ManagedPollOutcome::Idle);
6736 };
6737
6738 let task_id = task.task_id.clone();
6739 let attempt = task.workflow_task_attempt;
6740 let run_id = task.run_id.clone();
6741 let lease_owner = task
6742 .lease_owner
6743 .clone()
6744 .unwrap_or_else(|| self.worker_id.clone());
6745
6746 match self.execute_workflow_task_decision(task) {
6747 Ok(decision)
6748 if commands_use_workflow_memo_updates(&decision.commands)
6749 && !memo_updates_supported =>
6750 {
6751 self.client
6752 .fail_workflow_task(
6753 &task_id,
6754 &lease_owner,
6755 attempt,
6756 Error::WorkflowMemoUpdatesUnavailable.to_string(),
6757 )
6758 .await?;
6759 }
6760 Ok(decision) if decision.commands.is_empty() => {
6761 self.client
6767 .fail_workflow_task_with_type(
6768 &task_id,
6769 &lease_owner,
6770 attempt,
6771 WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE,
6772 WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE,
6773 )
6774 .await?;
6775 }
6776 Ok(decision) => {
6777 let completion = self
6778 .client
6779 .complete_workflow_task_with_message_streams(
6780 &task_id,
6781 &lease_owner,
6782 attempt,
6783 decision.commands,
6784 decision.message_stream_cursors,
6785 decision.message_stream_waits,
6786 )
6787 .await;
6788 if let Err(error) = completion {
6789 if !workflow_task_completion_is_terminal_timeout(
6790 &error,
6791 &task_id,
6792 attempt,
6793 run_id.as_deref(),
6794 ) {
6795 return Err(error);
6796 }
6797 }
6798 }
6799 Err(error) => {
6800 self.client
6801 .fail_workflow_task(&task_id, &lease_owner, attempt, error.to_string())
6802 .await?;
6803 }
6804 }
6805
6806 Ok(ManagedPollOutcome::Handled)
6807 }
6808
6809 async fn poll_workflows_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
6810 while !stop.load(Ordering::SeqCst) {
6811 if self.poll_workflow_once().await? == ManagedPollOutcome::Stop {
6812 stop.store(true, Ordering::SeqCst);
6813 break;
6814 }
6815 }
6816
6817 Ok(())
6818 }
6819
6820 async fn poll_activity_once(&self) -> Result<ManagedPollOutcome> {
6821 let poll_request_id = unique_request_id("rust-activity-poll");
6822 let response = self
6823 .retry_worker_operation(|| {
6824 self.client.poll_activity_task_response_with_request_id(
6825 &self.worker_id,
6826 &self.task_queue,
6827 self.poll_timeout,
6828 &poll_request_id,
6829 0,
6830 )
6831 })
6832 .await;
6833 let Some(response) = self.settle_worker_poll_response(response).await? else {
6834 return Ok(ManagedPollOutcome::Idle);
6835 };
6836 if response.outcome().should_stop() {
6837 return Ok(ManagedPollOutcome::Stop);
6838 }
6839 let Some(task) = response.task else {
6840 return Ok(ManagedPollOutcome::Idle);
6841 };
6842
6843 let task_id = task.task_id.clone();
6844 let attempt_id = task
6845 .activity_attempt_id
6846 .clone()
6847 .or(task.attempt_id.clone())
6848 .unwrap_or_default();
6849 let lease_owner = task
6850 .lease_owner
6851 .clone()
6852 .unwrap_or_else(|| self.worker_id.clone());
6853 let codec = task.payload_codec.clone();
6854 let result = self.execute_activity_task(task).await;
6855 match result {
6856 Err(error) if worker_storage_admission_body(&error).is_some() => return Err(error),
6857 Ok(value) => {
6858 let completion = self
6859 .client
6860 .complete_activity_task(&task_id, &attempt_id, &lease_owner, value, &codec)
6861 .await;
6862 if let Err(error) = completion {
6863 if !activity_task_rejection_is_final(&error) {
6864 return Err(error);
6865 }
6866 }
6867 }
6868 Err(error) => {
6869 let failure = self
6870 .client
6871 .fail_activity_task(
6872 &task_id,
6873 &attempt_id,
6874 &lease_owner,
6875 error.to_string(),
6876 false,
6877 )
6878 .await;
6879 if let Err(error) = failure {
6880 if !activity_task_rejection_is_final(&error) {
6881 return Err(error);
6882 }
6883 }
6884 }
6885 }
6886
6887 Ok(ManagedPollOutcome::Handled)
6888 }
6889
6890 async fn poll_activities_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
6891 while !stop.load(Ordering::SeqCst) {
6892 if self.poll_activity_once().await? == ManagedPollOutcome::Stop {
6893 stop.store(true, Ordering::SeqCst);
6894 break;
6895 }
6896 }
6897
6898 Ok(())
6899 }
6900
6901 async fn poll_query_once(&self) -> Result<ManagedPollOutcome> {
6902 let poll_request_id = unique_request_id("rust-query-poll");
6903 let response = self
6904 .retry_worker_operation(|| {
6905 self.client.poll_query_task_response_with_request_id(
6906 &self.worker_id,
6907 &self.task_queue,
6908 self.poll_timeout,
6909 &poll_request_id,
6910 0,
6911 )
6912 })
6913 .await;
6914 let Some(response) = self.settle_worker_poll_response(response).await? else {
6915 return Ok(ManagedPollOutcome::Idle);
6916 };
6917 if response.outcome().should_stop() {
6918 return Ok(ManagedPollOutcome::Stop);
6919 }
6920 let Some(task) = response.task else {
6921 return Ok(ManagedPollOutcome::Idle);
6922 };
6923
6924 let query_task_id = task.query_task_id.clone();
6925 let attempt = task.query_task_attempt;
6926 let lease_owner = task
6927 .lease_owner
6928 .clone()
6929 .unwrap_or_else(|| self.worker_id.clone());
6930 let codec = task.payload_codec.clone();
6931
6932 match self.execute_query_task(task).await {
6933 Ok(value) => {
6934 let result_envelope = match encode_typed_envelope(&value, &codec) {
6935 Ok(result_envelope) => result_envelope,
6936 Err(error) => {
6937 let failure = self
6938 .client
6939 .fail_query_task(
6940 &query_task_id,
6941 &lease_owner,
6942 attempt,
6943 error.to_string(),
6944 "query_result_encode_failed",
6945 "QueryResultEncodeFailed",
6946 )
6947 .await;
6948 if let Err(error) = failure {
6949 if !query_task_rejection_is_final(&error) {
6950 return Err(error);
6951 }
6952 }
6953 return Ok(ManagedPollOutcome::Handled);
6954 }
6955 };
6956
6957 if let Err(error) = self
6958 .client
6959 .complete_query_task_with_envelope(
6960 &query_task_id,
6961 &lease_owner,
6962 attempt,
6963 value.clone().into_json()?,
6964 result_envelope,
6965 )
6966 .await
6967 {
6968 if !query_task_rejection_is_final(&error) {
6969 return Err(error);
6970 }
6971 }
6972 }
6973 Err(failure) => {
6974 let result = self
6975 .client
6976 .fail_query_task(
6977 &query_task_id,
6978 &lease_owner,
6979 attempt,
6980 failure.message,
6981 failure.reason,
6982 failure.failure_type,
6983 )
6984 .await;
6985 if let Err(error) = result {
6986 if !query_task_rejection_is_final(&error) {
6987 return Err(error);
6988 }
6989 }
6990 }
6991 }
6992
6993 Ok(ManagedPollOutcome::Handled)
6994 }
6995
6996 async fn poll_queries_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
6997 while !stop.load(Ordering::SeqCst) {
6998 if self.poll_query_once().await? == ManagedPollOutcome::Stop {
6999 stop.store(true, Ordering::SeqCst);
7000 break;
7001 }
7002 }
7003
7004 Ok(())
7005 }
7006
7007 async fn retry_worker_operation<T, F, Fut>(&self, mut operation: F) -> Result<T>
7008 where
7009 F: FnMut() -> Fut,
7010 Fut: Future<Output = Result<T>>,
7011 {
7012 let mut retries = 0;
7013
7014 loop {
7015 match operation().await {
7016 Err(error)
7017 if worker_operation_is_retryable(&error)
7018 && retries < self.retry_policy.max_retries =>
7019 {
7020 retries += 1;
7021 tokio::time::sleep(worker_retry_delay(self.retry_policy, retries)).await;
7022 }
7023 result => return result,
7024 }
7025 }
7026 }
7027
7028 async fn settle_worker_poll_response<T>(&self, response: Result<T>) -> Result<Option<T>> {
7029 match response {
7030 Ok(response) => Ok(Some(response)),
7031 Err(error) => {
7032 let Some(advertised_delay) = worker_poll_capacity_retry_after(&error) else {
7033 return Err(error);
7034 };
7035 let minimum_delay = self
7036 .retry_policy
7037 .initial_backoff
7038 .max(Duration::from_millis(1));
7039 let maximum_delay = self.retry_policy.max_backoff.max(minimum_delay);
7040 tokio::time::sleep(advertised_delay.max(minimum_delay).min(maximum_delay)).await;
7041 Ok(None)
7042 }
7043 }
7044 }
7045
7046 async fn execute_query_task(
7047 &self,
7048 mut task: QueryTask,
7049 ) -> std::result::Result<AvroValue, QueryTaskExecutionFailure> {
7050 validate_query_task_payloads(&task).map_err(|error| {
7051 QueryTaskExecutionFailure::new(
7052 "query_payload_decode_failed",
7053 error.to_string(),
7054 "QueryPayloadDecodeFailed",
7055 )
7056 })?;
7057
7058 if !self.workflows.contains_key(&task.workflow_type) {
7059 return Err(QueryTaskExecutionFailure::new(
7060 "query_workflow_type_not_registered",
7061 format!("no workflow registered for type {:?}", task.workflow_type),
7062 "WorkflowTypeNotRegistered",
7063 ));
7064 }
7065
7066 let Some(handlers) = self.queries.get(&task.workflow_type) else {
7067 return Err(QueryTaskExecutionFailure::new(
7068 "query_handler_unavailable",
7069 format!(
7070 "query handlers are unavailable for workflow type {:?}",
7071 task.workflow_type
7072 ),
7073 "QueryHandlerUnavailable",
7074 ));
7075 };
7076 let Some(query) = handlers.get(&task.query_name) else {
7077 return Err(QueryTaskExecutionFailure::new(
7078 "rejected_unknown_query",
7079 format!("unknown query {:?}", task.query_name),
7080 "QueryFailed",
7081 ));
7082 };
7083
7084 let args = decode_task_avro_arguments(task.query_arguments.as_ref(), &task.payload_codec)
7085 .map_err(|error| {
7086 QueryTaskExecutionFailure::new(
7087 "query_payload_decode_failed",
7088 format!("cannot decode query arguments: {error}"),
7089 "QueryPayloadDecodeFailed",
7090 )
7091 })?;
7092 let workflow_input_typed =
7093 decode_task_avro_arguments(task.workflow_arguments.as_ref(), &task.payload_codec)
7094 .map_err(|error| {
7095 QueryTaskExecutionFailure::new(
7096 "query_workflow_state_unavailable",
7097 format!("cannot decode workflow start input: {error}"),
7098 "QueryWorkflowStateUnavailable",
7099 )
7100 })?;
7101 let workflow_input = workflow_input_typed.clone().into_json().map_err(|error| {
7102 QueryTaskExecutionFailure::new(
7103 "query_workflow_state_unavailable",
7104 format!("cannot project workflow start input: {error}"),
7105 "QueryWorkflowStateUnavailable",
7106 )
7107 })?;
7108 hydrate_query_history_from_export(&mut task).map_err(|error| {
7109 QueryTaskExecutionFailure::new(
7110 "query_workflow_state_unavailable",
7111 format!("cannot restore query history snapshot: {error}"),
7112 "QueryWorkflowStateUnavailable",
7113 )
7114 })?;
7115 enrich_query_history_from_export(&mut task).map_err(|error| {
7116 QueryTaskExecutionFailure::new(
7117 "query_workflow_state_unavailable",
7118 format!("cannot restore compact query history payloads: {error}"),
7119 "QueryWorkflowStateUnavailable",
7120 )
7121 })?;
7122 let signal_events = query_signal_events(&task).map_err(|error| {
7123 QueryTaskExecutionFailure::new(
7124 "query_workflow_state_unavailable",
7125 format!("cannot decode committed workflow signals: {error}"),
7126 "QueryWorkflowStateUnavailable",
7127 )
7128 })?;
7129 let history_events = Arc::new(std::mem::take(&mut task.history_events));
7130 let context = QueryContext {
7131 workflow_id: task.workflow_id,
7132 run_id: task.run_id,
7133 workflow_type: task.workflow_type.clone(),
7134 run_status: task.run_status,
7135 workflow_input,
7136 workflow_input_avro_value: workflow_input_typed.clone(),
7137 history_events: Arc::clone(&history_events),
7138 signal_events: Arc::new(signal_events),
7139 };
7140
7141 let future = match query {
7142 RegisteredQuery::Snapshot(handler) => handler(context, args),
7143 RegisteredQuery::Replayed {
7144 state_type,
7145 handler,
7146 } => {
7147 let workflow = self
7148 .workflows
7149 .get(&task.workflow_type)
7150 .expect("workflow registration was checked above");
7151 if workflow.state_type != Some(*state_type) {
7152 return Err(QueryTaskExecutionFailure::new(
7153 "query_workflow_state_unavailable",
7154 "replayed query state type does not match its workflow registration",
7155 "QueryWorkflowStateUnavailable",
7156 ));
7157 }
7158 let replay = workflow.replay.as_ref().ok_or_else(|| {
7159 QueryTaskExecutionFailure::new(
7160 "query_workflow_state_unavailable",
7161 format!(
7162 "workflow type {:?} is not registered for instance-state replay",
7163 task.workflow_type
7164 ),
7165 "QueryWorkflowStateUnavailable",
7166 )
7167 })?;
7168 let workflow_state = Arc::new(Mutex::new(
7169 WorkflowState::new_with_identity(
7170 history_events.as_ref().clone(),
7171 context.workflow_id.clone(),
7172 context.run_id.clone(),
7173 self.task_queue.clone(),
7174 task.payload_codec,
7175 None,
7176 )
7177 .map_err(|error| {
7178 QueryTaskExecutionFailure::new(
7179 "query_workflow_state_unavailable",
7180 format!("workflow replay failed before query: {error}"),
7181 "QueryWorkflowStateUnavailable",
7182 )
7183 })?,
7184 ));
7185 let workflow_context = WorkflowContext {
7186 state: workflow_state,
7187 };
7188 let mut invocation = replay(workflow_context.clone(), workflow_input_typed.clone());
7189 let mut cx = TaskContext::from_waker(noop_waker_ref());
7190 match invocation.future.as_mut().poll(&mut cx) {
7191 Poll::Ready(Ok(_)) => {
7192 workflow_context
7193 .ensure_history_consumed()
7194 .map_err(|error| {
7195 QueryTaskExecutionFailure::new(
7196 "query_workflow_state_unavailable",
7197 format!("workflow replay failed before query: {error}"),
7198 "QueryWorkflowStateUnavailable",
7199 )
7200 })?;
7201 }
7202 Poll::Ready(Err(error)) => {
7203 return Err(QueryTaskExecutionFailure::new(
7204 "query_workflow_state_unavailable",
7205 format!("workflow replay failed before query: {error}"),
7206 "QueryWorkflowStateUnavailable",
7207 ));
7208 }
7209 Poll::Pending => {
7210 let commands = workflow_context.take_commands().map_err(|error| {
7211 QueryTaskExecutionFailure::new(
7212 "query_workflow_state_unavailable",
7213 format!("workflow replay failed before query: {error}"),
7214 "QueryWorkflowStateUnavailable",
7215 )
7216 })?;
7217 if commands.is_empty()
7218 && !workflow_context
7219 .matched_recorded_pending()
7220 .map_err(|error| {
7221 QueryTaskExecutionFailure::new(
7222 "query_workflow_state_unavailable",
7223 format!("workflow replay failed before query: {error}"),
7224 "QueryWorkflowStateUnavailable",
7225 )
7226 })?
7227 {
7228 return Err(QueryTaskExecutionFailure::new(
7229 "query_workflow_state_unavailable",
7230 "workflow replay yielded without a durable command",
7231 "QueryWorkflowStateUnavailable",
7232 ));
7233 }
7234 }
7235 }
7236 let state = (invocation.snapshot)().map_err(|error| {
7237 QueryTaskExecutionFailure::new(
7238 "query_workflow_state_unavailable",
7239 format!("cannot snapshot replayed workflow state: {error}"),
7240 "QueryWorkflowStateUnavailable",
7241 )
7242 })?;
7243 handler(context, state, args).map_err(|message| {
7244 QueryTaskExecutionFailure::new(
7245 "query_workflow_state_unavailable",
7246 message,
7247 "QueryWorkflowStateUnavailable",
7248 )
7249 })?
7250 }
7251 };
7252
7253 future.await.map_err(|error| {
7254 QueryTaskExecutionFailure::new("query_rejected", error.to_string(), "QueryFailed")
7255 })
7256 }
7257
7258 #[cfg(test)]
7259 fn execute_workflow_task(&self, task: WorkflowTask) -> Result<Vec<Value>> {
7260 Ok(self.execute_workflow_task_decision(task)?.commands)
7261 }
7262
7263 fn execute_workflow_task_decision(&self, task: WorkflowTask) -> Result<WorkflowTaskDecision> {
7264 validate_workflow_task_payloads(&task)?;
7265
7266 if let Some(update_id) = task
7267 .workflow_update_id
7268 .as_deref()
7269 .filter(|update_id| !update_id.is_empty())
7270 {
7271 return self
7272 .execute_update_task(&task, update_id)
7273 .map(WorkflowTaskDecision::without_message_streams);
7274 }
7275
7276 let workflow = self
7277 .workflows
7278 .get(&task.workflow_type)
7279 .ok_or_else(|| Error::WorkflowNotRegistered(task.workflow_type.clone()))?;
7280 let input = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
7281 let resume_signal = decode_resume_signal(&task)?;
7282 let history_budget = WorkflowHistoryBudget {
7283 event_count: task
7284 .total_history_events
7285 .unwrap_or_else(|| u64::try_from(task.history_events.len()).unwrap_or(u64::MAX)),
7286 size_bytes: task.history_size_bytes,
7287 continue_as_new_recommended: task.continue_as_new_recommended.unwrap_or(false),
7288 pressure: task.history_budget_pressure.clone(),
7289 };
7290 let workflow_command_identity = task
7291 .workflow_command_id
7292 .clone()
7293 .filter(|identity| !identity.is_empty())
7294 .unwrap_or_default();
7295 let mut workflow_state = WorkflowState::new_with_identity(
7296 task.history_events,
7297 task.workflow_id,
7298 task.run_id,
7299 self.task_queue.clone(),
7300 task.payload_codec.clone(),
7301 resume_signal,
7302 )?;
7303 workflow_state.history_budget = history_budget;
7304 workflow_state.workflow_command_identity = workflow_command_identity;
7305 workflow_state.cancel_requested = task.cancel_requested;
7306 let state = Arc::new(Mutex::new(workflow_state));
7307 let ctx = WorkflowContext { state };
7308 let mut future = (workflow.execute)(ctx.clone(), input);
7309 let mut cx = TaskContext::from_waker(noop_waker_ref());
7310
7311 match future.as_mut().poll(&mut cx) {
7312 Poll::Ready(Ok(result)) => {
7313 ctx.ensure_history_consumed()?;
7314 let result = encode_typed_envelope(&result, &task.payload_codec)?;
7315 let mut commands = ctx.take_commands()?;
7316 commands.push(json!({
7317 "type": "complete_workflow",
7318 "result": result
7319 }));
7320 self.message_stream_decision(&ctx, commands)
7321 }
7322 Poll::Ready(Err(error)) => {
7323 if let Error::ContinueAsNew(request) = error {
7324 let mut commands = ctx.take_commands()?;
7325 if let Some(command) = ctx.continue_as_new_command(request)? {
7326 commands.push(command);
7327 }
7328 ctx.ensure_history_consumed()?;
7329 return self.message_stream_decision(&ctx, commands);
7330 }
7331 if workflow_task_integrity_error(&error) {
7332 return Err(error);
7337 }
7338 ctx.ensure_history_consumed()?;
7341 let mut commands = ctx.take_commands()?;
7342 commands.push(workflow_failure_command(&error));
7343 self.message_stream_decision(&ctx, commands)
7344 }
7345 Poll::Pending => {
7346 let commands = ctx.take_commands()?;
7347 if commands.is_empty() && !ctx.matched_recorded_pending()? {
7348 Err(Error::WorkflowYieldedWithoutCommand)
7349 } else {
7350 self.message_stream_decision(&ctx, commands)
7351 }
7352 }
7353 }
7354 }
7355
7356 fn message_stream_decision(
7357 &self,
7358 ctx: &WorkflowContext,
7359 commands: Vec<Value>,
7360 ) -> Result<WorkflowTaskDecision> {
7361 let (message_stream_cursors, message_stream_waits) = ctx.message_stream_metadata()?;
7362 Ok(WorkflowTaskDecision {
7363 commands,
7364 message_stream_cursors,
7365 message_stream_waits,
7366 })
7367 }
7368
7369 fn execute_update_task(&self, task: &WorkflowTask, update_id: &str) -> Result<Vec<Value>> {
7370 if !self.workflows.contains_key(&task.workflow_type) {
7371 return Err(Error::WorkflowNotRegistered(task.workflow_type.clone()));
7372 }
7373
7374 let accepted = task.history_events.iter().rev().find_map(|event| {
7375 (event.event_type == "UpdateAccepted"
7376 && event.payload.get("update_id").and_then(Value::as_str) == Some(update_id))
7377 .then_some(&event.payload)
7378 });
7379 let update_name = accepted
7380 .and_then(|payload| payload.get("update_name"))
7381 .and_then(Value::as_str)
7382 .or(task.update_name.as_deref())
7383 .unwrap_or_default();
7384 let Some(handler) = self
7385 .updates
7386 .get(&task.workflow_type)
7387 .and_then(|handlers| handlers.get(update_name))
7388 else {
7389 return Ok(vec![json!({
7390 "type": "fail_update",
7391 "update_id": update_id,
7392 "message": format!(
7393 "no update handler is registered for {}.{update_name}",
7394 task.workflow_type
7395 ),
7396 "exception_type": "UnknownUpdate",
7397 "non_retryable": true,
7398 })]);
7399 };
7400 let arguments = accepted
7401 .and_then(|payload| payload.get("arguments"))
7402 .or(task.arguments.as_ref());
7403 let arguments = decode_task_avro_arguments(arguments, &task.payload_codec)?;
7404 let context = QueryContext {
7405 workflow_id: task.workflow_id.clone(),
7406 run_id: task.run_id.clone(),
7407 workflow_type: task.workflow_type.clone(),
7408 run_status: Some("running".to_string()),
7409 workflow_input: Value::Null,
7410 workflow_input_avro_value: AvroValue::Null,
7411 history_events: Arc::new(task.history_events.clone()),
7412 signal_events: Arc::new(Vec::new()),
7413 };
7414 let mut future = handler(context, arguments);
7415 let mut cx = TaskContext::from_waker(noop_waker_ref());
7416
7417 match future.as_mut().poll(&mut cx) {
7418 Poll::Ready(Ok(result)) => Ok(vec![json!({
7419 "type": "complete_update",
7420 "update_id": update_id,
7421 "result": encode_typed_envelope(&result, &task.payload_codec)?,
7422 })]),
7423 Poll::Ready(Err(error)) => Ok(vec![json!({
7424 "type": "fail_update",
7425 "update_id": update_id,
7426 "message": error.to_string(),
7427 "exception_type": "UpdateFailed",
7428 "non_retryable": true,
7429 })]),
7430 Poll::Pending => Err(Error::WorkflowYieldedWithoutCommand),
7431 }
7432 }
7433
7434 async fn execute_activity_task(&self, task: ActivityTask) -> Result<AvroValue> {
7435 validate_activity_task_payloads(&task)?;
7436
7437 let handler = self
7438 .activities
7439 .get(&task.activity_type)
7440 .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?;
7441 let args = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
7442 let attempt_id = task
7443 .activity_attempt_id
7444 .clone()
7445 .or(task.attempt_id.clone())
7446 .unwrap_or_default();
7447 let lease_owner = task
7448 .lease_owner
7449 .clone()
7450 .unwrap_or_else(|| self.worker_id.clone());
7451 let ctx = ActivityContext {
7452 client: self.client.clone(),
7453 task_id: task.task_id,
7454 activity_attempt_id: attempt_id,
7455 lease_owner,
7456 activity_type: task.activity_type,
7457 attempt_number: task.attempt_number,
7458 task_queue: self.task_queue.clone(),
7459 worker_id: self.worker_id.clone(),
7460 };
7461
7462 handler(ctx, args).await
7463 }
7464}
7465
7466fn poller_result(
7467 kind: &str,
7468 result: std::result::Result<Result<()>, tokio::task::JoinError>,
7469) -> Result<()> {
7470 match result {
7471 Ok(result) => result,
7472 Err(error) => Err(Error::WorkerLoop(format!(
7473 "{kind} poller join error: {error}"
7474 ))),
7475 }
7476}
7477
7478fn optional_poller_result(
7479 kind: &str,
7480 result: Option<std::result::Result<Result<()>, tokio::task::JoinError>>,
7481) -> Result<()> {
7482 match result {
7483 Some(result) => poller_result(kind, result),
7484 None => Ok(()),
7485 }
7486}
7487
7488async fn join_pollers(
7489 workflow_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7490 activity_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7491 query_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7492) -> Result<()> {
7493 let mut first_error = None;
7494
7495 if let Some(handle) = workflow_poller {
7496 if let Err(error) = poller_result("workflow", handle.await) {
7497 first_error.get_or_insert(error);
7498 }
7499 }
7500
7501 if let Some(handle) = activity_poller {
7502 if let Err(error) = poller_result("activity", handle.await) {
7503 first_error.get_or_insert(error);
7504 }
7505 }
7506
7507 if let Some(handle) = query_poller {
7508 if let Err(error) = poller_result("query", handle.await) {
7509 first_error.get_or_insert(error);
7510 }
7511 }
7512
7513 if let Some(error) = first_error {
7514 Err(error)
7515 } else {
7516 Ok(())
7517 }
7518}
7519
7520fn default_worker_id() -> String {
7521 let millis = SystemTime::now()
7522 .duration_since(UNIX_EPOCH)
7523 .unwrap_or_default()
7524 .as_millis();
7525 format!("rust-worker-{}-{millis}", std::process::id())
7526}
7527
7528fn percent_encode_path_segment(segment: &str) -> String {
7529 const HEX: &[u8; 16] = b"0123456789ABCDEF";
7530 let mut encoded = String::with_capacity(segment.len());
7531
7532 for byte in segment.bytes() {
7533 if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') {
7534 encoded.push(char::from(byte));
7535 } else {
7536 encoded.push('%');
7537 encoded.push(char::from(HEX[(byte >> 4) as usize]));
7538 encoded.push(char::from(HEX[(byte & 0x0f) as usize]));
7539 }
7540 }
7541
7542 encoded
7543}
7544
7545fn unique_request_id(prefix: &str) -> String {
7546 let nanos = SystemTime::now()
7547 .duration_since(UNIX_EPOCH)
7548 .unwrap_or_default()
7549 .as_nanos();
7550 format!("{prefix}-{}-{nanos}", std::process::id())
7551}
7552
7553#[derive(Debug)]
7554struct QueryTaskExecutionFailure {
7555 reason: String,
7556 message: String,
7557 failure_type: String,
7558}
7559
7560impl QueryTaskExecutionFailure {
7561 fn new(
7562 reason: impl Into<String>,
7563 message: impl Into<String>,
7564 failure_type: impl Into<String>,
7565 ) -> Self {
7566 Self {
7567 reason: reason.into(),
7568 message: message.into(),
7569 failure_type: failure_type.into(),
7570 }
7571 }
7572}
7573
7574#[derive(Clone, Debug)]
7580pub struct WorkflowInstance<S> {
7581 state: Arc<Mutex<S>>,
7582}
7583
7584impl<S> WorkflowInstance<S> {
7585 fn new(state: S) -> Self {
7586 Self {
7587 state: Arc::new(Mutex::new(state)),
7588 }
7589 }
7590
7591 pub fn read<R>(&self, reader: impl FnOnce(&S) -> R) -> Result<R> {
7593 let state = self
7594 .state
7595 .lock()
7596 .map_err(|_| Error::WorkflowStatePoisoned)?;
7597 Ok(reader(&state))
7598 }
7599
7600 pub fn update<R>(&self, transition: impl FnOnce(&mut S) -> R) -> Result<R> {
7602 let mut state = self
7603 .state
7604 .lock()
7605 .map_err(|_| Error::WorkflowStatePoisoned)?;
7606 Ok(transition(&mut state))
7607 }
7608}
7609
7610impl<S: Clone> WorkflowInstance<S> {
7611 fn snapshot(&self) -> Result<S> {
7612 self.read(Clone::clone)
7613 }
7614}
7615
7616#[derive(Clone, Debug, PartialEq)]
7617pub struct MessageStreamMessage {
7618 pub stream_name: String,
7619 pub message_id: String,
7620 pub position: u64,
7621 pub arguments: Vec<AvroValue>,
7622}
7623
7624#[derive(Clone, Debug)]
7625pub struct MessageStream {
7626 ctx: WorkflowContext,
7627 name: String,
7628}
7629
7630impl MessageStream {
7631 pub async fn receive(&self, max_items: usize) -> Result<Vec<MessageStreamMessage>> {
7633 if !(1..=MESSAGE_STREAM_MAX_BATCH).contains(&max_items) {
7634 return Err(Error::Codec(format!(
7635 "message stream max_items must be between 1 and {MESSAGE_STREAM_MAX_BATCH}"
7636 )));
7637 }
7638 loop {
7639 if let Some(batch) = self.ctx.take_message_stream_batch(&self.name, max_items)? {
7640 return Ok(batch);
7641 }
7642
7643 self.ctx.record_message_stream_wait(&self.name)?;
7644 let replay_wait_sequence = self.ctx.next_message_stream_wait_sequence()?;
7645 let arguments = self.ctx.wait_runtime_signal(MESSAGE_STREAM_SIGNAL).await?;
7646 self.ctx.buffer_message_stream_delivery(arguments)?;
7647 if let Some(sequence) = replay_wait_sequence {
7648 self.ctx.buffer_message_stream_history_for_wait(sequence)?;
7649 }
7650 }
7651 }
7652
7653 pub async fn receive_one(&self) -> Result<MessageStreamMessage> {
7654 self.receive(1)
7655 .await?
7656 .into_iter()
7657 .next()
7658 .ok_or_else(|| Error::Codec("message stream resumed without a message".to_string()))
7659 }
7660}
7661
7662#[derive(Clone, Debug)]
7663pub struct WorkflowContext {
7664 state: Arc<Mutex<WorkflowState>>,
7665}
7666
7667fn valid_memo_key(key: &str) -> bool {
7668 let numeric_candidate = key.strip_prefix('-').unwrap_or(key);
7669
7670 !key.is_empty()
7671 && key.len() <= 64
7672 && (numeric_candidate.is_empty()
7673 || !numeric_candidate.bytes().all(|byte| byte.is_ascii_digit()))
7674 && key
7675 .bytes()
7676 .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b':' | b'-'))
7677}
7678
7679fn avro_encoded_size(value: &AvroValue) -> Result<usize> {
7680 BASE64
7681 .decode(encode_avro_value(value)?.blob)
7682 .map(|bytes| bytes.len())
7683 .map_err(|error| Error::Codec(format!("memo Avro encoding was not strict base64: {error}")))
7684}
7685
7686fn canonical_memo_entries(value: AvroValue, require_entries: bool) -> Result<AvroValue> {
7687 let AvroValue::Map(entries) = value else {
7688 return Err(Error::InvalidMemoUpdate(
7689 "entries must serialize to an Avro string-keyed map".to_string(),
7690 ));
7691 };
7692 if require_entries && entries.is_empty() {
7693 return Err(Error::InvalidMemoUpdate(
7694 "at least one entry is required".to_string(),
7695 ));
7696 }
7697 if entries.len() > MAX_MEMO_ENTRIES {
7698 return Err(Error::InvalidMemoUpdate(format!(
7699 "at most {MAX_MEMO_ENTRIES} entries are allowed"
7700 )));
7701 }
7702
7703 for (key, value) in &entries {
7704 if !valid_memo_key(&key) {
7705 return Err(Error::InvalidMemoUpdate(
7706 "keys must match ^(?!-?[0-9]+$)[A-Za-z0-9_.:-]{1,64}$".to_string(),
7707 ));
7708 }
7709 if avro_encoded_size(value)? > MAX_MEMO_VALUE_SIZE_BYTES {
7710 return Err(Error::InvalidMemoUpdate(format!(
7711 "value {key:?} exceeds the {MAX_MEMO_VALUE_SIZE_BYTES}-byte limit"
7712 )));
7713 }
7714 }
7715
7716 let value = AvroValue::Map(entries);
7717 if avro_encoded_size(&value)? > MAX_MEMO_TOTAL_SIZE_BYTES {
7718 return Err(Error::InvalidMemoUpdate(format!(
7719 "update exceeds the {MAX_MEMO_TOTAL_SIZE_BYTES}-byte total limit"
7720 )));
7721 }
7722 Ok(value)
7723}
7724
7725fn decode_memo_history_map(envelope: &Value, require_entries: bool) -> Result<AvroValue> {
7726 let object = envelope.as_object().ok_or_else(|| {
7727 Error::InvalidMemoUpdate(
7728 "history field must use the public {codec, blob} payload envelope".to_string(),
7729 )
7730 })?;
7731 if object.len() != 2 || !object.contains_key("codec") || !object.contains_key("blob") {
7732 return Err(Error::InvalidMemoUpdate(
7733 "history field must use exactly the public {codec, blob} payload envelope".to_string(),
7734 ));
7735 }
7736
7737 canonical_memo_entries(
7738 decode_wire_avro_value(envelope, DEFAULT_CODEC)?,
7739 require_entries,
7740 )
7741}
7742
7743impl WorkflowContext {
7744 pub fn message_stream(&self, name: impl Into<String>) -> Result<MessageStream> {
7745 let name = name.into();
7746 if name.is_empty()
7747 || name.len() > 128
7748 || !name.bytes().all(|byte| {
7749 byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b':' | b'-')
7750 })
7751 {
7752 return Err(Error::Codec(
7753 "message stream names must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens"
7754 .to_string(),
7755 ));
7756 }
7757 Ok(MessageStream {
7758 ctx: self.clone(),
7759 name,
7760 })
7761 }
7762
7763 fn record_message_stream_wait(&self, name: &str) -> Result<()> {
7764 let mut state = self
7765 .state
7766 .lock()
7767 .map_err(|_| Error::WorkflowStatePoisoned)?;
7768 let position = state.message_stream_cursors.get(name).copied().unwrap_or(0);
7769 state
7770 .message_stream_waits
7771 .insert(name.to_string(), position);
7772 Ok(())
7773 }
7774
7775 fn buffer_message_stream(&self, message: MessageStreamMessage) -> Result<()> {
7776 let mut state = self
7777 .state
7778 .lock()
7779 .map_err(|_| Error::WorkflowStatePoisoned)?;
7780 let cursor = state
7781 .message_stream_cursors
7782 .get(&message.stream_name)
7783 .copied()
7784 .unwrap_or(0);
7785 if message.position <= cursor {
7786 return Ok(());
7787 }
7788 let pending = state
7789 .message_stream_messages
7790 .entry(message.stream_name.clone())
7791 .or_default();
7792 if pending.iter().any(|candidate| {
7793 candidate.position == message.position || candidate.message_id == message.message_id
7794 }) {
7795 return Ok(());
7796 }
7797 pending.push(message);
7798 pending.sort_by_key(|candidate| candidate.position);
7799 Ok(())
7800 }
7801
7802 fn buffer_message_stream_delivery(&self, arguments: Vec<Value>) -> Result<Option<String>> {
7803 if let Some(delivery) = decode_message_stream_delivery(arguments)? {
7804 match delivery {
7805 MessageStreamDelivery::Message(message) => {
7806 let stream_name = message.stream_name.clone();
7807 self.buffer_message_stream(message)?;
7808 return Ok(Some(stream_name));
7809 }
7810 MessageStreamDelivery::Cursor {
7811 stream_name,
7812 through_position,
7813 } => self.apply_message_stream_cursor(&stream_name, through_position)?,
7814 }
7815 }
7816 Ok(None)
7817 }
7818
7819 fn next_message_stream_wait_sequence(&self) -> Result<Option<u64>> {
7820 let state = self
7821 .state
7822 .lock()
7823 .map_err(|_| Error::WorkflowStatePoisoned)?;
7824 Ok(match state.recorded_commands.get(state.command_cursor) {
7825 Some(RecordedCommand::SignalWait {
7826 sequence,
7827 signal_name,
7828 ..
7829 }) if signal_name == MESSAGE_STREAM_SIGNAL => Some(*sequence),
7830 _ => None,
7831 })
7832 }
7833
7834 fn buffer_message_stream_history_for_wait(&self, wait_sequence: u64) -> Result<()> {
7835 let (history, payload_codec) = {
7836 let state = self
7837 .state
7838 .lock()
7839 .map_err(|_| Error::WorkflowStatePoisoned)?;
7840 (
7841 Arc::clone(&state.history_events),
7842 state.payload_codec.clone(),
7843 )
7844 };
7845
7846 let Some(opened_index) = history.iter().position(|event| {
7847 event.event_type == "SignalWaitOpened"
7848 && durable_event_sequence(event) == Some(wait_sequence)
7849 && event.payload.get("signal_name").and_then(Value::as_str)
7850 == Some(MESSAGE_STREAM_SIGNAL)
7851 }) else {
7852 return Ok(());
7853 };
7854 let boundary_index = history
7855 .iter()
7856 .enumerate()
7857 .skip(opened_index + 1)
7858 .find_map(|(index, event)| {
7859 (durable_event_sequence(event).is_some_and(|sequence| sequence > wait_sequence)
7860 && is_authored_command_open_event(event))
7861 .then_some(index)
7862 })
7863 .unwrap_or(history.len());
7864
7865 for event in history[opened_index + 1..boundary_index]
7866 .iter()
7867 .filter(|event| {
7868 event.event_type == "SignalReceived"
7869 && event.payload.get("signal_name").and_then(Value::as_str)
7870 == Some(MESSAGE_STREAM_SIGNAL)
7871 })
7872 {
7873 let arguments = decode_signal_event_arguments(event, &payload_codec)?
7874 .into_iter()
7875 .map(AvroValue::into_json)
7876 .collect::<Result<Vec<_>>>()?;
7877 self.buffer_message_stream_delivery(arguments)?;
7878 }
7879 Ok(())
7880 }
7881
7882 fn apply_message_stream_cursor(&self, name: &str, through_position: u64) -> Result<()> {
7883 let mut state = self
7884 .state
7885 .lock()
7886 .map_err(|_| Error::WorkflowStatePoisoned)?;
7887 let cursor = state
7888 .message_stream_cursors
7889 .entry(name.to_string())
7890 .or_default();
7891 *cursor = (*cursor).max(through_position);
7892 if let Some(pending) = state.message_stream_messages.get_mut(name) {
7893 pending.retain(|message| message.position > through_position);
7894 }
7895 Ok(())
7896 }
7897
7898 fn take_message_stream_batch(
7899 &self,
7900 name: &str,
7901 max_items: usize,
7902 ) -> Result<Option<Vec<MessageStreamMessage>>> {
7903 let mut state = self
7904 .state
7905 .lock()
7906 .map_err(|_| Error::WorkflowStatePoisoned)?;
7907 let cursor = state.message_stream_cursors.get(name).copied().unwrap_or(0);
7908 let pending = state
7909 .message_stream_messages
7910 .entry(name.to_string())
7911 .or_default();
7912 let count = contiguous_message_stream_count(pending, cursor, max_items);
7913 if count == 0 {
7914 return Ok(None);
7915 }
7916 let batch = pending.drain(..count).collect::<Vec<_>>();
7917 let position = batch.last().map(|message| message.position).unwrap_or(0);
7918 state
7919 .message_stream_cursors
7920 .insert(name.to_string(), position);
7921 state.message_stream_waits.remove(name);
7922 Ok(Some(batch))
7923 }
7924
7925 fn message_stream_metadata(&self) -> Result<(Vec<Value>, Vec<Value>)> {
7926 let state = self
7927 .state
7928 .lock()
7929 .map_err(|_| Error::WorkflowStatePoisoned)?;
7930 let mut cursors = state.message_stream_cursors.iter().collect::<Vec<_>>();
7931 cursors.sort_by_key(|(name, _)| *name);
7932 let mut waits = state.message_stream_waits.iter().collect::<Vec<_>>();
7933 waits.sort_by_key(|(name, _)| *name);
7934 Ok((
7935 cursors
7936 .into_iter()
7937 .map(|(name, position)| json!({"stream_name": name, "through_position": position}))
7938 .collect(),
7939 waits
7940 .into_iter()
7941 .map(|(name, position)| json!({"stream_name": name, "after_position": position}))
7942 .collect(),
7943 ))
7944 }
7945 pub fn workflow_identity(&self) -> Result<WorkflowIdentity> {
7947 let state = self
7948 .state
7949 .lock()
7950 .map_err(|_| Error::WorkflowStatePoisoned)?;
7951 Ok(WorkflowIdentity {
7952 workflow_id: state.workflow_id.clone(),
7953 run_id: state.run_id.clone(),
7954 })
7955 }
7956
7957 pub fn history_budget(&self) -> Result<WorkflowHistoryBudget> {
7959 let state = self
7960 .state
7961 .lock()
7962 .map_err(|_| Error::WorkflowStatePoisoned)?;
7963 Ok(state.history_budget.clone())
7964 }
7965
7966 pub fn continue_as_new<T: Serialize>(&self, args: T) -> Result<Value> {
7972 self.continue_as_new_with_options(ContinueAsNewOptions::new(), args)
7973 }
7974
7975 pub fn continue_as_new_with_options<T: Serialize>(
7977 &self,
7978 options: ContinueAsNewOptions,
7979 args: T,
7980 ) -> Result<Value> {
7981 options.validate()?;
7982 Err(Error::ContinueAsNew(ContinueAsNewRequest {
7983 arguments: normalize_avro_arguments(AvroValue::from_serialize(&args)?),
7984 options,
7985 }))
7986 }
7987
7988 pub fn activity<T: Serialize>(
7989 &self,
7990 activity_type: impl Into<String>,
7991 args: T,
7992 ) -> ActivityCall {
7993 self.activity_with_options(activity_type, ActivityOptions::new(), args)
7994 }
7995
7996 pub fn activity_on_queue<T, Q>(
7997 &self,
7998 activity_type: impl Into<String>,
7999 task_queue: Option<Q>,
8000 args: T,
8001 ) -> ActivityCall
8002 where
8003 T: Serialize,
8004 Q: Into<String>,
8005 {
8006 let mut options = ActivityOptions::new();
8007 options.task_queue = task_queue.map(Into::into);
8008 self.activity_with_options(activity_type, options, args)
8009 }
8010
8011 pub fn activity_with_options<T: Serialize>(
8049 &self,
8050 activity_type: impl Into<String>,
8051 options: ActivityOptions,
8052 args: T,
8053 ) -> ActivityCall {
8054 ActivityCall {
8055 ctx: self.clone(),
8056 activity_type: activity_type.into(),
8057 options,
8058 args: Some(AvroValue::from_serialize(&args)),
8059 scheduled: false,
8060 parallel_group_path: Vec::new(),
8061 }
8062 }
8063
8064 pub async fn activity_avro_value<T: Serialize>(
8065 &self,
8066 activity_type: impl Into<String>,
8067 args: T,
8068 ) -> Result<AvroValue> {
8069 let mut call = self.activity(activity_type, args);
8070 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8071 }
8072
8073 pub async fn activity_avro_value_with_options<T: Serialize>(
8074 &self,
8075 activity_type: impl Into<String>,
8076 options: ActivityOptions,
8077 args: T,
8078 ) -> Result<AvroValue> {
8079 let mut call = self.activity_with_options(activity_type, options, args);
8080 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8081 }
8082
8083 pub async fn activity_typed<I, O>(&self, activity_type: impl Into<String>, args: I) -> Result<O>
8085 where
8086 I: Serialize,
8087 O: DeserializeOwned,
8088 {
8089 self.activity_typed_with_options(activity_type, ActivityOptions::new(), args)
8090 .await
8091 }
8092
8093 pub async fn activity_typed_with_options<I, O>(
8099 &self,
8100 activity_type: impl Into<String>,
8101 options: ActivityOptions,
8102 args: I,
8103 ) -> Result<O>
8104 where
8105 I: Serialize,
8106 O: DeserializeOwned,
8107 {
8108 let activity_type = activity_type.into();
8109 let encoded = AvroValue::from_serialize(&args).map_err(|error| {
8110 handler_type_error::<I>(
8111 HandlerKind::Activity,
8112 &activity_type,
8113 HandlerValueKind::Input,
8114 error.to_string(),
8115 )
8116 });
8117 let mut call = ActivityCall {
8118 ctx: self.clone(),
8119 activity_type: activity_type.clone(),
8120 options,
8121 args: Some(encoded),
8122 scheduled: false,
8123 parallel_group_path: Vec::new(),
8124 };
8125 let result = std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await?;
8126 decode_handler_result(result, HandlerKind::Activity, &activity_type)
8127 }
8128
8129 pub fn parallel(&self, operations: Vec<ParallelOperation>) -> ParallelCall {
8136 ParallelCall::new(self.clone(), operations)
8137 }
8138
8139 pub fn join(&self, operations: Vec<ParallelOperation>) -> ParallelCall {
8141 self.parallel(operations)
8142 }
8143
8144 pub async fn parallel_avro_value(
8146 &self,
8147 operations: Vec<ParallelOperation>,
8148 ) -> Result<Vec<ParallelAvroResult>> {
8149 let mut call = self.parallel(operations);
8150 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8151 }
8152
8153 pub fn select(&self, operations: Vec<ParallelOperation>) -> SelectCall {
8156 let operations = operations
8157 .into_iter()
8158 .enumerate()
8159 .map(|(index, operation)| (SelectionKey::Index(index), operation))
8160 .collect();
8161 SelectCall::new(self.clone(), operations)
8162 }
8163
8164 pub fn select_keyed<K>(&self, operations: Vec<(K, ParallelOperation)>) -> SelectCall
8166 where
8167 K: Into<SelectionKey>,
8168 {
8169 SelectCall::new(
8170 self.clone(),
8171 operations
8172 .into_iter()
8173 .map(|(key, operation)| (key.into(), operation))
8174 .collect(),
8175 )
8176 }
8177
8178 pub fn saga(&self) -> Saga {
8180 Saga::new(self.clone())
8181 }
8182
8183 pub fn is_cancellation_requested(&self) -> Result<bool> {
8185 let state = self
8186 .state
8187 .lock()
8188 .map_err(|_| Error::WorkflowStatePoisoned)?;
8189 Ok(state.cancel_requested)
8190 }
8191
8192 pub fn throw_if_cancellation_requested(&self) -> Result<()> {
8197 if self.is_cancellation_requested()? {
8198 return Err(Error::WorkflowCancellationRequested(
8199 WorkflowCancellationRequested,
8200 ));
8201 }
8202 Ok(())
8203 }
8204
8205 pub fn wait_signal(&self, signal_name: impl Into<String>) -> SignalCall {
8206 SignalCall {
8207 ctx: self.clone(),
8208 signal_name: signal_name.into(),
8209 runtime_reserved_allowed: false,
8210 opened_wait: false,
8211 matched_pending: false,
8212 parallel_group_path: Vec::new(),
8213 }
8214 }
8215
8216 fn wait_runtime_signal(&self, signal_name: impl Into<String>) -> SignalCall {
8217 SignalCall {
8218 ctx: self.clone(),
8219 signal_name: signal_name.into(),
8220 runtime_reserved_allowed: true,
8221 opened_wait: false,
8222 matched_pending: false,
8223 parallel_group_path: Vec::new(),
8224 }
8225 }
8226
8227 pub async fn wait_signal_avro_value(
8228 &self,
8229 signal_name: impl Into<String>,
8230 ) -> Result<Vec<AvroValue>> {
8231 let mut call = self.wait_signal(signal_name);
8232 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8233 }
8234
8235 pub fn signals(&self, signal_name: &str) -> Result<Vec<Vec<Value>>> {
8241 self.signals_avro_value(signal_name)?
8242 .into_iter()
8243 .map(|arguments| {
8244 arguments
8245 .into_iter()
8246 .map(AvroValue::into_json)
8247 .collect::<Result<Vec<_>>>()
8248 })
8249 .collect()
8250 }
8251
8252 pub fn signals_avro_value(&self, signal_name: &str) -> Result<Vec<Vec<AvroValue>>> {
8254 let state = self
8255 .state
8256 .lock()
8257 .map_err(|_| Error::WorkflowStatePoisoned)?;
8258 state
8259 .history_events
8260 .iter()
8261 .filter(|event| {
8262 event.event_type == "SignalReceived"
8263 && event.payload.get("signal_name").and_then(Value::as_str) == Some(signal_name)
8264 })
8265 .map(|event| decode_signal_event_arguments(event, &state.payload_codec))
8266 .collect()
8267 }
8268
8269 pub fn updates(&self, update_name: &str) -> Result<Vec<Vec<Value>>> {
8275 self.updates_avro_value(update_name)?
8276 .into_iter()
8277 .map(|arguments| {
8278 arguments
8279 .into_iter()
8280 .map(AvroValue::into_json)
8281 .collect::<Result<Vec<_>>>()
8282 })
8283 .collect()
8284 }
8285
8286 pub fn updates_avro_value(&self, update_name: &str) -> Result<Vec<Vec<AvroValue>>> {
8288 let state = self
8289 .state
8290 .lock()
8291 .map_err(|_| Error::WorkflowStatePoisoned)?;
8292 let mut seen = Vec::new();
8293 let mut updates = Vec::new();
8294 for event in state.history_events.iter() {
8295 if !matches!(
8296 event.event_type.as_str(),
8297 "UpdateAccepted" | "UpdateApplied"
8298 ) || event.payload.get("update_name").and_then(Value::as_str) != Some(update_name)
8299 || event.payload.get("arguments").is_none()
8300 {
8301 continue;
8302 }
8303 if let Some(update_id) = event.payload.get("update_id").and_then(Value::as_str) {
8304 if seen.iter().any(|recorded| recorded == update_id) {
8305 continue;
8306 }
8307 seen.push(update_id.to_string());
8308 }
8309 updates.push(decode_update_event_arguments(event, &state.payload_codec)?);
8310 }
8311 Ok(updates)
8312 }
8313
8314 pub fn wait_condition<F>(
8322 &self,
8323 options: ConditionWaitOptions,
8324 predicate: F,
8325 ) -> ConditionWaitCall
8326 where
8327 F: Fn() -> Result<bool> + Send + 'static,
8328 {
8329 ConditionWaitCall {
8330 ctx: self.clone(),
8331 options,
8332 predicate: Box::new(predicate),
8333 occurrence_id: None,
8334 opened_wait: false,
8335 parallel_group_path: Vec::new(),
8336 }
8337 }
8338
8339 pub fn sleep(&self, duration: Duration) -> TimerCall {
8360 let delay_seconds = duration
8361 .as_secs()
8362 .checked_add(u64::from(duration.subsec_nanos() > 0));
8363 TimerCall {
8364 ctx: self.clone(),
8365 delay_seconds,
8366 scheduled: false,
8367 matched_pending: false,
8368 parallel_group_path: Vec::new(),
8369 }
8370 }
8371
8372 pub fn start_timer(&self, duration: Duration) -> TimerCall {
8374 self.sleep(duration)
8375 }
8376
8377 pub fn side_effect<T, F>(&self, callback: F) -> Result<T>
8384 where
8385 T: Serialize + DeserializeOwned,
8386 F: FnOnce() -> T,
8387 {
8388 {
8389 let mut state = self
8390 .state
8391 .lock()
8392 .map_err(|_| Error::WorkflowStatePoisoned)?;
8393 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8394 return match recorded {
8395 RecordedCommand::SideEffect { sequence, value } => {
8396 state.command_cursor += 1;
8397 value.deserialize().map_err(|error| {
8398 Error::NonDeterministicReplay(ReplayFailure::new(
8399 "side_effect_type_mismatch",
8400 Some(sequence),
8401 Some(std::any::type_name::<T>().to_string()),
8402 Some(error.to_string()),
8403 "recorded side-effect value is incompatible with the requested Rust type",
8404 ))
8405 })
8406 }
8407 other => Err(command_mismatch(&other, "side effect")),
8408 };
8409 }
8410 }
8411
8412 let value = callback();
8413 let avro_value = AvroValue::from_serialize(&value)?;
8414 let mut state = self
8415 .state
8416 .lock()
8417 .map_err(|_| Error::WorkflowStatePoisoned)?;
8418 let result = encode_typed_envelope(&avro_value, &state.payload_codec)?;
8419 state.commands.push(json!({
8420 "type": "record_side_effect",
8421 "result": result,
8422 }));
8423 Ok(value)
8424 }
8425
8426 pub fn side_effect_avro_value<F>(&self, callback: F) -> Result<AvroValue>
8428 where
8429 F: FnOnce() -> AvroValue,
8430 {
8431 {
8432 let mut state = self
8433 .state
8434 .lock()
8435 .map_err(|_| Error::WorkflowStatePoisoned)?;
8436 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8437 return match recorded {
8438 RecordedCommand::SideEffect { value, .. } => {
8439 state.command_cursor += 1;
8440 Ok(value)
8441 }
8442 other => Err(command_mismatch(&other, "side effect")),
8443 };
8444 }
8445 }
8446
8447 let value = callback();
8448 let mut state = self
8449 .state
8450 .lock()
8451 .map_err(|_| Error::WorkflowStatePoisoned)?;
8452 let result = encode_typed_envelope(&value, &state.payload_codec)?;
8453 state.commands.push(json!({
8454 "type": "record_side_effect",
8455 "result": result,
8456 }));
8457 Ok(value)
8458 }
8459
8460 pub fn append_workflow_stream(
8466 &self,
8467 stream_name: impl Into<String>,
8468 items: &[WorkflowStreamAppendItem],
8469 max_pending_items: Option<u64>,
8470 ) -> Result<()> {
8471 if items.is_empty() {
8472 return Err(Error::Codec(
8473 "workflow_stream_items_empty: append requires at least one item".to_string(),
8474 ));
8475 }
8476 if max_pending_items == Some(0) {
8477 return Err(Error::Codec(
8478 "workflow_stream_pending_limit_invalid: max_pending_items must be positive"
8479 .to_string(),
8480 ));
8481 }
8482 let stream_name = stream_name.into();
8483 if stream_name.is_empty() {
8484 return Err(Error::Codec(
8485 "workflow_stream_name_invalid: stream name must not be empty".to_string(),
8486 ));
8487 }
8488
8489 let mut state = self
8490 .state
8491 .lock()
8492 .map_err(|_| Error::WorkflowStatePoisoned)?;
8493 let command_ordinal = state.workflow_stream_command_counter;
8494 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8495 state.workflow_stream_command_counter += 1;
8496 return match recorded {
8497 RecordedCommand::SideEffect { .. } => {
8498 state.command_cursor += 1;
8499 Ok(())
8500 }
8501 other => Err(command_mismatch(&other, "workflow stream append")),
8502 };
8503 }
8504
8505 let identity = Self::workflow_stream_command_identity(&state)?.to_string();
8506 state.workflow_stream_command_counter += 1;
8507 let wire_items = items
8508 .iter()
8509 .enumerate()
8510 .map(|(item_index, item)| {
8511 item.wire_value(Some(format!(
8512 "dw-stream:{identity}:{command_ordinal}:{item_index}"
8513 )))
8514 })
8515 .collect::<Vec<_>>();
8516 let mut directive = json!({
8517 "operation": "append",
8518 "stream_name": stream_name,
8519 "command_identity": identity,
8520 "command_ordinal": command_ordinal,
8521 "items": wire_items,
8522 });
8523 if let Some(max_pending_items) = max_pending_items {
8524 directive["max_pending_items"] = json!(max_pending_items);
8525 }
8526 let result = encode_typed_envelope(&AvroValue::Null, &state.payload_codec)?;
8527 state.commands.push(json!({
8528 "type": "record_side_effect",
8529 "result": result,
8530 "workflow_stream": directive,
8531 }));
8532 Ok(())
8533 }
8534
8535 pub fn close_workflow_stream(
8537 &self,
8538 stream_name: impl Into<String>,
8539 retention_seconds: Option<u64>,
8540 ) -> Result<()> {
8541 self.finish_workflow_stream(stream_name.into(), None, retention_seconds)
8542 }
8543
8544 pub fn error_workflow_stream(
8546 &self,
8547 stream_name: impl Into<String>,
8548 error_reason: impl Into<String>,
8549 retention_seconds: Option<u64>,
8550 ) -> Result<()> {
8551 let error_reason = error_reason.into();
8552 if error_reason.is_empty() {
8553 return Err(Error::Codec(
8554 "workflow_stream_error_invalid: error reason must not be empty".to_string(),
8555 ));
8556 }
8557 self.finish_workflow_stream(stream_name.into(), Some(error_reason), retention_seconds)
8558 }
8559
8560 fn finish_workflow_stream(
8561 &self,
8562 stream_name: String,
8563 error_reason: Option<String>,
8564 retention_seconds: Option<u64>,
8565 ) -> Result<()> {
8566 if stream_name.is_empty() {
8567 return Err(Error::Codec(
8568 "workflow_stream_name_invalid: stream name must not be empty".to_string(),
8569 ));
8570 }
8571 if retention_seconds == Some(0) {
8572 return Err(Error::Codec(
8573 "workflow_stream_retention_invalid: retention_seconds must be positive".to_string(),
8574 ));
8575 }
8576 let mut state = self
8577 .state
8578 .lock()
8579 .map_err(|_| Error::WorkflowStatePoisoned)?;
8580 let command_ordinal = state.workflow_stream_command_counter;
8581 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8582 state.workflow_stream_command_counter += 1;
8583 return match recorded {
8584 RecordedCommand::SideEffect { .. } => {
8585 state.command_cursor += 1;
8586 Ok(())
8587 }
8588 other => Err(command_mismatch(&other, "workflow stream close")),
8589 };
8590 }
8591 let identity = Self::workflow_stream_command_identity(&state)?.to_string();
8592 state.workflow_stream_command_counter += 1;
8593 let mut directive = json!({
8594 "operation": if error_reason.is_some() { "error" } else { "close" },
8595 "stream_name": stream_name,
8596 "command_identity": identity,
8597 "command_ordinal": command_ordinal,
8598 });
8599 if let Some(error_reason) = error_reason {
8600 directive["error_reason"] = json!(error_reason);
8601 }
8602 if let Some(retention_seconds) = retention_seconds {
8603 directive["retention_seconds"] = json!(retention_seconds);
8604 }
8605 let result = encode_typed_envelope(&AvroValue::Null, &state.payload_codec)?;
8606 state.commands.push(json!({
8607 "type": "record_side_effect",
8608 "result": result,
8609 "workflow_stream": directive,
8610 }));
8611 Ok(())
8612 }
8613
8614 fn workflow_stream_command_identity(state: &WorkflowState) -> Result<&str> {
8615 let identity = state.workflow_command_identity.as_str();
8616 if identity.is_empty() {
8617 return Err(Error::MissingWorkflowCommandIdentity);
8618 }
8619 Ok(identity)
8620 }
8621
8622 pub fn upsert_search_attributes(&self, update: SearchAttributeUpdate) -> Result<()> {
8628 update.validate()?;
8629 let (attributes, attribute_types) = update.into_wire_parts();
8630 let mut state = self
8631 .state
8632 .lock()
8633 .map_err(|_| Error::WorkflowStatePoisoned)?;
8634
8635 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8636 return match recorded {
8637 RecordedCommand::SearchAttributes {
8638 sequence,
8639 attributes: recorded_attributes,
8640 attribute_types: recorded_attribute_types,
8641 } => {
8642 if recorded_attributes != attributes {
8643 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8644 "search_attribute_value_mismatch",
8645 Some(sequence),
8646 Some(recorded_attributes.to_string()),
8647 Some(attributes.to_string()),
8648 "search-attribute values differ from the recorded durable command",
8649 )));
8650 }
8651 if let RecordedSnapshotValue::Known(recorded_types) = recorded_attribute_types {
8652 if recorded_types != attribute_types {
8653 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8654 "search_attribute_type_mismatch",
8655 Some(sequence),
8656 Some(json!(recorded_types).to_string()),
8657 Some(json!(attribute_types).to_string()),
8658 "search-attribute declared types differ from the recorded durable command",
8659 )));
8660 }
8661 }
8662 state.command_cursor += 1;
8663 Ok(())
8664 }
8665 other => Err(command_mismatch(&other, "search-attribute update")),
8666 };
8667 }
8668
8669 let mut command = serde_json::Map::from_iter([
8670 ("type".to_string(), json!("upsert_search_attributes")),
8671 ("attributes".to_string(), attributes),
8672 ]);
8673 if !attribute_types.is_empty() {
8674 command.insert("attribute_types".to_string(), json!(attribute_types));
8675 }
8676 state.commands.push(Value::Object(command));
8677 Ok(())
8678 }
8679
8680 pub fn uuid_v4(&self) -> Result<Uuid> {
8682 self.side_effect(Uuid::new_v4)
8683 }
8684
8685 pub fn get_version(
8688 &self,
8689 change_id: impl Into<String>,
8690 min_supported: i32,
8691 max_supported: i32,
8692 ) -> Result<i32> {
8693 let change_id = change_id.into();
8694 if change_id.trim().is_empty() {
8695 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8696 "version_change_id_invalid",
8697 None,
8698 Some("non-empty change ID".to_string()),
8699 Some(change_id),
8700 "version markers require a stable non-empty change ID",
8701 )));
8702 }
8703 if min_supported > max_supported {
8704 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8705 "version_range_invalid",
8706 None,
8707 Some("min_supported <= max_supported".to_string()),
8708 Some(format!("{min_supported}..={max_supported}")),
8709 "version marker supported range is invalid",
8710 )));
8711 }
8712
8713 let mut state = self
8714 .state
8715 .lock()
8716 .map_err(|_| Error::WorkflowStatePoisoned)?;
8717 if let Some((version, sequence)) = state.version_markers.get(&change_id).copied() {
8718 ensure_version_supported(&change_id, version, min_supported, max_supported, sequence)?;
8719 return Ok(version);
8720 }
8721
8722 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8723 return match recorded {
8724 RecordedCommand::VersionMarker {
8725 sequence,
8726 change_id: recorded_change_id,
8727 version,
8728 ..
8729 } => {
8730 if recorded_change_id != change_id {
8731 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8732 "version_change_id_mismatch",
8733 Some(sequence),
8734 Some(recorded_change_id),
8735 Some(change_id),
8736 "recorded version marker change ID differs from current workflow code",
8737 )));
8738 }
8739 ensure_version_supported(
8740 &change_id,
8741 version,
8742 min_supported,
8743 max_supported,
8744 sequence,
8745 )?;
8746 state.command_cursor += 1;
8747 state.version_markers.insert(change_id, (version, sequence));
8748 Ok(version)
8749 }
8750 other => Err(command_mismatch(
8751 &other,
8752 format!("version marker:{change_id}"),
8753 )),
8754 };
8755 }
8756
8757 let version = max_supported;
8758 state.commands.push(json!({
8759 "type": "record_version_marker",
8760 "change_id": change_id,
8761 "version": version,
8762 "min_supported": min_supported,
8763 "max_supported": max_supported,
8764 }));
8765 state.version_markers.insert(change_id, (version, 0));
8768 Ok(version)
8769 }
8770
8771 pub fn patched(&self, change_id: impl Into<String>) -> Result<bool> {
8773 Ok(self.get_version(change_id, -1, 1)? == 1)
8774 }
8775
8776 pub fn deprecate_patch(&self, change_id: impl Into<String>) -> Result<()> {
8778 self.get_version(change_id, -1, 1).map(|_| ())
8779 }
8780
8781 pub fn upsert_memo<T: Serialize>(&self, entries: T) -> Result<()> {
8786 let entries = canonical_memo_entries(AvroValue::from_serialize(&entries)?, true)?;
8787 let mut state = self
8788 .state
8789 .lock()
8790 .map_err(|_| Error::WorkflowStatePoisoned)?;
8791
8792 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8793 return match recorded {
8794 RecordedCommand::Memo {
8795 sequence,
8796 entries: recorded_entries,
8797 } => {
8798 if recorded_entries != entries {
8799 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8800 "memo_update_mismatch",
8801 Some(sequence),
8802 Some(format!("{recorded_entries:?}")),
8803 Some(format!("{entries:?}")),
8804 "recorded memo entries differ from the current workflow update",
8805 )));
8806 }
8807 state.command_cursor += 1;
8808 Ok(())
8809 }
8810 other => Err(command_mismatch(&other, "memo upsert")),
8811 };
8812 }
8813
8814 let entries_envelope = encode_typed_envelope(&entries, DEFAULT_CODEC)?;
8815 state.commands.push(json!({
8816 "type": "upsert_memo",
8817 "entries": entries_envelope,
8818 }));
8819 Ok(())
8820 }
8821
8822 pub fn start_child_workflow<T: Serialize>(
8849 &self,
8850 workflow_type: impl Into<String>,
8851 options: ChildWorkflowOptions,
8852 args: T,
8853 ) -> ChildWorkflowCall {
8854 ChildWorkflowCall {
8855 ctx: self.clone(),
8856 workflow_type: workflow_type.into(),
8857 options,
8858 args: Some(AvroValue::from_serialize(&args)),
8859 scheduled: false,
8860 matched_pending: false,
8861 parallel_group_path: Vec::new(),
8862 }
8863 }
8864
8865 pub async fn start_child_workflow_avro_value<T: Serialize>(
8866 &self,
8867 workflow_type: impl Into<String>,
8868 options: ChildWorkflowOptions,
8869 args: T,
8870 ) -> Result<ChildWorkflowAvroResult> {
8871 let mut call = self.start_child_workflow(workflow_type, options, args);
8872 std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8873 }
8874
8875 fn take_commands(&self) -> Result<Vec<Value>> {
8876 let mut state = self
8877 .state
8878 .lock()
8879 .map_err(|_| Error::WorkflowStatePoisoned)?;
8880 Ok(std::mem::take(&mut state.commands))
8881 }
8882
8883 fn continue_as_new_command(&self, request: ContinueAsNewRequest) -> Result<Option<Value>> {
8884 let mut state = self
8885 .state
8886 .lock()
8887 .map_err(|_| Error::WorkflowStatePoisoned)?;
8888
8889 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8890 return Err(command_mismatch(&recorded, "continue as new"));
8891 }
8892 if state.recorded_continue_as_new_sequence.is_some() {
8893 state.continue_as_new_consumed = true;
8894 return Ok(None);
8895 }
8896
8897 let arguments = encode_typed_envelope(&request.arguments, &state.payload_codec)?;
8898 let mut command = serde_json::Map::from_iter([
8899 ("type".to_string(), json!("continue_as_new")),
8900 ("arguments".to_string(), arguments),
8901 ("queue".to_string(), json!(state.task_queue.clone())),
8902 ]);
8903 if let Some(workflow_type) = request.options.workflow_type {
8904 command.insert("workflow_type".to_string(), json!(workflow_type));
8905 }
8906 if let Some(task_queue) = request.options.task_queue {
8907 command.insert("queue".to_string(), json!(task_queue));
8908 }
8909 Ok(Some(Value::Object(command)))
8910 }
8911
8912 fn matched_recorded_pending(&self) -> Result<bool> {
8913 let state = self
8914 .state
8915 .lock()
8916 .map_err(|_| Error::WorkflowStatePoisoned)?;
8917 Ok(state.matched_recorded_pending)
8918 }
8919
8920 fn ensure_history_consumed(&self) -> Result<()> {
8921 let state = self
8922 .state
8923 .lock()
8924 .map_err(|_| Error::WorkflowStatePoisoned)?;
8925 if let Some(command) = state.recorded_commands.get(state.command_cursor) {
8926 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8927 "recorded_commands_unconsumed",
8928 Some(command.sequence()),
8929 Some(command.shape().to_string()),
8930 Some("workflow completion".to_string()),
8931 "workflow completed before consuming all recorded durable commands",
8932 )));
8933 }
8934 if let Some(sequence) = state
8935 .recorded_continue_as_new_sequence
8936 .filter(|_| !state.continue_as_new_consumed)
8937 {
8938 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8939 "recorded_continue_as_new_unconsumed",
8940 Some(sequence),
8941 Some("continue as new".to_string()),
8942 Some("workflow completion".to_string()),
8943 "workflow completed without consuming its recorded continue-as-new transition",
8944 )));
8945 }
8946 Ok(())
8947 }
8948}
8949
8950fn contiguous_message_stream_count(
8951 pending: &[MessageStreamMessage],
8952 cursor: u64,
8953 max_items: usize,
8954) -> usize {
8955 pending
8956 .iter()
8957 .take(max_items)
8958 .enumerate()
8959 .take_while(|(offset, message)| {
8960 u64::try_from(*offset)
8961 .ok()
8962 .and_then(|offset| cursor.checked_add(offset + 1))
8963 == Some(message.position)
8964 })
8965 .count()
8966}
8967
8968fn is_authored_command_open_event(event: &HistoryEvent) -> bool {
8969 matches!(
8970 event.event_type.as_str(),
8971 "ActivityScheduled"
8972 | "TimerScheduled"
8973 | "ChildWorkflowScheduled"
8974 | "SignalWaitOpened"
8975 | "ConditionWaitOpened"
8976 | "SearchAttributesUpserted"
8977 | "SideEffectRecorded"
8978 | "VersionMarkerRecorded"
8979 | "MemoUpserted"
8980 | "WorkflowContinuedAsNew"
8981 )
8982}
8983
8984#[derive(Debug)]
8985struct WorkflowState {
8986 workflow_id: Option<String>,
8987 run_id: Option<String>,
8988 task_queue: String,
8989 payload_codec: String,
8990 history_events: Arc<Vec<HistoryEvent>>,
8991 history_budget: WorkflowHistoryBudget,
8992 cancel_requested: bool,
8993 resume_signal: Option<ResumeSignal>,
8994 recorded_commands: Vec<RecordedCommand>,
8995 selection_markers: Vec<SelectionMarker>,
8996 selection_marker_cursor: usize,
8997 cancelled_selection_members: Vec<SelectionCancellation>,
8998 recorded_continue_as_new_sequence: Option<u64>,
8999 continue_as_new_consumed: bool,
9000 command_cursor: usize,
9001 condition_wait_occurrence_counter: u64,
9002 matched_recorded_pending: bool,
9003 version_markers: HashMap<String, (i32, u64)>,
9004 workflow_command_identity: String,
9005 workflow_stream_command_counter: u64,
9006 commands: Vec<Value>,
9007 message_stream_messages: HashMap<String, Vec<MessageStreamMessage>>,
9008 message_stream_cursors: HashMap<String, u64>,
9009 message_stream_waits: HashMap<String, u64>,
9010}
9011
9012impl WorkflowState {
9013 #[cfg(test)]
9014 fn new(
9015 history: Vec<HistoryEvent>,
9016 task_queue: String,
9017 payload_codec: String,
9018 resume_signal: Option<ResumeSignal>,
9019 ) -> Result<Self> {
9020 Self::new_with_identity(
9021 history,
9022 None,
9023 None,
9024 task_queue,
9025 payload_codec,
9026 resume_signal,
9027 )
9028 }
9029
9030 fn new_with_identity(
9031 history: Vec<HistoryEvent>,
9032 workflow_id: Option<String>,
9033 run_id: Option<String>,
9034 task_queue: String,
9035 payload_codec: String,
9036 resume_signal: Option<ResumeSignal>,
9037 ) -> Result<Self> {
9038 let recorded_commands = recorded_commands(
9039 &history,
9040 &payload_codec,
9041 WorkflowIdentity {
9042 workflow_id: workflow_id.clone(),
9043 run_id: run_id.clone(),
9044 },
9045 )?;
9046 let selection_markers = recorded_selection_markers(&history)?;
9047 let cancelled_selection_members = recorded_selection_cancellations(&history)?;
9048 let recorded_continue_as_new = history
9049 .iter()
9050 .filter(|event| event.event_type == "WorkflowContinuedAsNew")
9051 .collect::<Vec<_>>();
9052 if recorded_continue_as_new.len() > 1 {
9053 return Err(invalid_recorded_history(
9054 "duplicate_continue_as_new_transition",
9055 recorded_continue_as_new
9056 .last()
9057 .and_then(|event| durable_event_sequence(event))
9058 .unwrap_or(0),
9059 "one WorkflowContinuedAsNew event",
9060 &format!(
9061 "{} WorkflowContinuedAsNew events",
9062 recorded_continue_as_new.len()
9063 ),
9064 "workflow history records one continue-as-new transition more than once",
9065 ));
9066 }
9067 let recorded_continue_as_new_sequence = recorded_continue_as_new
9068 .first()
9069 .map(|event| {
9070 durable_event_sequence(event).ok_or_else(|| {
9071 Error::NonDeterministicReplay(ReplayFailure::new(
9072 "continue_as_new_sequence_missing",
9073 None,
9074 Some("recorded transition sequence".to_string()),
9075 Some("missing sequence".to_string()),
9076 "WorkflowContinuedAsNew history is missing its recorded sequence",
9077 ))
9078 })
9079 })
9080 .transpose()?;
9081 let mut message_stream_cursors = HashMap::new();
9082 for event in &history {
9083 if !matches!(
9084 event.event_type.as_str(),
9085 "SignalReceived" | "SignalApplied"
9086 ) || event.payload.get("signal_name").and_then(Value::as_str)
9087 != Some(MESSAGE_STREAM_SIGNAL)
9088 {
9089 continue;
9090 }
9091 let arguments = decode_signal_event_arguments(event, &payload_codec)?;
9092 if arguments.len() != 1 {
9093 continue;
9094 }
9095 let envelope = arguments[0].clone().into_json()?;
9096 let Some(envelope) = envelope.as_object() else {
9097 continue;
9098 };
9099 if envelope.get("schema").and_then(Value::as_str) != Some(MESSAGE_STREAM_CURSOR_SCHEMA)
9100 {
9101 continue;
9102 }
9103 let Some(stream_name) = envelope.get("stream_name").and_then(Value::as_str) else {
9104 continue;
9105 };
9106 let Some(through_position) = envelope.get("through_position").and_then(Value::as_u64)
9107 else {
9108 continue;
9109 };
9110 let cursor = message_stream_cursors
9111 .entry(stream_name.to_string())
9112 .or_insert(0);
9113 *cursor = (*cursor).max(through_position);
9114 }
9115 let event_count = u64::try_from(history.len()).unwrap_or(u64::MAX);
9116 let cancel_requested = history.iter().any(|event| {
9117 matches!(
9118 event.event_type.as_str(),
9119 "WorkflowCancellationRequested" | "WorkflowCancelRequested"
9120 )
9121 });
9122 Ok(Self {
9123 workflow_command_identity: String::new(),
9124 workflow_stream_command_counter: 0,
9125 workflow_id,
9126 run_id,
9127 task_queue,
9128 payload_codec,
9129 history_events: Arc::new(history),
9130 history_budget: WorkflowHistoryBudget {
9131 event_count,
9132 ..WorkflowHistoryBudget::default()
9133 },
9134 cancel_requested,
9135 resume_signal,
9136 recorded_commands,
9137 selection_markers,
9138 selection_marker_cursor: 0,
9139 cancelled_selection_members,
9140 recorded_continue_as_new_sequence,
9141 continue_as_new_consumed: false,
9142 command_cursor: 0,
9143 condition_wait_occurrence_counter: 0,
9144 matched_recorded_pending: false,
9145 version_markers: HashMap::new(),
9146 commands: Vec::new(),
9147 message_stream_messages: HashMap::new(),
9148 message_stream_cursors,
9149 message_stream_waits: HashMap::new(),
9150 })
9151 }
9152}
9153
9154enum MessageStreamDelivery {
9155 Message(MessageStreamMessage),
9156 Cursor {
9157 stream_name: String,
9158 through_position: u64,
9159 },
9160}
9161
9162fn decode_message_stream_delivery(arguments: Vec<Value>) -> Result<Option<MessageStreamDelivery>> {
9163 if arguments.len() != 1 {
9164 return Ok(None);
9165 }
9166 let envelope = arguments
9167 .into_iter()
9168 .next()
9169 .expect("one argument was checked");
9170 let Some(envelope) = envelope.as_object() else {
9171 return Ok(None);
9172 };
9173 let Some(stream_name) = envelope.get("stream_name").and_then(Value::as_str) else {
9174 return Ok(None);
9175 };
9176 if envelope.get("schema").and_then(Value::as_str) == Some(MESSAGE_STREAM_CURSOR_SCHEMA) {
9177 let Some(through_position) = envelope.get("through_position").and_then(Value::as_u64)
9178 else {
9179 return Ok(None);
9180 };
9181 return Ok(Some(MessageStreamDelivery::Cursor {
9182 stream_name: stream_name.to_string(),
9183 through_position,
9184 }));
9185 }
9186 if envelope.get("schema").and_then(Value::as_str) != Some(MESSAGE_STREAM_SCHEMA) {
9187 return Ok(None);
9188 }
9189 let Some(message_id) = envelope.get("message_id").and_then(Value::as_str) else {
9190 return Ok(None);
9191 };
9192 let Some(position) = envelope
9193 .get("position")
9194 .and_then(Value::as_u64)
9195 .filter(|value| *value > 0)
9196 else {
9197 return Ok(None);
9198 };
9199 let Some(payload_envelope) = envelope.get("payload_envelope") else {
9200 return Ok(None);
9201 };
9202 let Ok(payload_envelope) = serde_json::from_value::<PayloadEnvelope>(payload_envelope.clone())
9203 else {
9204 return Ok(None);
9205 };
9206 let decoded = decode_avro_value(&payload_envelope)?;
9207 let AvroValue::Array(values) = decoded else {
9208 return Ok(None);
9209 };
9210 Ok(Some(MessageStreamDelivery::Message(MessageStreamMessage {
9211 stream_name: stream_name.to_string(),
9212 message_id: message_id.to_string(),
9213 position,
9214 arguments: values,
9215 })))
9216}
9217
9218#[derive(Clone, Debug)]
9219enum RecordedCommand {
9220 Activity {
9221 sequence: u64,
9222 activity_type: Option<String>,
9223 options: Option<RecordedActivityOptions>,
9224 outcome: Option<ActivityOutcome>,
9225 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9226 },
9227 Timer {
9228 sequence: u64,
9229 delay_seconds: u64,
9230 fired: bool,
9231 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9232 },
9233 ChildWorkflow {
9234 sequence: u64,
9235 workflow_type: Option<String>,
9236 outcome: Option<ChildWorkflowOutcome>,
9237 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9238 },
9239 SignalWait {
9240 sequence: u64,
9241 signal_name: String,
9242 value: Option<Vec<AvroValue>>,
9243 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9244 },
9245 ConditionWait {
9246 sequence: u64,
9247 occurrence_id: String,
9248 condition_key: Option<String>,
9249 predicate_identity: String,
9250 timeout_seconds: Option<u64>,
9251 result: Option<ConditionWaitResult>,
9252 parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9253 },
9254 SearchAttributes {
9255 sequence: u64,
9256 attributes: Value,
9257 attribute_types: RecordedSnapshotValue<BTreeMap<String, String>>,
9258 },
9259 SideEffect {
9260 sequence: u64,
9261 value: AvroValue,
9262 },
9263 VersionMarker {
9264 sequence: u64,
9265 change_id: String,
9266 version: i32,
9267 },
9268 Memo {
9269 sequence: u64,
9270 entries: AvroValue,
9271 },
9272}
9273
9274#[derive(Clone, Debug, PartialEq, Eq)]
9275struct SelectionMarker {
9276 selection_group_id: String,
9277 selection_group_base_sequence: u64,
9278 selection_group_size: usize,
9279 member_key: SelectionKey,
9280 member_index: usize,
9281 member_base_sequence: u64,
9282 member_size: usize,
9283 operation_kind: String,
9284 operation_identity: String,
9285 outcome: String,
9286 resolution_event_id: String,
9287 resolution_event_type: String,
9288}
9289
9290#[derive(Clone, Debug, PartialEq, Eq)]
9291struct SelectionCancellation {
9292 selection_group_id: String,
9293 member_key: SelectionKey,
9294 member_index: usize,
9295 member_base_sequence: u64,
9296 member_size: usize,
9297 operation_kind: String,
9298 operation_identity: String,
9299}
9300
9301fn recorded_selection_markers(events: &[HistoryEvent]) -> Result<Vec<SelectionMarker>> {
9302 let mut markers: Vec<SelectionMarker> = Vec::new();
9303 for event in events
9304 .iter()
9305 .filter(|event| event.event_type == "SelectionResolved")
9306 {
9307 let payload = &event.payload;
9308 let base_sequence = required_selection_u64(payload, "selection_group_base_sequence")?;
9309 let group_size = required_selection_usize(payload, "selection_group_size")?;
9310 let member_base_sequence = required_selection_u64(payload, "member_base_sequence")?;
9311 let member_size = required_selection_usize(payload, "member_size")?;
9312 let member_index = required_selection_usize_allow_zero(payload, "member_index")?;
9313 let group_id = payload_string(payload, "selection_group_id").ok_or_else(|| {
9314 invalid_recorded_history(
9315 "selection_marker_invalid",
9316 base_sequence,
9317 "non-empty selection_group_id",
9318 &payload.to_string(),
9319 "selection winner history is missing its durable group identity",
9320 )
9321 })?;
9322 let expected_group_id = format!("select-calls:{base_sequence}:{group_size}");
9323 if group_id != expected_group_id {
9324 return Err(invalid_recorded_history(
9325 "selection_marker_invalid",
9326 base_sequence,
9327 &expected_group_id,
9328 &group_id,
9329 "selection winner history contains an incompatible group identity",
9330 ));
9331 }
9332 let group_end = base_sequence
9333 .checked_add(u64::try_from(group_size).unwrap_or(u64::MAX))
9334 .unwrap_or(u64::MAX);
9335 let member_end = member_base_sequence
9336 .checked_add(u64::try_from(member_size).unwrap_or(u64::MAX))
9337 .unwrap_or(u64::MAX);
9338 if member_index >= group_size
9339 || member_base_sequence < base_sequence
9340 || member_end > group_end
9341 {
9342 return Err(invalid_recorded_history(
9343 "selection_marker_invalid",
9344 base_sequence,
9345 "winner member within selection group bounds",
9346 &payload.to_string(),
9347 "selection winner history contains an invalid member range",
9348 ));
9349 }
9350 let operation_kind = payload_string(payload, "operation_kind").ok_or_else(|| {
9351 invalid_recorded_history(
9352 "selection_marker_invalid",
9353 base_sequence,
9354 "selection operation kind",
9355 &payload.to_string(),
9356 "selection winner history is missing its operation kind",
9357 )
9358 })?;
9359 if !matches!(
9360 operation_kind.as_str(),
9361 "activity" | "child" | "timer" | "signal" | "condition" | "group"
9362 ) {
9363 return Err(invalid_recorded_history(
9364 "selection_marker_invalid",
9365 base_sequence,
9366 "activity, child, timer, signal, condition, or group",
9367 &operation_kind,
9368 "selection winner history contains an unsupported operation kind",
9369 ));
9370 }
9371 let operation_identity =
9372 payload_string(payload, "operation_identity").ok_or_else(|| {
9373 invalid_recorded_history(
9374 "selection_marker_invalid",
9375 base_sequence,
9376 "non-empty operation identity",
9377 &payload.to_string(),
9378 "selection winner history is missing its durable operation identity",
9379 )
9380 })?;
9381 let outcome = payload_string(payload, "outcome").ok_or_else(|| {
9382 invalid_recorded_history(
9383 "selection_marker_invalid",
9384 base_sequence,
9385 "completed or failed selection outcome",
9386 &payload.to_string(),
9387 "selection winner history is missing its outcome",
9388 )
9389 })?;
9390 if !matches!(outcome.as_str(), "completed" | "failed") {
9391 return Err(invalid_recorded_history(
9392 "selection_marker_invalid",
9393 base_sequence,
9394 "completed or failed selection outcome",
9395 &outcome,
9396 "selection winner history contains an unsupported outcome",
9397 ));
9398 }
9399 let marker = SelectionMarker {
9400 selection_group_id: group_id,
9401 selection_group_base_sequence: base_sequence,
9402 selection_group_size: group_size,
9403 member_key: selection_key_from_value(payload.get("member_key"), base_sequence)?,
9404 member_index,
9405 member_base_sequence,
9406 member_size,
9407 operation_kind,
9408 operation_identity,
9409 outcome,
9410 resolution_event_id: payload_string(payload, "resolution_event_id").ok_or_else(
9411 || {
9412 invalid_recorded_history(
9413 "selection_marker_invalid",
9414 base_sequence,
9415 "durable resolution_event_id",
9416 &payload.to_string(),
9417 "selection winner history is missing its terminal event identity",
9418 )
9419 },
9420 )?,
9421 resolution_event_type: payload_string(payload, "resolution_event_type").ok_or_else(
9422 || {
9423 invalid_recorded_history(
9424 "selection_marker_invalid",
9425 base_sequence,
9426 "durable resolution_event_type",
9427 &payload.to_string(),
9428 "selection winner history is missing its terminal event type",
9429 )
9430 },
9431 )?,
9432 };
9433 if let Some(existing) = markers
9434 .iter()
9435 .find(|existing| existing.selection_group_id == marker.selection_group_id)
9436 {
9437 if existing != &marker {
9438 return Err(invalid_recorded_history(
9439 "selection_marker_conflict",
9440 base_sequence,
9441 &format!("one winner for {}", marker.selection_group_id),
9442 &payload.to_string(),
9443 "selection history records conflicting winners for one durable group",
9444 ));
9445 }
9446 continue;
9447 }
9448 markers.push(marker);
9449 }
9450 Ok(markers)
9451}
9452
9453fn recorded_selection_cancellations(events: &[HistoryEvent]) -> Result<Vec<SelectionCancellation>> {
9454 let mut cancelled: Vec<SelectionCancellation> = Vec::new();
9455 for event in events
9456 .iter()
9457 .filter(|event| event.event_type == "SelectionOperationCancelled")
9458 {
9459 let group_id = payload_string(&event.payload, "selection_group_id").ok_or_else(|| {
9460 invalid_recorded_history(
9461 "selection_cancellation_invalid",
9462 0,
9463 "non-empty selection_group_id",
9464 &event.payload.to_string(),
9465 "selection cancellation history is missing its group identity",
9466 )
9467 })?;
9468 let member_base_sequence = required_selection_u64(&event.payload, "member_base_sequence")?;
9469 let marker = SelectionCancellation {
9470 selection_group_id: group_id,
9471 member_key: selection_key_from_value(
9472 event.payload.get("member_key"),
9473 member_base_sequence,
9474 )?,
9475 member_index: required_selection_usize_allow_zero(&event.payload, "member_index")?,
9476 member_base_sequence,
9477 member_size: required_selection_usize(&event.payload, "member_size")?,
9478 operation_kind: payload_string(&event.payload, "operation_kind").ok_or_else(|| {
9479 invalid_recorded_history(
9480 "selection_cancellation_invalid",
9481 member_base_sequence,
9482 "selection operation kind",
9483 &event.payload.to_string(),
9484 "selection cancellation is missing its operation kind",
9485 )
9486 })?,
9487 operation_identity: payload_string(&event.payload, "operation_identity").ok_or_else(
9488 || {
9489 invalid_recorded_history(
9490 "selection_cancellation_invalid",
9491 member_base_sequence,
9492 "selection operation identity",
9493 &event.payload.to_string(),
9494 "selection cancellation is missing its operation identity",
9495 )
9496 },
9497 )?,
9498 };
9499 if let Some(existing) = cancelled.iter().find(|recorded| {
9500 recorded.selection_group_id == marker.selection_group_id
9501 && recorded.member_base_sequence == marker.member_base_sequence
9502 }) {
9503 if existing != &marker {
9504 return Err(invalid_recorded_history(
9505 "selection_cancellation_conflict",
9506 member_base_sequence,
9507 "one stable SelectionOperationCancelled marker",
9508 &event.payload.to_string(),
9509 "selection cancellation history contains conflicting member metadata",
9510 ));
9511 }
9512 } else {
9513 cancelled.push(marker);
9514 }
9515 }
9516 Ok(cancelled)
9517}
9518
9519fn required_selection_u64(payload: &Value, field: &str) -> Result<u64> {
9520 payload
9521 .get(field)
9522 .and_then(value_as_u64)
9523 .filter(|value| *value > 0)
9524 .ok_or_else(|| {
9525 invalid_recorded_history(
9526 "selection_marker_invalid",
9527 0,
9528 &format!("positive integer {field}"),
9529 &payload.to_string(),
9530 "selection history contains invalid durable identity metadata",
9531 )
9532 })
9533}
9534
9535fn required_selection_usize(payload: &Value, field: &str) -> Result<usize> {
9536 required_selection_usize_allow_zero(payload, field).and_then(|value| {
9537 if value > 0 {
9538 Ok(value)
9539 } else {
9540 Err(invalid_recorded_history(
9541 "selection_marker_invalid",
9542 0,
9543 &format!("positive integer {field}"),
9544 &payload.to_string(),
9545 "selection history contains invalid durable identity metadata",
9546 ))
9547 }
9548 })
9549}
9550
9551fn required_selection_usize_allow_zero(payload: &Value, field: &str) -> Result<usize> {
9552 payload
9553 .get(field)
9554 .and_then(value_as_u64)
9555 .and_then(|value| usize::try_from(value).ok())
9556 .ok_or_else(|| {
9557 invalid_recorded_history(
9558 "selection_marker_invalid",
9559 0,
9560 &format!("non-negative integer {field}"),
9561 &payload.to_string(),
9562 "selection history contains invalid durable identity metadata",
9563 )
9564 })
9565}
9566
9567#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9568struct RecordedActivityOptions {
9569 task_queue: RecordedSnapshotValue<Option<String>>,
9570 execution_mode: RecordedSnapshotValue<Option<String>>,
9571 retry_policy: ActivityRetrySnapshot,
9572}
9573
9574#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9575enum RecordedSnapshotValue<T> {
9576 Unknown,
9578 Known(T),
9579}
9580
9581impl<T: PartialEq> RecordedSnapshotValue<T> {
9582 fn matches_current(&self, current: &Self) -> bool {
9583 match self {
9584 Self::Unknown => true,
9585 Self::Known(recorded) => matches!(current, Self::Known(value) if value == recorded),
9586 }
9587 }
9588}
9589
9590#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9591struct ActivityRetrySnapshot {
9592 snapshot_version: RecordedSnapshotValue<Option<u64>>,
9593 max_attempts: RecordedSnapshotValue<Option<u64>>,
9594 backoff_seconds: RecordedSnapshotValue<Vec<u64>>,
9595 start_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
9596 schedule_to_start_timeout: RecordedSnapshotValue<Option<u64>>,
9597 schedule_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
9598 heartbeat_timeout: RecordedSnapshotValue<Option<u64>>,
9599 non_retryable_error_types: RecordedSnapshotValue<Vec<String>>,
9600}
9601
9602impl ActivityRetrySnapshot {
9603 fn matches_current(&self, current: &Self) -> bool {
9604 self.snapshot_version
9605 .matches_current(¤t.snapshot_version)
9606 && self.max_attempts.matches_current(¤t.max_attempts)
9607 && self
9608 .backoff_seconds
9609 .matches_current(¤t.backoff_seconds)
9610 && self
9611 .start_to_close_timeout
9612 .matches_current(¤t.start_to_close_timeout)
9613 && self
9614 .schedule_to_start_timeout
9615 .matches_current(¤t.schedule_to_start_timeout)
9616 && self
9617 .schedule_to_close_timeout
9618 .matches_current(¤t.schedule_to_close_timeout)
9619 && self
9620 .heartbeat_timeout
9621 .matches_current(¤t.heartbeat_timeout)
9622 && self
9623 .non_retryable_error_types
9624 .matches_current(¤t.non_retryable_error_types)
9625 }
9626}
9627
9628fn recorded_optional_u64(
9629 object: Option<&serde_json::Map<String, Value>>,
9630 field: &str,
9631) -> RecordedSnapshotValue<Option<u64>> {
9632 match object.and_then(|object| object.get(field)) {
9633 None => RecordedSnapshotValue::Unknown,
9634 Some(Value::Null) => RecordedSnapshotValue::Known(None),
9635 Some(value) => RecordedSnapshotValue::Known(value_as_u64(value)),
9636 }
9637}
9638
9639fn recorded_optional_string(
9640 object: &serde_json::Map<String, Value>,
9641 field: &str,
9642) -> RecordedSnapshotValue<Option<String>> {
9643 match object.get(field) {
9644 None => RecordedSnapshotValue::Unknown,
9645 Some(Value::Null) => RecordedSnapshotValue::Known(None),
9646 Some(value) => RecordedSnapshotValue::Known(value.as_str().map(str::to_string)),
9647 }
9648}
9649
9650fn recorded_activity_retry_snapshot(policy: Option<&Value>) -> ActivityRetrySnapshot {
9651 let policy = policy.and_then(Value::as_object);
9652 let backoff_seconds = policy
9653 .and_then(|policy| policy.get("backoff_seconds"))
9654 .and_then(Value::as_array)
9655 .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
9656 .map_or(RecordedSnapshotValue::Unknown, RecordedSnapshotValue::Known);
9657 let mut non_retryable_error_types = Vec::new();
9658 for error_type in policy
9659 .and_then(|policy| policy.get("non_retryable_error_types"))
9660 .and_then(Value::as_array)
9661 .into_iter()
9662 .flatten()
9663 .filter_map(Value::as_str)
9664 .map(str::trim)
9665 .filter(|error_type| !error_type.is_empty())
9666 {
9667 if !non_retryable_error_types
9668 .iter()
9669 .any(|recorded| recorded == error_type)
9670 {
9671 non_retryable_error_types.push(error_type.to_string());
9672 }
9673 }
9674
9675 ActivityRetrySnapshot {
9676 snapshot_version: recorded_optional_u64(policy, "snapshot_version"),
9677 max_attempts: recorded_optional_u64(policy, "max_attempts"),
9678 backoff_seconds,
9679 start_to_close_timeout: recorded_optional_u64(policy, "start_to_close_timeout"),
9680 schedule_to_start_timeout: recorded_optional_u64(policy, "schedule_to_start_timeout"),
9681 schedule_to_close_timeout: recorded_optional_u64(policy, "schedule_to_close_timeout"),
9682 heartbeat_timeout: recorded_optional_u64(policy, "heartbeat_timeout"),
9683 non_retryable_error_types: if policy
9684 .is_some_and(|policy| policy.contains_key("non_retryable_error_types"))
9685 {
9686 RecordedSnapshotValue::Known(non_retryable_error_types)
9687 } else {
9688 RecordedSnapshotValue::Unknown
9689 },
9690 }
9691}
9692
9693fn current_activity_retry_snapshot(options: &ValidatedActivityOptions) -> ActivityRetrySnapshot {
9694 let policy = options.retry_policy.as_ref();
9695 let max_attempts = match policy.and_then(|policy| policy.get("max_attempts")) {
9696 Some(Value::Null) => None,
9697 Some(value) => value_as_u64(value),
9698 None => Some(1),
9699 };
9700 let backoff_seconds = policy
9701 .and_then(|policy| policy.get("backoff_seconds"))
9702 .and_then(Value::as_array)
9703 .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
9704 .unwrap_or_default();
9705 let non_retryable_error_types = policy
9706 .and_then(|policy| policy.get("non_retryable_error_types"))
9707 .and_then(Value::as_array)
9708 .into_iter()
9709 .flatten()
9710 .filter_map(Value::as_str)
9711 .map(str::to_string)
9712 .collect();
9713
9714 ActivityRetrySnapshot {
9715 snapshot_version: RecordedSnapshotValue::Known(Some(1)),
9716 max_attempts: RecordedSnapshotValue::Known(max_attempts),
9717 backoff_seconds: RecordedSnapshotValue::Known(backoff_seconds),
9718 start_to_close_timeout: RecordedSnapshotValue::Known(options.start_to_close_timeout),
9719 schedule_to_start_timeout: RecordedSnapshotValue::Known(options.schedule_to_start_timeout),
9720 schedule_to_close_timeout: RecordedSnapshotValue::Known(options.schedule_to_close_timeout),
9721 heartbeat_timeout: RecordedSnapshotValue::Known(options.heartbeat_timeout),
9722 non_retryable_error_types: RecordedSnapshotValue::Known(non_retryable_error_types),
9723 }
9724}
9725
9726fn activity_options_description(options: &RecordedActivityOptions) -> String {
9727 serde_json::to_string(options).unwrap_or_else(|_| format!("{options:?}"))
9728}
9729
9730impl RecordedCommand {
9731 fn sequence(&self) -> u64 {
9732 match self {
9733 Self::Activity { sequence, .. }
9734 | Self::Timer { sequence, .. }
9735 | Self::ChildWorkflow { sequence, .. }
9736 | Self::SignalWait { sequence, .. }
9737 | Self::ConditionWait { sequence, .. }
9738 | Self::SearchAttributes { sequence, .. }
9739 | Self::SideEffect { sequence, .. }
9740 | Self::VersionMarker { sequence, .. }
9741 | Self::Memo { sequence, .. } => *sequence,
9742 }
9743 }
9744
9745 fn shape(&self) -> &'static str {
9746 match self {
9747 Self::Activity { .. } => "activity",
9748 Self::Timer { .. } => "timer",
9749 Self::ChildWorkflow { .. } => "child workflow",
9750 Self::SignalWait { .. } => "signal wait",
9751 Self::ConditionWait { .. } => "condition wait",
9752 Self::SearchAttributes { .. } => "search-attribute update",
9753 Self::SideEffect { .. } => "side effect",
9754 Self::VersionMarker { .. } => "version marker",
9755 Self::Memo { .. } => "memo upsert",
9756 }
9757 }
9758}
9759
9760fn ensure_version_supported(
9761 change_id: &str,
9762 version: i32,
9763 min_supported: i32,
9764 max_supported: i32,
9765 sequence: u64,
9766) -> Result<()> {
9767 if (min_supported..=max_supported).contains(&version) {
9768 return Ok(());
9769 }
9770 Err(Error::NonDeterministicReplay(ReplayFailure::new(
9771 "version_marker_incompatible_range",
9772 (sequence != 0).then_some(sequence),
9773 Some(format!("{min_supported}..={max_supported}")),
9774 Some(format!("{change_id}:{version}")),
9775 "recorded workflow version is outside the range supported by current code",
9776 )))
9777}
9778
9779#[derive(Clone, Debug)]
9780struct ResumeSignal {
9781 signal_name: String,
9782 arguments: Vec<AvroValue>,
9783}
9784
9785const MAX_PARALLEL_OPERATIONS: usize = 1000;
9786
9787fn parallel_group_prefix(kind: &str) -> &'static str {
9788 match kind {
9789 "activity" => "parallel-activities",
9790 "child" => "parallel-children",
9791 "timer" => "parallel-timers",
9792 _ => "parallel-calls",
9793 }
9794}
9795
9796fn parallel_group_entry(
9797 base_sequence: u64,
9798 size: usize,
9799 index: usize,
9800 kind: &str,
9801) -> ParallelGroupMetadata {
9802 ParallelGroupMetadata {
9803 parallel_group_id: format!("{}:{base_sequence}:{size}", parallel_group_prefix(kind)),
9804 parallel_group_kind: kind.to_string(),
9805 parallel_group_base_sequence: base_sequence,
9806 parallel_group_size: size,
9807 parallel_group_index: index,
9808 parallel_group_mode: None,
9809 selection_member_key: None,
9810 selection_member_index: None,
9811 selection_member_base_sequence: None,
9812 selection_member_size: None,
9813 selection_member_kind: None,
9814 }
9815}
9816
9817struct SelectionMemberMetadata {
9818 key: SelectionKey,
9819 index: usize,
9820 base_sequence: u64,
9821 size: usize,
9822 kind: String,
9823}
9824
9825fn selection_group_entry(
9826 base_sequence: u64,
9827 size: usize,
9828 index: usize,
9829 kind: &str,
9830 member: &SelectionMemberMetadata,
9831) -> ParallelGroupMetadata {
9832 ParallelGroupMetadata {
9833 parallel_group_id: format!("select-calls:{base_sequence}:{size}"),
9834 parallel_group_kind: kind.to_string(),
9835 parallel_group_base_sequence: base_sequence,
9836 parallel_group_size: size,
9837 parallel_group_index: index,
9838 parallel_group_mode: Some("select".to_string()),
9839 selection_member_key: Some(member.key.clone()),
9840 selection_member_index: Some(member.index),
9841 selection_member_base_sequence: Some(member.base_sequence),
9842 selection_member_size: Some(member.size),
9843 selection_member_kind: Some(member.kind.clone()),
9844 }
9845}
9846
9847fn apply_parallel_group_path(
9848 command: &mut serde_json::Map<String, Value>,
9849 path: &[ParallelGroupMetadata],
9850) {
9851 let Some(inner) = path.last() else {
9852 return;
9853 };
9854 command.insert(
9855 "parallel_group_id".to_string(),
9856 json!(inner.parallel_group_id),
9857 );
9858 command.insert(
9859 "parallel_group_kind".to_string(),
9860 json!(inner.parallel_group_kind),
9861 );
9862 command.insert(
9863 "parallel_group_base_sequence".to_string(),
9864 json!(inner.parallel_group_base_sequence),
9865 );
9866 command.insert(
9867 "parallel_group_size".to_string(),
9868 json!(inner.parallel_group_size),
9869 );
9870 command.insert(
9871 "parallel_group_index".to_string(),
9872 json!(inner.parallel_group_index),
9873 );
9874 if let Some(mode) = &inner.parallel_group_mode {
9875 command.insert("parallel_group_mode".to_string(), json!(mode));
9876 }
9877 if let Some(key) = &inner.selection_member_key {
9878 command.insert("selection_member_key".to_string(), json!(key));
9879 }
9880 if let Some(index) = inner.selection_member_index {
9881 command.insert("selection_member_index".to_string(), json!(index));
9882 }
9883 if let Some(base_sequence) = inner.selection_member_base_sequence {
9884 command.insert(
9885 "selection_member_base_sequence".to_string(),
9886 json!(base_sequence),
9887 );
9888 }
9889 if let Some(size) = inner.selection_member_size {
9890 command.insert("selection_member_size".to_string(), json!(size));
9891 }
9892 if let Some(kind) = &inner.selection_member_kind {
9893 command.insert("selection_member_kind".to_string(), json!(kind));
9894 }
9895 command.insert("parallel_group_path".to_string(), json!(path));
9896}
9897
9898fn ensure_parallel_path_matches(
9899 sequence: u64,
9900 recorded: Option<&[ParallelGroupMetadata]>,
9901 expected: &[ParallelGroupMetadata],
9902) -> Result<()> {
9903 match (recorded, expected.is_empty()) {
9904 (None, true) => Ok(()),
9905 (Some(recorded), false) if recorded == expected => Ok(()),
9906 (None, false) => Err(invalid_recorded_history(
9907 "parallel_group_metadata_missing",
9908 sequence,
9909 &serde_json::to_string(expected).unwrap_or_default(),
9910 "<missing>",
9911 "recorded parallel member is missing its durable group path",
9912 )),
9913 (Some(recorded), true) => Err(invalid_recorded_history(
9914 "parallel_group_shape_mismatch",
9915 sequence,
9916 "sequential command",
9917 &serde_json::to_string(recorded).unwrap_or_default(),
9918 "recorded command belonged to a parallel group but current code schedules it sequentially",
9919 )),
9920 (Some(recorded), false) => Err(invalid_recorded_history(
9921 "parallel_group_shape_mismatch",
9922 sequence,
9923 &serde_json::to_string(recorded).unwrap_or_default(),
9924 &serde_json::to_string(expected).unwrap_or_default(),
9925 "recorded parallel-group identity or path changed during replay",
9926 )),
9927 }
9928}
9929
9930#[derive(Clone, Debug)]
9931enum ParallelShape {
9932 Leaf,
9933 Group(Vec<ParallelShape>),
9934}
9935
9936struct ParallelDescriptor {
9937 operation: ParallelOperation,
9938 offset: usize,
9939 member_path: Vec<usize>,
9940 group_path: Vec<ParallelGroupMetadata>,
9941}
9942
9943fn parallel_leaf_count(operations: &[ParallelOperation]) -> usize {
9944 operations
9945 .iter()
9946 .map(|operation| match operation {
9947 ParallelOperation::Group(children) => parallel_leaf_count(children),
9948 _ => 1,
9949 })
9950 .sum()
9951}
9952
9953fn parallel_operation_kind(operation: &ParallelOperation) -> Option<&'static str> {
9954 match operation {
9955 ParallelOperation::Activity { .. } => Some("activity"),
9956 ParallelOperation::ChildWorkflow { .. } => Some("child"),
9957 ParallelOperation::Timer(_) => Some("timer"),
9958 ParallelOperation::Signal(_) => Some("signal"),
9959 ParallelOperation::Condition { .. } => Some("condition"),
9960 ParallelOperation::Group(children) => parallel_group_kind(children),
9961 }
9962}
9963
9964fn parallel_group_kind(operations: &[ParallelOperation]) -> Option<&'static str> {
9965 let mut kind = None;
9966 for operation in operations {
9967 let Some(operation_kind) = parallel_operation_kind(operation) else {
9968 continue;
9969 };
9970 match kind {
9971 None => kind = Some(operation_kind),
9972 Some(current) if current == operation_kind => {}
9973 Some(_) => return Some("mixed"),
9974 }
9975 }
9976 kind
9977}
9978
9979fn validate_parallel_operations(
9980 operations: &[ParallelOperation],
9981 member_path: &mut Vec<usize>,
9982 root: bool,
9983) -> Result<()> {
9984 let leaves = parallel_leaf_count(operations);
9985 if leaves > MAX_PARALLEL_OPERATIONS {
9986 return Err(Error::InvalidParallelGroup(ParallelGroupError {
9987 reason: "fan_out_limit_exceeded",
9988 member_path: member_path.clone(),
9989 message: format!(
9990 "group contains {leaves} durable leaves; the limit is {MAX_PARALLEL_OPERATIONS}"
9991 ),
9992 }));
9993 }
9994 if !root && operations.is_empty() {
9995 return Err(Error::InvalidParallelGroup(ParallelGroupError {
9996 reason: "nested_group_empty",
9997 member_path: member_path.clone(),
9998 message: "a nested group must contain at least one durable leaf".to_string(),
9999 }));
10000 }
10001
10002 for (index, operation) in operations.iter().enumerate() {
10003 member_path.push(index);
10004 match operation {
10005 ParallelOperation::Activity {
10006 options, arguments, ..
10007 } => {
10008 options
10009 .validate()
10010 .map_err(|error| Error::InvalidActivityOptions(error))?;
10011 if let Err(error) = arguments {
10012 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10013 reason: "arguments_invalid",
10014 member_path: member_path.clone(),
10015 message: error.to_string(),
10016 }));
10017 }
10018 }
10019 ParallelOperation::ChildWorkflow {
10020 options, arguments, ..
10021 } => {
10022 validate_parallel_child_options(options)?;
10023 if let Err(error) = arguments {
10024 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10025 reason: "arguments_invalid",
10026 member_path: member_path.clone(),
10027 message: error.to_string(),
10028 }));
10029 }
10030 }
10031 ParallelOperation::Timer(duration)
10032 if duration.as_secs() == u64::MAX && duration.subsec_nanos() > 0 =>
10033 {
10034 return Err(Error::TimerDurationOverflow);
10035 }
10036 ParallelOperation::Timer(_) => {}
10037 ParallelOperation::Signal(signal_name) => {
10038 validate_user_signal_name(signal_name)?;
10039 if signal_name.trim().is_empty() {
10040 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10041 reason: "signal_name_empty",
10042 member_path: member_path.clone(),
10043 message: "signal wait name must not be empty".to_string(),
10044 }));
10045 }
10046 }
10047 ParallelOperation::Condition { options, .. } => {
10048 options.validate()?;
10049 }
10050 ParallelOperation::Group(children) => {
10051 validate_parallel_operations(children, member_path, false)?;
10052 }
10053 }
10054 member_path.pop();
10055 }
10056 Ok(())
10057}
10058
10059fn validate_parallel_child_options(options: &ChildWorkflowOptions) -> Result<()> {
10060 if options.task_queue.trim().is_empty() {
10061 return Err(Error::InvalidChildWorkflowOptions(
10062 "task_queue must not be empty".to_string(),
10063 ));
10064 }
10065 for (name, value) in [
10066 (
10067 "execution_timeout_seconds",
10068 options.execution_timeout_seconds,
10069 ),
10070 ("run_timeout_seconds", options.run_timeout_seconds),
10071 ] {
10072 if value == Some(0) {
10073 return Err(Error::InvalidChildWorkflowOptions(format!(
10074 "{name} must be at least 1"
10075 )));
10076 }
10077 }
10078 if options
10079 .retry_policy
10080 .as_ref()
10081 .is_some_and(|policy| policy.max_attempts == Some(0))
10082 {
10083 return Err(Error::InvalidChildWorkflowOptions(
10084 "retry_policy.max_attempts must be at least 1".to_string(),
10085 ));
10086 }
10087 Ok(())
10088}
10089
10090fn parallel_shape(operations: &[ParallelOperation]) -> ParallelShape {
10091 ParallelShape::Group(
10092 operations
10093 .iter()
10094 .map(|operation| match operation {
10095 ParallelOperation::Group(children) => parallel_shape(children),
10096 _ => ParallelShape::Leaf,
10097 })
10098 .collect(),
10099 )
10100}
10101
10102fn parallel_descriptors(
10103 operations: Vec<ParallelOperation>,
10104 base_sequence: u64,
10105) -> Result<Vec<ParallelDescriptor>> {
10106 let size = parallel_leaf_count(&operations);
10107 let kind = parallel_group_kind(&operations).unwrap_or("activity");
10108 let mut descriptors = Vec::with_capacity(size);
10109 let mut cursor = 0;
10110
10111 for (index, operation) in operations.into_iter().enumerate() {
10112 match operation {
10113 ParallelOperation::Group(children) => {
10114 let child_base = base_sequence
10115 .checked_add(u64::try_from(cursor).unwrap_or(u64::MAX))
10116 .ok_or(Error::TimerDurationOverflow)?;
10117 for mut descriptor in parallel_descriptors(children, child_base)? {
10118 let outer_index = cursor + descriptor.offset;
10119 descriptor.group_path.insert(
10120 0,
10121 parallel_group_entry(base_sequence, size, outer_index, kind),
10122 );
10123 descriptor.member_path.insert(0, index);
10124 descriptor.offset = outer_index;
10125 descriptors.push(descriptor);
10126 }
10127 cursor = descriptors.len();
10128 }
10129 operation => {
10130 descriptors.push(ParallelDescriptor {
10131 operation,
10132 offset: cursor,
10133 member_path: vec![index],
10134 group_path: vec![parallel_group_entry(base_sequence, size, cursor, kind)],
10135 });
10136 cursor += 1;
10137 }
10138 }
10139 }
10140 Ok(descriptors)
10141}
10142
10143enum ParallelLeafCall {
10144 Activity(ActivityCall),
10145 ChildWorkflow(ChildWorkflowCall),
10146 Timer(TimerCall),
10147 Signal(SignalCall),
10148 Condition(ConditionWaitCall),
10149}
10150
10151fn parallel_leaf_call(
10152 ctx: &WorkflowContext,
10153 operation: ParallelOperation,
10154 parallel_group_path: Vec<ParallelGroupMetadata>,
10155) -> ParallelLeafCall {
10156 match operation {
10157 ParallelOperation::Activity {
10158 activity_type,
10159 options,
10160 arguments,
10161 } => ParallelLeafCall::Activity(ActivityCall {
10162 ctx: ctx.clone(),
10163 activity_type,
10164 options,
10165 args: Some(arguments),
10166 scheduled: false,
10167 parallel_group_path,
10168 }),
10169 ParallelOperation::ChildWorkflow {
10170 workflow_type,
10171 options,
10172 arguments,
10173 } => ParallelLeafCall::ChildWorkflow(ChildWorkflowCall {
10174 ctx: ctx.clone(),
10175 workflow_type,
10176 options,
10177 args: Some(arguments),
10178 scheduled: false,
10179 matched_pending: false,
10180 parallel_group_path,
10181 }),
10182 ParallelOperation::Timer(duration) => {
10183 let delay_seconds = duration
10184 .as_secs()
10185 .checked_add(u64::from(duration.subsec_nanos() > 0));
10186 ParallelLeafCall::Timer(TimerCall {
10187 ctx: ctx.clone(),
10188 delay_seconds,
10189 scheduled: false,
10190 matched_pending: false,
10191 parallel_group_path,
10192 })
10193 }
10194 ParallelOperation::Signal(signal_name) => ParallelLeafCall::Signal(SignalCall {
10195 ctx: ctx.clone(),
10196 signal_name,
10197 runtime_reserved_allowed: false,
10198 opened_wait: false,
10199 matched_pending: false,
10200 parallel_group_path,
10201 }),
10202 ParallelOperation::Condition { options, predicate } => {
10203 ParallelLeafCall::Condition(ConditionWaitCall {
10204 ctx: ctx.clone(),
10205 options,
10206 predicate,
10207 occurrence_id: None,
10208 opened_wait: false,
10209 parallel_group_path,
10210 })
10211 }
10212 ParallelOperation::Group(_) => {
10213 unreachable!("parallel descriptors contain only durable leaves")
10214 }
10215 }
10216}
10217
10218impl ParallelLeafCall {
10219 fn poll_avro_value(&mut self, cx: &mut TaskContext<'_>) -> Poll<Result<ParallelAvroResult>> {
10220 match self {
10221 Self::Activity(call) => Pin::new(call)
10222 .poll_avro_value(cx)
10223 .map_ok(ParallelAvroResult::Activity),
10224 Self::ChildWorkflow(call) => Pin::new(call)
10225 .poll_avro_value(cx)
10226 .map_ok(ParallelAvroResult::ChildWorkflow),
10227 Self::Timer(call) => Pin::new(call)
10228 .poll(cx)
10229 .map_ok(|()| ParallelAvroResult::Timer),
10230 Self::Signal(call) => Pin::new(call)
10231 .poll_avro_value(cx)
10232 .map_ok(ParallelAvroResult::Signal),
10233 Self::Condition(call) => Pin::new(call)
10234 .poll(cx)
10235 .map_ok(ParallelAvroResult::Condition),
10236 }
10237 }
10238}
10239
10240struct ParallelLeaf {
10241 call: ParallelLeafCall,
10242 member_path: Vec<usize>,
10243 group_path: Vec<ParallelGroupMetadata>,
10244 result: Option<ParallelAvroResult>,
10245}
10246
10247pub struct ParallelCall {
10249 ctx: WorkflowContext,
10250 operations: Option<Vec<ParallelOperation>>,
10251 shape: Option<ParallelShape>,
10252 leaves: Vec<ParallelLeaf>,
10253}
10254
10255impl ParallelCall {
10256 fn new(ctx: WorkflowContext, operations: Vec<ParallelOperation>) -> Self {
10257 Self {
10258 ctx,
10259 operations: Some(operations),
10260 shape: None,
10261 leaves: Vec::new(),
10262 }
10263 }
10264
10265 fn initialize(&mut self) -> Result<()> {
10266 let operations = self.operations.take().unwrap_or_default();
10267 validate_parallel_operations(&operations, &mut Vec::new(), true)?;
10268 self.shape = Some(parallel_shape(&operations));
10269 if operations.is_empty() {
10270 return Ok(());
10271 }
10272
10273 let base_sequence = {
10274 let state = self
10275 .ctx
10276 .state
10277 .lock()
10278 .map_err(|_| Error::WorkflowStatePoisoned)?;
10279 if let Some(recorded) = state.recorded_commands.get(state.command_cursor) {
10280 recorded.sequence()
10281 } else {
10282 let last = state
10283 .recorded_commands
10284 .last()
10285 .map(RecordedCommand::sequence)
10286 .unwrap_or(0);
10287 last.checked_add(u64::try_from(state.commands.len()).unwrap_or(u64::MAX))
10288 .and_then(|sequence| sequence.checked_add(1))
10289 .ok_or_else(|| {
10290 Error::InvalidParallelGroup(ParallelGroupError {
10291 reason: "sequence_overflow",
10292 member_path: Vec::new(),
10293 message: "parallel group sequence identity overflowed u64".to_string(),
10294 })
10295 })?
10296 }
10297 };
10298
10299 self.leaves = parallel_descriptors(operations, base_sequence)?
10300 .into_iter()
10301 .map(|descriptor| {
10302 let call = parallel_leaf_call(
10303 &self.ctx,
10304 descriptor.operation,
10305 descriptor.group_path.clone(),
10306 );
10307 ParallelLeaf {
10308 call,
10309 member_path: descriptor.member_path,
10310 group_path: descriptor.group_path,
10311 result: None,
10312 }
10313 })
10314 .collect();
10315 Ok(())
10316 }
10317
10318 fn poll_avro_value(
10319 mut self: Pin<&mut Self>,
10320 cx: &mut TaskContext<'_>,
10321 ) -> Poll<Result<Vec<ParallelAvroResult>>> {
10322 if self.operations.is_some() {
10323 if let Err(error) = self.initialize() {
10324 return Poll::Ready(Err(error));
10325 }
10326 }
10327 if self.leaves.is_empty() {
10328 return Poll::Ready(Ok(Vec::new()));
10329 }
10330
10331 let mut failures = Vec::new();
10332 let mut pending = false;
10333 for (index, leaf) in self.leaves.iter_mut().enumerate() {
10334 if leaf.result.is_some() {
10335 continue;
10336 }
10337 match leaf.call.poll_avro_value(cx) {
10338 Poll::Ready(Ok(result)) => leaf.result = Some(result),
10339 Poll::Ready(Err(error)) => failures.push((index, error)),
10340 Poll::Pending => pending = true,
10341 }
10342 }
10343
10344 if !failures.is_empty() {
10345 if let Some(position) = failures
10346 .iter()
10347 .position(|(_, error)| workflow_task_integrity_error(error))
10348 {
10349 return Poll::Ready(Err(failures.remove(position).1));
10350 }
10351 failures.sort_by_key(|(index, _)| *index);
10352 let (failed_index, cause) = failures.remove(0);
10353 let failed = &self.leaves[failed_index];
10354 let completed = self
10355 .leaves
10356 .iter()
10357 .filter_map(|leaf| {
10358 leaf.result
10359 .clone()
10360 .and_then(|result| result.into_json_result().ok())
10361 .map(|result| ParallelCompletion {
10362 member_path: leaf.member_path.clone(),
10363 result,
10364 })
10365 })
10366 .collect();
10367 let group_id = failed
10368 .group_path
10369 .first()
10370 .map(|entry| entry.parallel_group_id.clone())
10371 .unwrap_or_default();
10372 return Poll::Ready(Err(Error::ParallelFailed(ParallelFailure {
10373 group_id,
10374 member_path: failed.member_path.clone(),
10375 group_path: failed.group_path.clone(),
10376 completed,
10377 cause: Box::new(cause),
10378 })));
10379 }
10380 if pending {
10381 return Poll::Pending;
10382 }
10383
10384 let mut flat_results = self
10385 .leaves
10386 .iter_mut()
10387 .map(|leaf| leaf.result.take().expect("completed parallel leaf"))
10388 .collect::<Vec<_>>()
10389 .into_iter();
10390 let results = parallel_results_for_shape(
10391 self.shape.as_ref().expect("initialized parallel shape"),
10392 &mut flat_results,
10393 );
10394 Poll::Ready(Ok(match results {
10395 ParallelAvroResult::Group(results) => results,
10396 ParallelAvroResult::Activity(_)
10397 | ParallelAvroResult::ChildWorkflow(_)
10398 | ParallelAvroResult::Timer
10399 | ParallelAvroResult::Signal(_)
10400 | ParallelAvroResult::Condition(_) => {
10401 unreachable!("root parallel shape is a group")
10402 }
10403 }))
10404 }
10405}
10406
10407fn parallel_results_for_shape(
10408 shape: &ParallelShape,
10409 flat_results: &mut impl Iterator<Item = ParallelAvroResult>,
10410) -> ParallelAvroResult {
10411 match shape {
10412 ParallelShape::Leaf => flat_results.next().expect("one result per parallel leaf"),
10413 ParallelShape::Group(children) => ParallelAvroResult::Group(
10414 children
10415 .iter()
10416 .map(|child| parallel_results_for_shape(child, flat_results))
10417 .collect(),
10418 ),
10419 }
10420}
10421
10422impl Future for ParallelCall {
10423 type Output = Result<Vec<ParallelResult>>;
10424
10425 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
10426 self.poll_avro_value(cx)
10427 .map_ok(|results| {
10428 results
10429 .into_iter()
10430 .map(ParallelAvroResult::into_json_result)
10431 .collect::<Result<Vec<_>>>()
10432 })
10433 .map_ok(|result| result)
10434 .flatten_result()
10435 }
10436}
10437
10438#[derive(Clone, Debug)]
10439struct SelectionMemberPlan {
10440 key: SelectionKey,
10441 index: usize,
10442 base_sequence: u64,
10443 size: usize,
10444 kind: String,
10445 shape: ParallelShape,
10446 leaf_start: usize,
10447}
10448
10449fn selection_operation_kind(operation: &ParallelOperation) -> &'static str {
10450 match operation {
10451 ParallelOperation::Activity { .. } => "activity",
10452 ParallelOperation::ChildWorkflow { .. } => "child",
10453 ParallelOperation::Timer(_) => "timer",
10454 ParallelOperation::Signal(_) => "signal",
10455 ParallelOperation::Condition { .. } => "condition",
10456 ParallelOperation::Group(_) => "group",
10457 }
10458}
10459
10460fn selection_operation_shape(operation: &ParallelOperation) -> ParallelShape {
10461 match operation {
10462 ParallelOperation::Group(children) => parallel_shape(children),
10463 _ => ParallelShape::Leaf,
10464 }
10465}
10466
10467fn selection_descriptors(
10468 operations: Vec<(SelectionKey, ParallelOperation)>,
10469 base_sequence: u64,
10470) -> Result<(Vec<ParallelDescriptor>, Vec<SelectionMemberPlan>)> {
10471 if operations.is_empty() {
10472 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10473 reason: "selection_empty",
10474 member_path: Vec::new(),
10475 message: "durable selection requires at least one operation".to_string(),
10476 }));
10477 }
10478 let operation_refs = operations
10479 .iter()
10480 .map(|(_, operation)| operation)
10481 .collect::<Vec<_>>();
10482 let total_size = operation_refs
10483 .iter()
10484 .map(|operation| match operation {
10485 ParallelOperation::Group(children) => parallel_leaf_count(children),
10486 _ => 1,
10487 })
10488 .sum::<usize>();
10489 if total_size > MAX_PARALLEL_OPERATIONS {
10490 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10491 reason: "fan_out_limit_exceeded",
10492 member_path: Vec::new(),
10493 message: format!(
10494 "selection contains {total_size} durable leaves; the limit is {MAX_PARALLEL_OPERATIONS}"
10495 ),
10496 }));
10497 }
10498 let group_kind = {
10499 let mut kind = None;
10500 for operation in &operation_refs {
10501 let operation_kind = parallel_operation_kind(operation).unwrap_or("mixed");
10502 match kind {
10503 None => kind = Some(operation_kind),
10504 Some(current) if current == operation_kind => {}
10505 Some(_) => {
10506 kind = Some("mixed");
10507 break;
10508 }
10509 }
10510 }
10511 kind.unwrap_or("mixed")
10512 };
10513
10514 let mut descriptors = Vec::with_capacity(total_size);
10515 let mut members = Vec::with_capacity(operations.len());
10516 let mut cursor = 0usize;
10517 let mut seen_keys: Vec<SelectionKey> = Vec::new();
10518 for (member_index, (key, operation)) in operations.into_iter().enumerate() {
10519 if matches!(&key, SelectionKey::Name(value) if value.is_empty()) {
10520 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10521 reason: "selection_key_invalid",
10522 member_path: vec![member_index],
10523 message: "selection member keys must be non-empty strings or non-negative integers"
10524 .to_string(),
10525 }));
10526 }
10527 if seen_keys.contains(&key) {
10528 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10529 reason: "selection_key_duplicate",
10530 member_path: vec![member_index],
10531 message: format!("selection member key {key:?} is duplicated"),
10532 }));
10533 }
10534 seen_keys.push(key.clone());
10535 let member_size = match &operation {
10536 ParallelOperation::Group(children) => parallel_leaf_count(children),
10537 _ => 1,
10538 };
10539 if member_size == 0 {
10540 return Err(Error::InvalidParallelGroup(ParallelGroupError {
10541 reason: "selection_member_empty",
10542 member_path: vec![member_index],
10543 message: "a selection member must contain at least one durable leaf".to_string(),
10544 }));
10545 }
10546 let member_base = base_sequence
10547 .checked_add(u64::try_from(cursor).unwrap_or(u64::MAX))
10548 .ok_or(Error::TimerDurationOverflow)?;
10549 let member_kind = selection_operation_kind(&operation).to_string();
10550 let member_shape = selection_operation_shape(&operation);
10551 let leaf_start = descriptors.len();
10552 match operation {
10553 ParallelOperation::Group(children) => {
10554 validate_parallel_operations(&children, &mut vec![member_index], false)?;
10555 for mut descriptor in parallel_descriptors(children, member_base)? {
10556 let flat_index = cursor + descriptor.offset;
10557 descriptor.group_path.insert(
10558 0,
10559 selection_group_entry(
10560 base_sequence,
10561 total_size,
10562 flat_index,
10563 group_kind,
10564 &SelectionMemberMetadata {
10565 key: key.clone(),
10566 index: member_index,
10567 base_sequence: member_base,
10568 size: member_size,
10569 kind: member_kind.clone(),
10570 },
10571 ),
10572 );
10573 descriptor.member_path.insert(0, member_index);
10574 descriptor.offset = flat_index;
10575 descriptors.push(descriptor);
10576 }
10577 }
10578 operation => {
10579 validate_parallel_operations(
10580 std::slice::from_ref(&operation),
10581 &mut Vec::new(),
10582 true,
10583 )?;
10584 descriptors.push(ParallelDescriptor {
10585 operation,
10586 offset: cursor,
10587 member_path: vec![member_index],
10588 group_path: vec![selection_group_entry(
10589 base_sequence,
10590 total_size,
10591 cursor,
10592 group_kind,
10593 &SelectionMemberMetadata {
10594 key: key.clone(),
10595 index: member_index,
10596 base_sequence: member_base,
10597 size: member_size,
10598 kind: member_kind.clone(),
10599 },
10600 )],
10601 });
10602 }
10603 }
10604 members.push(SelectionMemberPlan {
10605 key,
10606 index: member_index,
10607 base_sequence: member_base,
10608 size: member_size,
10609 kind: member_kind,
10610 shape: member_shape,
10611 leaf_start,
10612 });
10613 cursor += member_size;
10614 }
10615 Ok((descriptors, members))
10616}
10617
10618struct SelectionLeaf {
10619 call: ParallelLeafCall,
10620 outcome: Option<Result<ParallelAvroResult>>,
10621}
10622
10623#[derive(Clone)]
10625pub struct DurableOperationHandle {
10626 ctx: WorkflowContext,
10627 pub key: SelectionKey,
10628 pub index: usize,
10629 pub kind: String,
10630 pub identity: String,
10631 pub base_sequence: u64,
10632 pub size: usize,
10633 pub selection_group_id: String,
10634 shape: ParallelShape,
10635}
10636
10637impl std::fmt::Debug for DurableOperationHandle {
10638 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10639 formatter
10640 .debug_struct("DurableOperationHandle")
10641 .field("key", &self.key)
10642 .field("index", &self.index)
10643 .field("kind", &self.kind)
10644 .field("identity", &self.identity)
10645 .field("base_sequence", &self.base_sequence)
10646 .field("size", &self.size)
10647 .field("selection_group_id", &self.selection_group_id)
10648 .finish()
10649 }
10650}
10651
10652impl DurableOperationHandle {
10653 pub fn await_result(&self) -> DurableOperationAwaitCall {
10655 DurableOperationAwaitCall {
10656 handle: self.clone(),
10657 }
10658 }
10659
10660 pub fn cancel(&self) -> CancelDurableOperationCall {
10664 CancelDurableOperationCall {
10665 handle: self.clone(),
10666 emitted: false,
10667 }
10668 }
10669}
10670
10671#[derive(Debug)]
10673pub struct SelectionResult {
10674 pub key: SelectionKey,
10675 pub index: usize,
10676 pub kind: String,
10677 pub identity: String,
10678 pub value: Option<ParallelResult>,
10679 pub failure: Option<Error>,
10680 pub winner: DurableOperationHandle,
10681 pub handles: Vec<DurableOperationHandle>,
10682}
10683
10684impl SelectionResult {
10685 pub fn succeeded(&self) -> bool {
10686 self.failure.is_none()
10687 }
10688
10689 pub fn handle(&self, key: &SelectionKey) -> Option<&DurableOperationHandle> {
10690 self.handles.iter().find(|handle| &handle.key == key)
10691 }
10692
10693 pub fn remaining(&self) -> Vec<&DurableOperationHandle> {
10694 self.handles
10695 .iter()
10696 .filter(|handle| handle.index != self.index)
10697 .collect()
10698 }
10699
10700 pub fn into_result(self) -> Result<ParallelResult> {
10701 match (self.value, self.failure) {
10702 (Some(value), None) => Ok(value),
10703 (_, Some(error)) => Err(error),
10704 _ => Err(Error::WorkerLoop(
10705 "selection result contained neither a value nor a failure".to_string(),
10706 )),
10707 }
10708 }
10709}
10710
10711pub struct SelectCall {
10713 ctx: WorkflowContext,
10714 operations: Option<Vec<(SelectionKey, ParallelOperation)>>,
10715 members: Vec<SelectionMemberPlan>,
10716 leaves: Vec<SelectionLeaf>,
10717 group_id: Option<String>,
10718}
10719
10720impl SelectCall {
10721 fn new(ctx: WorkflowContext, operations: Vec<(SelectionKey, ParallelOperation)>) -> Self {
10722 Self {
10723 ctx,
10724 operations: Some(operations),
10725 members: Vec::new(),
10726 leaves: Vec::new(),
10727 group_id: None,
10728 }
10729 }
10730
10731 fn initialize(&mut self) -> Result<()> {
10732 let operations = self.operations.take().unwrap_or_default();
10733 let base_sequence = {
10734 let state = self
10735 .ctx
10736 .state
10737 .lock()
10738 .map_err(|_| Error::WorkflowStatePoisoned)?;
10739 if let Some(marker) = state.selection_markers.get(state.selection_marker_cursor) {
10740 marker.selection_group_base_sequence
10741 } else if let Some(recorded) = state.recorded_commands.get(state.command_cursor) {
10742 recorded.sequence()
10743 } else {
10744 let last = state
10745 .recorded_commands
10746 .last()
10747 .map(RecordedCommand::sequence)
10748 .unwrap_or(0);
10749 last.checked_add(u64::try_from(state.commands.len()).unwrap_or(u64::MAX))
10750 .and_then(|sequence| sequence.checked_add(1))
10751 .ok_or(Error::TimerDurationOverflow)?
10752 }
10753 };
10754 let (descriptors, members) = selection_descriptors(operations, base_sequence)?;
10755 let group_id = format!("select-calls:{base_sequence}:{}", descriptors.len());
10756 self.leaves = descriptors
10757 .into_iter()
10758 .map(|descriptor| SelectionLeaf {
10759 call: parallel_leaf_call(&self.ctx, descriptor.operation, descriptor.group_path),
10760 outcome: None,
10761 })
10762 .collect();
10763 self.members = members;
10764 self.group_id = Some(group_id);
10765 Ok(())
10766 }
10767}
10768
10769impl Future for SelectCall {
10770 type Output = Result<SelectionResult>;
10771
10772 fn poll(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
10773 if self.operations.is_some() {
10774 if let Err(error) = self.initialize() {
10775 return Poll::Ready(Err(error));
10776 }
10777 }
10778
10779 for leaf in &mut self.leaves {
10780 if leaf.outcome.is_some() {
10781 continue;
10782 }
10783 if let Poll::Ready(outcome) = leaf.call.poll_avro_value(cx) {
10784 if outcome
10785 .as_ref()
10786 .err()
10787 .is_some_and(workflow_task_integrity_error)
10788 {
10789 return Poll::Ready(outcome.map(|_| unreachable!()));
10790 }
10791 leaf.outcome = Some(outcome);
10792 }
10793 }
10794
10795 let all_members_terminal = self.leaves.iter().all(|leaf| leaf.outcome.is_some());
10796 let selection_member_range = self
10797 .members
10798 .first()
10799 .map(|member| member.base_sequence)
10800 .zip(self.leaves.len().try_into().ok())
10801 .map(|(base_sequence, size): (u64, u64)| {
10802 base_sequence..base_sequence.saturating_add(size)
10803 });
10804 let marker = {
10805 let mut state = match self.ctx.state.lock() {
10806 Ok(state) => state,
10807 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
10808 };
10809 let marker = state
10810 .selection_markers
10811 .get(state.selection_marker_cursor)
10812 .cloned();
10813 if marker.is_none()
10814 && all_members_terminal
10815 && selection_member_range.as_ref().is_some_and(|member_range| {
10816 state
10817 .recorded_commands
10818 .iter()
10819 .any(|command| member_range.contains(&command.sequence()))
10820 })
10821 {
10822 state.matched_recorded_pending = true;
10828 }
10829 marker
10830 };
10831 let Some(marker) = marker else {
10832 return Poll::Pending;
10833 };
10834 if self.group_id.as_deref() != Some(marker.selection_group_id.as_str())
10835 || marker.selection_group_size != self.leaves.len()
10836 || self.members.first().map(|member| member.base_sequence)
10837 != Some(marker.selection_group_base_sequence)
10838 {
10839 return Poll::Ready(Err(invalid_recorded_history(
10840 "selection_group_shape_mismatch",
10841 marker.selection_group_base_sequence,
10842 self.group_id
10843 .as_deref()
10844 .unwrap_or("initialized selection group"),
10845 &marker.selection_group_id,
10846 "recorded selection group differs from current workflow code",
10847 )));
10848 }
10849 let Some(member_position) = self.members.iter().position(|member| {
10850 member.key == marker.member_key
10851 && member.index == marker.member_index
10852 && member.base_sequence == marker.member_base_sequence
10853 && member.size == marker.member_size
10854 && member.kind == marker.operation_kind
10855 }) else {
10856 return Poll::Ready(Err(invalid_recorded_history(
10857 "selection_member_shape_mismatch",
10858 marker.member_base_sequence,
10859 "winner member matching current workflow code",
10860 &format!("{:?}", marker.member_key),
10861 "recorded selection winner differs from the authored member identity",
10862 )));
10863 };
10864 let member = self.members[member_position].clone();
10865 let (handles, resolution_sequence) = {
10866 let mut state = match self.ctx.state.lock() {
10867 Ok(state) => state,
10868 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
10869 };
10870 let identities = self
10871 .members
10872 .iter()
10873 .map(|candidate| {
10874 selection_operation_identity(
10875 &state,
10876 &candidate.kind,
10877 candidate.base_sequence,
10878 candidate.size,
10879 )
10880 })
10881 .collect::<Vec<_>>();
10882 if let Some((position, missing)) = identities
10883 .iter()
10884 .enumerate()
10885 .find(|(_, identity)| identity.is_empty())
10886 .map(|(position, identity)| (position, identity.clone()))
10887 {
10888 let candidate = &self.members[position];
10889 return Poll::Ready(Err(invalid_recorded_history(
10890 "selection_operation_identity_missing",
10891 candidate.base_sequence,
10892 &format!(
10893 "durable {} resource identity from scheduled/open history",
10894 candidate.kind
10895 ),
10896 &missing,
10897 "selection member history is missing its canonical durable identity",
10898 )));
10899 }
10900 let expected_winner_identity = &identities[member_position];
10901 let resolution_sequence = match validated_selection_resolution_sequence(
10902 &state,
10903 &marker,
10904 &member,
10905 expected_winner_identity,
10906 ) {
10907 Ok(sequence) => sequence,
10908 Err(error) => return Poll::Ready(Err(error)),
10909 };
10910 let handles = self
10911 .members
10912 .iter()
10913 .zip(identities)
10914 .map(|(member, identity)| DurableOperationHandle {
10915 ctx: self.ctx.clone(),
10916 key: member.key.clone(),
10917 index: member.index,
10918 kind: member.kind.clone(),
10919 identity,
10920 base_sequence: member.base_sequence,
10921 size: member.size,
10922 selection_group_id: marker.selection_group_id.clone(),
10923 shape: member.shape.clone(),
10924 })
10925 .collect::<Vec<_>>();
10926 if let Err(error) = validate_selection_cancellations_for_handles(&state, &handles) {
10927 return Poll::Ready(Err(error));
10928 }
10929 state.selection_marker_cursor += 1;
10930 (handles, resolution_sequence)
10931 };
10932
10933 let mut winner_failure = None;
10934 let mut flat_results = Vec::with_capacity(member.size);
10935 if marker.outcome == "failed" {
10936 let resolution_offset = match resolution_sequence
10937 .checked_sub(member.base_sequence)
10938 .and_then(|offset| usize::try_from(offset).ok())
10939 {
10940 Some(offset) if offset < member.size => offset,
10941 _ => {
10942 return Poll::Ready(Err(invalid_recorded_history(
10943 "selection_resolution_event_mismatch",
10944 member.base_sequence,
10945 "failure event within selected member bounds",
10946 &resolution_sequence.to_string(),
10947 "selection failure event is outside the authored member",
10948 )))
10949 }
10950 };
10951 let leaf = &mut self.leaves[member.leaf_start + resolution_offset];
10952 match leaf.outcome.take() {
10953 Some(Err(error)) => winner_failure = Some(error),
10954 _ => {
10955 return Poll::Ready(Err(invalid_recorded_history(
10956 "selection_winner_outcome_mismatch",
10957 member.base_sequence,
10958 "exact failed terminal history referenced by SelectionResolved",
10959 "missing or successful resolution event",
10960 "selection winner marker disagrees with terminal operation history",
10961 )))
10962 }
10963 }
10964 } else {
10965 for leaf in &mut self.leaves[member.leaf_start..member.leaf_start + member.size] {
10966 match leaf.outcome.take() {
10967 Some(Ok(result)) => flat_results.push(result),
10968 Some(Err(_)) => {
10969 return Poll::Ready(Err(invalid_recorded_history(
10970 "selection_winner_outcome_mismatch",
10971 member.base_sequence,
10972 "fully completed nested selection member",
10973 "failed durable leaf",
10974 "completed selection winner contains a failed leaf",
10975 )))
10976 }
10977 None => {
10978 return Poll::Ready(Err(invalid_recorded_history(
10979 "selection_winner_unresolved",
10980 member.base_sequence,
10981 "terminal history for every completed winner leaf",
10982 "pending member history",
10983 "completed SelectionResolved member has an unfinished durable barrier",
10984 )))
10985 }
10986 }
10987 }
10988 }
10989 let value = if winner_failure.is_none() {
10990 let mut flat_results = flat_results.into_iter();
10991 let value = parallel_results_for_shape(&member.shape, &mut flat_results);
10992 match value.into_json_result() {
10993 Ok(value) => Some(value),
10994 Err(error) => return Poll::Ready(Err(error)),
10995 }
10996 } else {
10997 None
10998 };
10999 let winner = handles[member_position].clone();
11000 Poll::Ready(Ok(SelectionResult {
11001 key: winner.key.clone(),
11002 index: winner.index,
11003 kind: winner.kind.clone(),
11004 identity: winner.identity.clone(),
11005 value,
11006 failure: winner_failure,
11007 winner,
11008 handles,
11009 }))
11010 }
11011}
11012
11013fn selection_operation_identity(
11014 state: &WorkflowState,
11015 kind: &str,
11016 base_sequence: u64,
11017 size: usize,
11018) -> String {
11019 if kind == "group" {
11020 return format!("group:{base_sequence}:{size}");
11021 }
11022 let fields: &[&str] = match kind {
11023 "activity" => &["activity_execution_id"],
11024 "child" => &["child_workflow_run_id"],
11025 "timer" => &["timer_id"],
11026 "signal" => &["signal_wait_id"],
11027 "condition" => &["condition_wait_id"],
11028 _ => &[],
11029 };
11030 for sequence in base_sequence..base_sequence.saturating_add(size as u64) {
11031 for event in state
11032 .history_events
11033 .iter()
11034 .filter(|event| durable_event_sequence(event) == Some(sequence))
11035 {
11036 for field in fields {
11037 if let Some(identity) = event.payload.get(*field).and_then(Value::as_str) {
11038 if !identity.is_empty() {
11039 return identity.to_string();
11040 }
11041 }
11042 }
11043 }
11044 }
11045 String::new()
11046}
11047
11048fn validated_selection_resolution_sequence(
11049 state: &WorkflowState,
11050 marker: &SelectionMarker,
11051 member: &SelectionMemberPlan,
11052 expected_identity: &str,
11053) -> Result<u64> {
11054 if expected_identity.is_empty() {
11055 return Err(invalid_recorded_history(
11056 "selection_operation_identity_missing",
11057 member.base_sequence,
11058 &format!(
11059 "durable {} resource identity from scheduled/open history",
11060 member.kind
11061 ),
11062 "missing operation identity",
11063 "selection member history is missing its canonical durable identity",
11064 ));
11065 }
11066 if marker.operation_identity != expected_identity {
11067 return Err(invalid_recorded_history(
11068 "selection_operation_identity_mismatch",
11069 member.base_sequence,
11070 expected_identity,
11071 &marker.operation_identity,
11072 "selection winner identity does not match durable scheduled/open history",
11073 ));
11074 }
11075
11076 let failure_types = [
11077 "ActivityFailed",
11078 "ActivityCancelled",
11079 "ActivityTimedOut",
11080 "ChildRunFailed",
11081 "ChildRunCancelled",
11082 "ChildRunTerminated",
11083 ];
11084 let success_types = [
11085 "ActivityCompleted",
11086 "ChildRunCompleted",
11087 "TimerFired",
11088 "SignalApplied",
11089 "ConditionWaitSatisfied",
11090 "ConditionWaitTimedOut",
11091 ];
11092 let terminal_types: &[&str] = if marker.outcome == "failed" {
11093 &failure_types
11094 } else {
11095 &success_types
11096 };
11097 let mut candidates = Vec::new();
11098 for event in state.history_events.iter() {
11099 let Some(sequence) = durable_event_sequence(event) else {
11100 continue;
11101 };
11102 if sequence < member.base_sequence
11103 || sequence >= member.base_sequence.saturating_add(member.size as u64)
11104 || !terminal_types.contains(&event.event_type.as_str())
11105 {
11106 continue;
11107 }
11108 let event_id = event
11109 .raw
11110 .get("id")
11111 .or_else(|| event.raw.get("event_id"))
11112 .and_then(Value::as_str)
11113 .filter(|value| !value.is_empty())
11114 .ok_or_else(|| {
11115 invalid_recorded_history(
11116 "selection_resolution_event_id_missing",
11117 member.base_sequence,
11118 "terminal selection history with a durable event id",
11119 &event.payload.to_string(),
11120 "selection terminal history cannot be bound to its winner marker",
11121 )
11122 })?;
11123 candidates.push((event_id.to_string(), event.event_type.clone(), sequence));
11124 }
11125 let resolution = if marker.outcome == "failed" {
11126 candidates.first()
11127 } else {
11128 candidates.last()
11129 };
11130 let Some((event_id, event_type, sequence)) = resolution else {
11131 return Err(invalid_recorded_history(
11132 "selection_resolution_event_missing",
11133 member.base_sequence,
11134 "terminal history for the selected member",
11135 &format!("{:?}", marker.member_key),
11136 "selection winner marker has no matching durable terminal event",
11137 ));
11138 };
11139 if event_id != &marker.resolution_event_id || event_type != &marker.resolution_event_type {
11140 return Err(invalid_recorded_history(
11141 "selection_resolution_event_mismatch",
11142 member.base_sequence,
11143 &format!("{event_type}:{event_id}"),
11144 &format!(
11145 "{}:{}",
11146 marker.resolution_event_type, marker.resolution_event_id
11147 ),
11148 "selection winner marker does not reference the event that made its member terminal",
11149 ));
11150 }
11151 Ok(*sequence)
11152}
11153
11154fn recorded_selection_member_outcome(
11155 state: &WorkflowState,
11156 handle: &DurableOperationHandle,
11157) -> Result<Option<ParallelResult>> {
11158 for event in state.history_events.iter() {
11159 let Some(sequence) = durable_event_sequence(event) else {
11160 continue;
11161 };
11162 if sequence < handle.base_sequence
11163 || sequence >= handle.base_sequence.saturating_add(handle.size as u64)
11164 || !matches!(
11165 event.event_type.as_str(),
11166 "ActivityFailed"
11167 | "ActivityCancelled"
11168 | "ActivityTimedOut"
11169 | "ChildRunFailed"
11170 | "ChildRunCancelled"
11171 | "ChildRunTerminated"
11172 )
11173 {
11174 continue;
11175 }
11176 let Some(command) = state
11177 .recorded_commands
11178 .iter()
11179 .find(|command| command.sequence() == sequence)
11180 else {
11181 continue;
11182 };
11183 match command {
11184 RecordedCommand::Activity {
11185 outcome: Some(Err(failure)),
11186 ..
11187 } => return Err(Error::ActivityFailed(failure.clone())),
11188 RecordedCommand::ChildWorkflow {
11189 outcome: Some(Err(failure)),
11190 ..
11191 } => return Err(Error::ChildWorkflowFailed(failure.clone())),
11192 _ => {}
11193 }
11194 }
11195
11196 let mut results = Vec::with_capacity(handle.size);
11197 for sequence in handle.base_sequence..handle.base_sequence.saturating_add(handle.size as u64) {
11198 let Some(command) = state
11199 .recorded_commands
11200 .iter()
11201 .find(|command| command.sequence() == sequence)
11202 else {
11203 return Ok(None);
11204 };
11205 let result = match command {
11206 RecordedCommand::Activity { outcome, .. } => match outcome {
11207 Some(Ok(value)) => ParallelAvroResult::Activity(value.clone()),
11208 Some(Err(failure)) => return Err(Error::ActivityFailed(failure.clone())),
11209 None => return Ok(None),
11210 },
11211 RecordedCommand::Timer { fired, .. } => {
11212 if !fired {
11213 return Ok(None);
11214 }
11215 ParallelAvroResult::Timer
11216 }
11217 RecordedCommand::ChildWorkflow { outcome, .. } => match outcome {
11218 Some(Ok(value)) => ParallelAvroResult::ChildWorkflow(value.clone()),
11219 Some(Err(failure)) => return Err(Error::ChildWorkflowFailed(failure.clone())),
11220 None => return Ok(None),
11221 },
11222 RecordedCommand::SignalWait { value, .. } => match value {
11223 Some(value) => ParallelAvroResult::Signal(value.clone()),
11224 None => return Ok(None),
11225 },
11226 RecordedCommand::ConditionWait { result, .. } => match result {
11227 Some(result) => ParallelAvroResult::Condition(*result),
11228 None => return Ok(None),
11229 },
11230 other => {
11231 return Err(command_mismatch(
11232 other,
11233 format!("selected {} member", handle.kind),
11234 ))
11235 }
11236 };
11237 results.push(result);
11238 }
11239 let mut results = results.into_iter();
11240 parallel_results_for_shape(&handle.shape, &mut results)
11241 .into_json_result()
11242 .map(Some)
11243}
11244
11245fn recorded_selection_member_is_terminal(
11246 state: &WorkflowState,
11247 handle: &DurableOperationHandle,
11248) -> bool {
11249 let mut completed = 0usize;
11250 let mut all_completed = true;
11251 for sequence in handle.base_sequence..handle.base_sequence.saturating_add(handle.size as u64) {
11252 let Some(command) = state
11253 .recorded_commands
11254 .iter()
11255 .find(|command| command.sequence() == sequence)
11256 else {
11257 all_completed = false;
11258 continue;
11259 };
11260 let terminal = match command {
11261 RecordedCommand::Activity {
11262 outcome: Some(Err(_)),
11263 ..
11264 }
11265 | RecordedCommand::ChildWorkflow {
11266 outcome: Some(Err(_)),
11267 ..
11268 } => return true,
11269 RecordedCommand::Activity { outcome, .. } => outcome.is_some(),
11270 RecordedCommand::ChildWorkflow { outcome, .. } => outcome.is_some(),
11271 RecordedCommand::Timer { fired, .. } => *fired,
11272 RecordedCommand::SignalWait { value, .. } => value.is_some(),
11273 RecordedCommand::ConditionWait { result, .. } => result.is_some(),
11274 RecordedCommand::SearchAttributes { .. }
11275 | RecordedCommand::SideEffect { .. }
11276 | RecordedCommand::VersionMarker { .. }
11277 | RecordedCommand::Memo { .. } => false,
11278 };
11279 if !terminal {
11280 all_completed = false;
11281 continue;
11282 }
11283 completed += 1;
11284 }
11285 all_completed && completed == handle.size
11286}
11287
11288fn selection_cancellation_for_handle(
11289 state: &WorkflowState,
11290 handle: &DurableOperationHandle,
11291) -> Result<bool> {
11292 let Some(marker) = state.cancelled_selection_members.iter().find(|recorded| {
11293 recorded.selection_group_id == handle.selection_group_id
11294 && recorded.member_base_sequence == handle.base_sequence
11295 }) else {
11296 return Ok(false);
11297 };
11298 validate_selection_cancellation_marker(marker, handle)?;
11299 Ok(true)
11300}
11301
11302fn validate_selection_cancellations_for_handles(
11303 state: &WorkflowState,
11304 handles: &[DurableOperationHandle],
11305) -> Result<()> {
11306 let Some(group_id) = handles
11307 .first()
11308 .map(|handle| handle.selection_group_id.as_str())
11309 else {
11310 return Ok(());
11311 };
11312 for marker in state
11313 .cancelled_selection_members
11314 .iter()
11315 .filter(|marker| marker.selection_group_id == group_id)
11316 {
11317 let Some(handle) = handles
11318 .iter()
11319 .find(|handle| handle.base_sequence == marker.member_base_sequence)
11320 else {
11321 return Err(invalid_recorded_history(
11322 "selection_cancellation_member_mismatch",
11323 marker.member_base_sequence,
11324 "SelectionOperationCancelled matching an authored selection handle",
11325 &format!("{marker:?}"),
11326 "selection cancellation member base does not name an authored member",
11327 ));
11328 };
11329 validate_selection_cancellation_marker(marker, handle)?;
11330 }
11331 Ok(())
11332}
11333
11334fn validate_selection_cancellation_marker(
11335 marker: &SelectionCancellation,
11336 handle: &DurableOperationHandle,
11337) -> Result<()> {
11338 if marker.selection_group_id != handle.selection_group_id
11339 || marker.member_key != handle.key
11340 || marker.member_index != handle.index
11341 || marker.member_base_sequence != handle.base_sequence
11342 || marker.member_size != handle.size
11343 || marker.operation_kind != handle.kind
11344 || marker.operation_identity != handle.identity
11345 {
11346 return Err(invalid_recorded_history(
11347 "selection_cancellation_member_mismatch",
11348 handle.base_sequence,
11349 "SelectionOperationCancelled matching the authored selection handle",
11350 &format!("{marker:?}"),
11351 "selection cancellation history targets different authored member metadata",
11352 ));
11353 }
11354 Ok(())
11355}
11356
11357pub struct DurableOperationAwaitCall {
11359 handle: DurableOperationHandle,
11360}
11361
11362impl Future for DurableOperationAwaitCall {
11363 type Output = Result<ParallelResult>;
11364
11365 fn poll(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11366 let state = match self.handle.ctx.state.lock() {
11367 Ok(state) => state,
11368 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11369 };
11370 match selection_cancellation_for_handle(&state, &self.handle) {
11371 Err(error) => return Poll::Ready(Err(error)),
11372 Ok(false) => {}
11373 Ok(true) => {
11374 return Poll::Ready(Err(Error::DurableOperationCancelled(
11375 DurableOperationCancelled {
11376 selection_group_id: self.handle.selection_group_id.clone(),
11377 member_key: self.handle.key.clone(),
11378 member_index: self.handle.index,
11379 operation_kind: self.handle.kind.clone(),
11380 operation_identity: self.handle.identity.clone(),
11381 },
11382 )));
11383 }
11384 }
11385 match recorded_selection_member_outcome(&state, &self.handle) {
11386 Ok(Some(result)) => Poll::Ready(Ok(result)),
11387 Ok(None) => Poll::Pending,
11388 Err(error) => Poll::Ready(Err(error)),
11389 }
11390 }
11391}
11392
11393pub struct CancelDurableOperationCall {
11395 handle: DurableOperationHandle,
11396 emitted: bool,
11397}
11398
11399impl Future for CancelDurableOperationCall {
11400 type Output = Result<()>;
11401
11402 fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11403 let ctx = self.handle.ctx.clone();
11404 let mut state = match ctx.state.lock() {
11405 Ok(state) => state,
11406 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11407 };
11408 match selection_cancellation_for_handle(&state, &self.handle) {
11409 Err(error) => return Poll::Ready(Err(error)),
11410 Ok(true) => return Poll::Ready(Ok(())),
11411 Ok(false) => {}
11412 }
11413 if recorded_selection_member_is_terminal(&state, &self.handle) {
11414 return Poll::Ready(Ok(()));
11415 }
11416 if !self.emitted {
11417 state.commands.push(json!({
11418 "type": "cancel_selection_operation",
11419 "selection_group_id": self.handle.selection_group_id,
11420 "member_key": self.handle.key,
11421 "member_index": self.handle.index,
11422 "member_base_sequence": self.handle.base_sequence,
11423 "member_size": self.handle.size,
11424 "operation_kind": self.handle.kind,
11425 "operation_identity": self.handle.identity,
11426 }));
11427 self.emitted = true;
11428 }
11429 Poll::Pending
11433 }
11434}
11435
11436trait PollNestedResultExt<T> {
11437 fn flatten_result(self) -> Poll<Result<T>>;
11438}
11439
11440impl<T> PollNestedResultExt<T> for Poll<Result<Result<T>>> {
11441 fn flatten_result(self) -> Poll<Result<T>> {
11442 match self {
11443 Poll::Ready(Ok(result)) => Poll::Ready(result),
11444 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
11445 Poll::Pending => Poll::Pending,
11446 }
11447 }
11448}
11449
11450struct SagaCompensation {
11451 activity_type: String,
11452 options: ActivityOptions,
11453 arguments: AvroValue,
11454 registration_order: usize,
11455}
11456
11457pub struct Saga {
11465 ctx: WorkflowContext,
11466 compensations: Vec<SagaCompensation>,
11467}
11468
11469impl Saga {
11470 fn new(ctx: WorkflowContext) -> Self {
11471 Self {
11472 ctx,
11473 compensations: Vec::new(),
11474 }
11475 }
11476
11477 pub fn add_compensation<T: Serialize>(
11478 &mut self,
11479 activity_type: impl Into<String>,
11480 args: T,
11481 ) -> Result<&mut Self> {
11482 self.add_compensation_with_options(activity_type, ActivityOptions::new(), args)
11483 }
11484
11485 pub fn add_compensation_with_options<T: Serialize>(
11486 &mut self,
11487 activity_type: impl Into<String>,
11488 options: ActivityOptions,
11489 args: T,
11490 ) -> Result<&mut Self> {
11491 let activity_type = activity_type.into();
11492 if activity_type.trim().is_empty() || activity_type.trim() != activity_type {
11493 return Err(Error::Codec(
11494 "saga compensation activity type must be non-empty without surrounding whitespace"
11495 .to_string(),
11496 ));
11497 }
11498 options.validate().map_err(Error::InvalidActivityOptions)?;
11499 let arguments = AvroValue::from_serialize(&args)?;
11500 let registration_order = self.compensations.len() + 1;
11501 self.compensations.push(SagaCompensation {
11502 activity_type,
11503 options,
11504 arguments,
11505 registration_order,
11506 });
11507 Ok(self)
11508 }
11509
11510 pub async fn compensate(mut self, initiating_failure: Error) -> Error {
11512 while let Some(compensation) = self.compensations.pop() {
11513 if let Err(compensation_failure) = self
11514 .ctx
11515 .activity_with_options(
11516 compensation.activity_type.clone(),
11517 compensation.options,
11518 compensation.arguments,
11519 )
11520 .await
11521 {
11522 if workflow_task_integrity_error(&compensation_failure) {
11523 return compensation_failure;
11524 }
11525 return Error::SagaCompensationFailed(SagaCompensationFailure {
11526 initiating_failure: Box::new(initiating_failure),
11527 compensation_failure: Box::new(compensation_failure),
11528 compensation_activity_type: compensation.activity_type,
11529 compensation_registration_order: compensation.registration_order,
11530 });
11531 }
11532 }
11533 initiating_failure
11534 }
11535
11536 pub async fn finish<T>(self, outcome: Result<T>) -> Result<T> {
11538 match outcome {
11539 Ok(value) => Ok(value),
11540 Err(error) => Err(self.compensate(error).await),
11541 }
11542 }
11543}
11544
11545pub struct ActivityCall {
11546 ctx: WorkflowContext,
11547 activity_type: String,
11548 options: ActivityOptions,
11549 args: Option<Result<AvroValue>>,
11550 scheduled: bool,
11551 parallel_group_path: Vec<ParallelGroupMetadata>,
11552}
11553
11554impl ActivityCall {
11555 fn poll_avro_value(
11556 mut self: Pin<&mut Self>,
11557 _cx: &mut TaskContext<'_>,
11558 ) -> Poll<Result<AvroValue>> {
11559 let ctx = self.ctx.clone();
11560 let mut state = match ctx.state.lock() {
11561 Ok(state) => state,
11562 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11563 };
11564
11565 if self.scheduled {
11566 return Poll::Pending;
11567 }
11568
11569 let options = match self.options.validate() {
11570 Ok(options) => options,
11571 Err(error) => {
11572 return Poll::Ready(Err(Error::InvalidActivityOptions(error)));
11573 }
11574 };
11575 let task_queue = options
11576 .task_queue
11577 .clone()
11578 .unwrap_or_else(|| state.task_queue.clone());
11579 let current_recorded_options = RecordedActivityOptions {
11580 task_queue: RecordedSnapshotValue::Known(Some(task_queue.clone())),
11581 execution_mode: RecordedSnapshotValue::Known(None),
11584 retry_policy: current_activity_retry_snapshot(&options),
11585 };
11586
11587 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
11588 let sequence = recorded.sequence();
11589 match recorded {
11590 RecordedCommand::Activity {
11591 activity_type,
11592 options: recorded_options,
11593 outcome,
11594 parallel_group_path,
11595 ..
11596 } => {
11597 if let Err(error) = ensure_parallel_path_matches(
11598 sequence,
11599 parallel_group_path.as_deref(),
11600 &self.parallel_group_path,
11601 ) {
11602 return Poll::Ready(Err(error));
11603 }
11604 if let Some(recorded_type) = activity_type {
11605 if recorded_type != self.activity_type {
11606 return Poll::Ready(Err(Error::NonDeterministicReplay(
11607 ReplayFailure::new(
11608 "recorded_command_detail_mismatch",
11609 Some(sequence),
11610 Some(format!("activity:{recorded_type}")),
11611 Some(format!("activity:{}", self.activity_type)),
11612 "recorded activity type differs from the current workflow command",
11613 ),
11614 )));
11615 }
11616 }
11617 if let Some(recorded_options) = recorded_options {
11618 if !recorded_options
11619 .task_queue
11620 .matches_current(¤t_recorded_options.task_queue)
11621 {
11622 return Poll::Ready(Err(Error::NonDeterministicReplay(
11623 ReplayFailure::new(
11624 "activity_task_queue_mismatch",
11625 Some(sequence),
11626 Some(activity_options_description(&recorded_options)),
11627 Some(activity_options_description(¤t_recorded_options)),
11628 "recorded activity task queue differs from the current workflow command",
11629 ),
11630 )));
11631 }
11632 if !recorded_options
11633 .execution_mode
11634 .matches_current(¤t_recorded_options.execution_mode)
11635 {
11636 return Poll::Ready(Err(Error::NonDeterministicReplay(
11637 ReplayFailure::new(
11638 "activity_execution_mode_mismatch",
11639 Some(sequence),
11640 Some(activity_options_description(&recorded_options)),
11641 Some(activity_options_description(¤t_recorded_options)),
11642 "recorded activity execution mode differs from the current workflow command",
11643 ),
11644 )));
11645 }
11646 if !recorded_options
11647 .retry_policy
11648 .matches_current(¤t_recorded_options.retry_policy)
11649 {
11650 return Poll::Ready(Err(Error::NonDeterministicReplay(
11651 ReplayFailure::new(
11652 "activity_retry_policy_mismatch",
11653 Some(sequence),
11654 Some(activity_options_description(&recorded_options)),
11655 Some(activity_options_description(¤t_recorded_options)),
11656 "recorded activity retry policy differs from the current workflow command",
11657 ),
11658 )));
11659 }
11660 }
11661 state.command_cursor += 1;
11662 if let Some(outcome) = outcome {
11663 return Poll::Ready(outcome.map_err(Error::ActivityFailed));
11664 }
11665 state.matched_recorded_pending = true;
11666 self.scheduled = true;
11667 return Poll::Pending;
11668 }
11669 other => {
11670 return Poll::Ready(Err(command_mismatch(
11671 &other,
11672 format!("activity:{}", self.activity_type),
11673 )));
11674 }
11675 }
11676 }
11677
11678 if !self.scheduled {
11679 let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
11680 Ok(args) => args,
11681 Err(error) => return Poll::Ready(Err(error)),
11682 };
11683 let arguments = normalize_avro_arguments(args);
11684 let envelope = match encode_typed_envelope(&arguments, &state.payload_codec) {
11685 Ok(envelope) => envelope,
11686 Err(error) => return Poll::Ready(Err(error)),
11687 };
11688
11689 let mut command = serde_json::Map::from_iter([
11690 ("type".to_string(), json!("schedule_activity")),
11691 (
11692 "activity_type".to_string(),
11693 json!(self.activity_type.clone()),
11694 ),
11695 ("queue".to_string(), json!(task_queue)),
11696 ("arguments".to_string(), envelope),
11697 ]);
11698 for (field, value) in [
11699 ("start_to_close_timeout", options.start_to_close_timeout),
11700 (
11701 "schedule_to_start_timeout",
11702 options.schedule_to_start_timeout,
11703 ),
11704 (
11705 "schedule_to_close_timeout",
11706 options.schedule_to_close_timeout,
11707 ),
11708 ("heartbeat_timeout", options.heartbeat_timeout),
11709 ] {
11710 if let Some(value) = value {
11711 command.insert(field.to_string(), json!(value));
11712 }
11713 }
11714 if let Some(retry_policy) = options.retry_policy {
11715 command.insert("retry_policy".to_string(), retry_policy);
11716 }
11717 apply_parallel_group_path(&mut command, &self.parallel_group_path);
11718 state.commands.push(Value::Object(command));
11719 self.scheduled = true;
11720 }
11721
11722 Poll::Pending
11723 }
11724}
11725
11726impl Future for ActivityCall {
11727 type Output = Result<Value>;
11728
11729 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11730 match self.poll_avro_value(cx) {
11731 Poll::Ready(Ok(value)) => Poll::Ready(value.into_json()),
11732 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
11733 Poll::Pending => Poll::Pending,
11734 }
11735 }
11736}
11737
11738pub struct TimerCall {
11740 ctx: WorkflowContext,
11741 delay_seconds: Option<u64>,
11742 scheduled: bool,
11743 matched_pending: bool,
11744 parallel_group_path: Vec<ParallelGroupMetadata>,
11745}
11746
11747impl Future for TimerCall {
11748 type Output = Result<()>;
11749
11750 fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11751 if self.matched_pending {
11752 return Poll::Pending;
11753 }
11754
11755 let ctx = self.ctx.clone();
11756 let Some(requested_delay) = self.delay_seconds else {
11757 return Poll::Ready(Err(Error::TimerDurationOverflow));
11758 };
11759 let mut state = match ctx.state.lock() {
11760 Ok(state) => state,
11761 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11762 };
11763
11764 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
11765 match recorded {
11766 RecordedCommand::Timer {
11767 sequence,
11768 delay_seconds,
11769 fired,
11770 parallel_group_path,
11771 ..
11772 } => {
11773 if let Err(error) = ensure_parallel_path_matches(
11774 sequence,
11775 parallel_group_path.as_deref(),
11776 &self.parallel_group_path,
11777 ) {
11778 return Poll::Ready(Err(error));
11779 }
11780 if delay_seconds != requested_delay {
11781 return Poll::Ready(Err(Error::NonDeterministicReplay(
11782 ReplayFailure::new(
11783 "timer_delay_mismatch",
11784 Some(sequence),
11785 Some(format!("timer:{delay_seconds}s")),
11786 Some(format!("timer:{requested_delay}s")),
11787 "recorded timer delay differs from the current workflow command",
11788 ),
11789 )));
11790 }
11791 state.command_cursor += 1;
11792 if fired {
11793 return Poll::Ready(Ok(()));
11794 }
11795 state.matched_recorded_pending = true;
11796 self.scheduled = true;
11797 self.matched_pending = true;
11798 return Poll::Pending;
11799 }
11800 other => return Poll::Ready(Err(command_mismatch(&other, "timer"))),
11801 }
11802 }
11803
11804 if !self.scheduled {
11805 let mut command = serde_json::Map::from_iter([
11806 ("type".to_string(), json!("start_timer")),
11807 ("delay_seconds".to_string(), json!(requested_delay)),
11808 ]);
11809 apply_parallel_group_path(&mut command, &self.parallel_group_path);
11810 state.commands.push(Value::Object(command));
11811 self.scheduled = true;
11812 }
11813
11814 Poll::Pending
11815 }
11816}
11817
11818pub struct ConditionWaitCall {
11820 ctx: WorkflowContext,
11821 options: ConditionWaitOptions,
11822 predicate: Box<dyn Fn() -> Result<bool> + Send + 'static>,
11823 occurrence_id: Option<String>,
11824 opened_wait: bool,
11825 parallel_group_path: Vec<ParallelGroupMetadata>,
11826}
11827
11828impl Future for ConditionWaitCall {
11829 type Output = Result<ConditionWaitResult>;
11830
11831 fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11832 if self.opened_wait {
11833 return Poll::Pending;
11834 }
11835
11836 let options = match self.options.validate() {
11837 Ok(options) => options,
11838 Err(error) => return Poll::Ready(Err(Error::InvalidConditionWaitOptions(error))),
11839 };
11840 let ctx = self.ctx.clone();
11841 let occurrence_id = match self.occurrence_id.as_ref() {
11842 Some(occurrence_id) => occurrence_id.clone(),
11843 None => {
11844 let mut state = match ctx.state.lock() {
11845 Ok(state) => state,
11846 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11847 };
11848 let ordinal = state.condition_wait_occurrence_counter;
11849 state.condition_wait_occurrence_counter = match ordinal.checked_add(1) {
11850 Some(next) => next,
11851 None => {
11852 return Poll::Ready(Err(Error::WorkerLoop(
11853 "condition wait occurrence counter overflowed".to_string(),
11854 )))
11855 }
11856 };
11857 let occurrence_id = format!("{CONDITION_WAIT_OCCURRENCE_PREFIX}{ordinal}");
11858 drop(state);
11859 self.occurrence_id = Some(occurrence_id.clone());
11860 occurrence_id
11861 }
11862 };
11863
11864 let recorded_result = {
11865 let mut state = match ctx.state.lock() {
11866 Ok(state) => state,
11867 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11868 };
11869 let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() else {
11870 drop(state);
11871 return self.poll_new_condition(options);
11872 };
11873 if !matches!(recorded, RecordedCommand::ConditionWait { .. }) {
11874 return Poll::Ready(Err(command_mismatch(&recorded, "condition wait")));
11875 }
11876
11877 let mut cursor = state.command_cursor;
11878 let mut result = None;
11879 loop {
11880 let Some(RecordedCommand::ConditionWait {
11881 sequence,
11882 occurrence_id: recorded_occurrence_id,
11883 condition_key,
11884 predicate_identity,
11885 timeout_seconds,
11886 result: recorded_result,
11887 parallel_group_path,
11888 ..
11889 }) = state.recorded_commands.get(cursor)
11890 else {
11891 break;
11892 };
11893
11894 if cursor > state.command_cursor && recorded_occurrence_id != &occurrence_id {
11895 break;
11896 }
11897 if let Err(error) = ensure_parallel_path_matches(
11898 *sequence,
11899 parallel_group_path.as_deref(),
11900 &self.parallel_group_path,
11901 ) {
11902 return Poll::Ready(Err(error));
11903 }
11904 if let Err(error) = validate_recorded_condition_wait(
11905 *sequence,
11906 recorded_occurrence_id,
11907 condition_key.as_deref(),
11908 predicate_identity,
11909 *timeout_seconds,
11910 &occurrence_id,
11911 &options,
11912 ) {
11913 return Poll::Ready(Err(error));
11914 }
11915 if result == Some(ConditionWaitResult::TimedOut) {
11916 return Poll::Ready(Err(Error::NonDeterministicReplay(ReplayFailure::new(
11917 "condition_wait_reopened_after_timeout",
11918 Some(*sequence),
11919 Some("timed-out condition is terminal".to_string()),
11920 Some("another physical wait-open".to_string()),
11921 "condition history reopened one logical wait after its durable timeout",
11922 ))));
11923 }
11924 result = *recorded_result;
11925 cursor += 1;
11926 }
11927 state.command_cursor = cursor;
11928 result
11929 };
11930
11931 if let Some(result) = recorded_result {
11932 return Poll::Ready(Ok(result));
11933 }
11934
11935 self.poll_open_condition(options)
11936 }
11937}
11938
11939impl ConditionWaitCall {
11940 fn poll_new_condition(
11941 self: Pin<&mut Self>,
11942 options: ValidatedConditionWaitOptions,
11943 ) -> Poll<Result<ConditionWaitResult>> {
11944 self.poll_open_condition(options)
11945 }
11946
11947 fn poll_open_condition(
11948 mut self: Pin<&mut Self>,
11949 options: ValidatedConditionWaitOptions,
11950 ) -> Poll<Result<ConditionWaitResult>> {
11951 let selection_member = self
11952 .parallel_group_path
11953 .first()
11954 .is_some_and(|entry| entry.parallel_group_mode.as_deref() == Some("select"));
11955 match (self.predicate)() {
11956 Ok(true) if !selection_member => {
11957 return Poll::Ready(Ok(ConditionWaitResult::Satisfied))
11958 }
11959 Ok(_) => {}
11960 Err(error) => return Poll::Ready(Err(error)),
11961 }
11962 if options.timeout_seconds == Some(0) && !selection_member {
11963 return Poll::Ready(Ok(ConditionWaitResult::TimedOut));
11964 }
11965
11966 let ctx = self.ctx.clone();
11967 let mut state = match ctx.state.lock() {
11968 Ok(state) => state,
11969 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11970 };
11971 let mut command = serde_json::Map::from_iter([
11972 ("type".to_string(), json!("open_condition_wait")),
11973 (
11974 "condition_wait_occurrence_id".to_string(),
11975 json!(self.occurrence_id.as_deref().unwrap_or_default()),
11976 ),
11977 ("condition_key".to_string(), json!(options.condition_key)),
11978 (
11979 "condition_definition_fingerprint".to_string(),
11980 json!(options.predicate_identity),
11981 ),
11982 ]);
11983 if let Some(timeout_seconds) = options.timeout_seconds {
11984 command.insert("timeout_seconds".to_string(), json!(timeout_seconds));
11985 }
11986 apply_parallel_group_path(&mut command, &self.parallel_group_path);
11987 state.commands.push(Value::Object(command));
11988 drop(state);
11989 self.opened_wait = true;
11990 Poll::Pending
11991 }
11992}
11993
11994fn validate_recorded_condition_wait(
11995 sequence: u64,
11996 recorded_occurrence_id: &str,
11997 recorded_key: Option<&str>,
11998 recorded_predicate_identity: &str,
11999 recorded_timeout_seconds: Option<u64>,
12000 current_occurrence_id: &str,
12001 current: &ValidatedConditionWaitOptions,
12002) -> Result<()> {
12003 if recorded_occurrence_id != current_occurrence_id {
12004 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12005 "condition_wait_occurrence_mismatch",
12006 Some(sequence),
12007 Some(recorded_occurrence_id.to_string()),
12008 Some(current_occurrence_id.to_string()),
12009 "recorded condition occurrence differs from the current authored wait position",
12010 )));
12011 }
12012 if recorded_key != Some(current.condition_key.as_str()) {
12013 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12014 "condition_wait_key_mismatch",
12015 Some(sequence),
12016 recorded_key.map(str::to_string),
12017 Some(current.condition_key.clone()),
12018 "recorded condition identity differs from the current workflow wait",
12019 )));
12020 }
12021 if recorded_predicate_identity != current.predicate_identity {
12022 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12023 "condition_wait_predicate_mismatch",
12024 Some(sequence),
12025 Some(recorded_predicate_identity.to_string()),
12026 Some(current.predicate_identity.clone()),
12027 "recorded condition predicate behavior differs from current workflow code",
12028 )));
12029 }
12030 if recorded_timeout_seconds != current.timeout_seconds {
12031 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12032 "condition_wait_timeout_mismatch",
12033 Some(sequence),
12034 recorded_timeout_seconds.map(|seconds| format!("{seconds}s")),
12035 current.timeout_seconds.map(|seconds| format!("{seconds}s")),
12036 "recorded condition timeout differs from the current workflow wait",
12037 )));
12038 }
12039 Ok(())
12040}
12041
12042pub struct ChildWorkflowCall {
12044 ctx: WorkflowContext,
12045 workflow_type: String,
12046 options: ChildWorkflowOptions,
12047 args: Option<Result<AvroValue>>,
12048 scheduled: bool,
12049 matched_pending: bool,
12050 parallel_group_path: Vec<ParallelGroupMetadata>,
12051}
12052
12053impl ChildWorkflowCall {
12054 fn poll_avro_value(
12055 mut self: Pin<&mut Self>,
12056 _cx: &mut TaskContext<'_>,
12057 ) -> Poll<Result<ChildWorkflowAvroResult>> {
12058 if self.matched_pending {
12059 return Poll::Pending;
12060 }
12061
12062 let ctx = self.ctx.clone();
12063 let mut state = match ctx.state.lock() {
12064 Ok(state) => state,
12065 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12066 };
12067
12068 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
12069 let sequence = recorded.sequence();
12070 match recorded {
12071 RecordedCommand::ChildWorkflow {
12072 workflow_type,
12073 outcome,
12074 parallel_group_path,
12075 ..
12076 } => {
12077 if let Err(error) = ensure_parallel_path_matches(
12078 sequence,
12079 parallel_group_path.as_deref(),
12080 &self.parallel_group_path,
12081 ) {
12082 return Poll::Ready(Err(error));
12083 }
12084 if let Some(recorded_type) = workflow_type {
12085 if recorded_type != self.workflow_type {
12086 return Poll::Ready(Err(Error::NonDeterministicReplay(
12087 ReplayFailure::new(
12088 "recorded_command_detail_mismatch",
12089 Some(sequence),
12090 Some(format!("child workflow:{recorded_type}")),
12091 Some(format!("child workflow:{}", self.workflow_type)),
12092 "recorded child workflow type differs from the current workflow command",
12093 ),
12094 )));
12095 }
12096 }
12097 state.command_cursor += 1;
12098 if let Some(outcome) = outcome {
12099 return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed));
12100 }
12101 state.matched_recorded_pending = true;
12102 self.scheduled = true;
12103 self.matched_pending = true;
12104 return Poll::Pending;
12105 }
12106 other => {
12107 return Poll::Ready(Err(command_mismatch(
12108 &other,
12109 format!("child workflow:{}", self.workflow_type),
12110 )));
12111 }
12112 }
12113 }
12114
12115 if !self.scheduled {
12116 if self.options.task_queue.trim().is_empty() {
12117 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
12118 "task_queue must not be empty".to_string(),
12119 )));
12120 }
12121 for (name, value) in [
12122 (
12123 "execution_timeout_seconds",
12124 self.options.execution_timeout_seconds,
12125 ),
12126 ("run_timeout_seconds", self.options.run_timeout_seconds),
12127 ] {
12128 if value == Some(0) {
12129 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(format!(
12130 "{name} must be at least 1"
12131 ))));
12132 }
12133 }
12134
12135 let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
12136 Ok(args) => args,
12137 Err(error) => return Poll::Ready(Err(error)),
12138 };
12139 let arguments = match encode_typed_envelope(
12140 &normalize_avro_arguments(args),
12141 &state.payload_codec,
12142 ) {
12143 Ok(arguments) => arguments,
12144 Err(error) => return Poll::Ready(Err(error)),
12145 };
12146 let mut command = json!({
12147 "type": "start_child_workflow",
12148 "workflow_type": self.workflow_type,
12149 "queue": self.options.task_queue,
12150 "parent_close_policy": self.options.parent_close_policy.as_str(),
12151 "arguments": arguments,
12152 });
12153 let object = command
12154 .as_object_mut()
12155 .expect("child workflow command is always an object");
12156 if let Some(policy) = &self.options.retry_policy {
12157 let mut retry_policy = serde_json::Map::new();
12158 if let Some(max_attempts) = policy.max_attempts {
12159 if max_attempts == 0 {
12160 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
12161 "retry_policy.max_attempts must be at least 1".to_string(),
12162 )));
12163 }
12164 retry_policy.insert("max_attempts".to_string(), json!(max_attempts));
12165 }
12166 if !policy.backoff_seconds.is_empty() {
12167 retry_policy
12168 .insert("backoff_seconds".to_string(), json!(policy.backoff_seconds));
12169 }
12170 if !policy.non_retryable_error_types.is_empty() {
12171 retry_policy.insert(
12172 "non_retryable_error_types".to_string(),
12173 json!(policy.non_retryable_error_types),
12174 );
12175 }
12176 if retry_policy.is_empty() {
12177 return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
12178 "retry_policy must configure at least one field".to_string(),
12179 )));
12180 }
12181 object.insert("retry_policy".to_string(), Value::Object(retry_policy));
12182 }
12183 if let Some(seconds) = self.options.execution_timeout_seconds {
12184 object.insert("execution_timeout_seconds".to_string(), json!(seconds));
12185 }
12186 if let Some(seconds) = self.options.run_timeout_seconds {
12187 object.insert("run_timeout_seconds".to_string(), json!(seconds));
12188 }
12189 apply_parallel_group_path(object, &self.parallel_group_path);
12190 state.commands.push(command);
12191 self.scheduled = true;
12192 }
12193
12194 Poll::Pending
12195 }
12196}
12197
12198impl Future for ChildWorkflowCall {
12199 type Output = Result<ChildWorkflowResult>;
12200
12201 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12202 match self.poll_avro_value(cx) {
12203 Poll::Ready(Ok(result)) => match result.result.into_json() {
12204 Ok(projected) => Poll::Ready(Ok(ChildWorkflowResult {
12205 parent: result.parent,
12206 child: result.child,
12207 child_workflow_type: result.child_workflow_type,
12208 result: projected,
12209 })),
12210 Err(error) => Poll::Ready(Err(error)),
12211 },
12212 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
12213 Poll::Pending => Poll::Pending,
12214 }
12215 }
12216}
12217
12218fn command_mismatch(recorded: &RecordedCommand, actual: impl Into<String>) -> Error {
12219 Error::NonDeterministicReplay(ReplayFailure::new(
12220 "recorded_command_mismatch",
12221 Some(recorded.sequence()),
12222 Some(recorded.shape().to_string()),
12223 Some(actual.into()),
12224 "current workflow command does not match the recorded durable command sequence",
12225 ))
12226}
12227
12228pub struct SignalCall {
12229 ctx: WorkflowContext,
12230 signal_name: String,
12231 runtime_reserved_allowed: bool,
12232 opened_wait: bool,
12233 matched_pending: bool,
12234 parallel_group_path: Vec<ParallelGroupMetadata>,
12235}
12236
12237impl SignalCall {
12238 fn poll_avro_value(
12239 mut self: Pin<&mut Self>,
12240 _cx: &mut TaskContext<'_>,
12241 ) -> Poll<Result<Vec<AvroValue>>> {
12242 if self.matched_pending {
12243 return Poll::Pending;
12244 }
12245 if !self.runtime_reserved_allowed {
12246 if let Err(error) = validate_user_signal_name(&self.signal_name) {
12247 return Poll::Ready(Err(error));
12248 }
12249 }
12250
12251 let ctx = self.ctx.clone();
12252 let mut state = match ctx.state.lock() {
12253 Ok(state) => state,
12254 Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12255 };
12256
12257 if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
12258 match recorded {
12259 RecordedCommand::SignalWait {
12260 sequence,
12261 signal_name,
12262 value,
12263 parallel_group_path,
12264 } => {
12265 if let Err(error) = ensure_parallel_path_matches(
12266 sequence,
12267 parallel_group_path.as_deref(),
12268 &self.parallel_group_path,
12269 ) {
12270 return Poll::Ready(Err(error));
12271 }
12272 if signal_name != self.signal_name {
12273 return Poll::Ready(Err(Error::NonDeterministicReplay(
12274 ReplayFailure::new(
12275 "recorded_command_detail_mismatch",
12276 Some(sequence),
12277 Some(format!("signal wait:{signal_name}")),
12278 Some(format!("signal wait:{}", self.signal_name)),
12279 "recorded signal name differs from the current workflow command",
12280 ),
12281 )));
12282 }
12283
12284 state.command_cursor += 1;
12285 if let Some(value) = value {
12286 return Poll::Ready(Ok(value));
12287 }
12288 if state
12289 .resume_signal
12290 .as_ref()
12291 .is_some_and(|signal| signal.signal_name == self.signal_name)
12292 {
12293 let signal = state
12294 .resume_signal
12295 .take()
12296 .expect("matching resume signal is present");
12297 return Poll::Ready(Ok(signal.arguments));
12298 }
12299
12300 state.matched_recorded_pending = true;
12301 self.opened_wait = true;
12302 self.matched_pending = true;
12303 return Poll::Pending;
12304 }
12305 other => {
12306 return Poll::Ready(Err(command_mismatch(
12307 &other,
12308 format!("signal wait:{}", self.signal_name),
12309 )));
12310 }
12311 }
12312 }
12313
12314 if state
12315 .resume_signal
12316 .as_ref()
12317 .is_some_and(|signal| signal.signal_name == self.signal_name)
12318 {
12319 let signal = state
12320 .resume_signal
12321 .take()
12322 .expect("matching resume signal is present");
12323 return Poll::Ready(Ok(signal.arguments));
12324 }
12325
12326 if !self.opened_wait {
12327 let mut command = serde_json::Map::from_iter([
12328 ("type".to_string(), json!("open_signal_wait")),
12329 ("signal_name".to_string(), json!(self.signal_name)),
12330 ]);
12331 apply_parallel_group_path(&mut command, &self.parallel_group_path);
12332 state.commands.push(Value::Object(command));
12333 self.opened_wait = true;
12334 }
12335
12336 Poll::Pending
12337 }
12338}
12339
12340impl Future for SignalCall {
12341 type Output = Result<Vec<Value>>;
12342
12343 fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12344 match self.poll_avro_value(cx) {
12345 Poll::Ready(Ok(values)) => Poll::Ready(
12346 values
12347 .into_iter()
12348 .map(AvroValue::into_json)
12349 .collect::<Result<Vec<_>>>(),
12350 ),
12351 Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
12352 Poll::Pending => Poll::Pending,
12353 }
12354 }
12355}
12356
12357#[derive(Clone, Debug)]
12358pub struct ActivityContext {
12359 client: Client,
12360 pub task_id: String,
12361 pub activity_attempt_id: String,
12362 pub lease_owner: String,
12363 pub activity_type: String,
12364 pub attempt_number: u64,
12365 pub task_queue: String,
12366 pub worker_id: String,
12367}
12368
12369impl ActivityContext {
12370 pub async fn heartbeat<T: Serialize>(&self, details: T) -> Result<ActivityHeartbeatResponse> {
12371 self.client
12372 .heartbeat_activity_task(
12373 &self.task_id,
12374 &self.activity_attempt_id,
12375 &self.lease_owner,
12376 details,
12377 )
12378 .await
12379 }
12380}
12381
12382fn decode_task_avro_arguments(value: Option<&Value>, codec: &str) -> Result<AvroValue> {
12383 validate_payload_codec(codec)?;
12384 match value {
12385 Some(value) => Ok(normalize_avro_arguments(decode_wire_avro_value(
12386 value, codec,
12387 )?)),
12388 None => Ok(AvroValue::Array(Vec::new())),
12389 }
12390}
12391
12392fn decode_resume_signal(task: &WorkflowTask) -> Result<Option<ResumeSignal>> {
12393 let Some(signal_name) = task
12394 .signal_name
12395 .as_deref()
12396 .filter(|value| !value.is_empty())
12397 else {
12398 return Ok(None);
12399 };
12400 let decoded = decode_task_avro_arguments(task.signal_arguments.as_ref(), &task.payload_codec)?;
12401 let AvroValue::Array(arguments) = decoded else {
12402 unreachable!("normalize_avro_arguments always returns an array");
12403 };
12404
12405 Ok(Some(ResumeSignal {
12406 signal_name: signal_name.to_string(),
12407 arguments,
12408 }))
12409}
12410
12411fn validate_workflow_task_payloads(task: &WorkflowTask) -> Result<()> {
12412 validate_payload_codec(&task.payload_codec)?;
12413 validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)?;
12414 validate_optional_inbound_payload(task.signal_arguments.as_ref(), &task.payload_codec)?;
12415 for event in &task.history_events {
12416 validate_history_event_payloads(event, &task.payload_codec)?;
12417 }
12418 Ok(())
12419}
12420
12421fn validate_activity_task_payloads(task: &ActivityTask) -> Result<()> {
12422 validate_payload_codec(&task.payload_codec)?;
12423 validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)
12424}
12425
12426fn validate_query_task_payloads(task: &QueryTask) -> Result<()> {
12427 validate_payload_codec(&task.payload_codec)?;
12428 validate_optional_inbound_payload(task.workflow_arguments.as_ref(), &task.payload_codec)?;
12429 validate_optional_inbound_payload(task.query_arguments.as_ref(), &task.payload_codec)?;
12430 for event in &task.history_events {
12431 validate_history_event_payloads(event, &task.payload_codec)?;
12432 }
12433
12434 let Some(export) = task.history_export.as_ref() else {
12435 return Ok(());
12436 };
12437 let export_codec = match export.get("payloads") {
12438 Some(payloads) => declared_payload_codec(payloads, "codec")?,
12439 None => None,
12440 }
12441 .unwrap_or(&task.payload_codec);
12442 validate_payload_codec(export_codec)?;
12443
12444 if let Some(events) = export.get("history_events").and_then(Value::as_array) {
12445 for event in events {
12446 let event_type = event
12447 .get("event_type")
12448 .or_else(|| event.get("type"))
12449 .and_then(Value::as_str)
12450 .unwrap_or_default();
12451 if let Some(payload) = event.get("payload") {
12452 validate_history_payloads(event_type, payload, export_codec)?;
12453 }
12454 }
12455 }
12456 for signal in export
12457 .get("signals")
12458 .and_then(Value::as_array)
12459 .into_iter()
12460 .flatten()
12461 {
12462 let codec = declared_payload_codec(signal, "payload_codec")?.unwrap_or(export_codec);
12463 validate_payload_codec(codec)?;
12464 validate_optional_inbound_payload(signal.get("arguments"), codec)?;
12465 }
12466 for activity in export
12467 .get("activities")
12468 .and_then(Value::as_array)
12469 .into_iter()
12470 .flatten()
12471 {
12472 let codec = declared_payload_codec(activity, "payload_codec")?.unwrap_or(export_codec);
12473 validate_payload_codec(codec)?;
12474 validate_optional_inbound_payload(activity.get("arguments"), codec)?;
12475 validate_optional_inbound_payload(activity.get("result"), codec)?;
12476 }
12477 Ok(())
12478}
12479
12480fn validate_history_event_payloads(event: &HistoryEvent, fallback_codec: &str) -> Result<()> {
12481 validate_history_payloads(&event.event_type, &event.payload, fallback_codec)
12482}
12483
12484fn validate_history_payloads(
12485 event_type: &str,
12486 payload: &Value,
12487 fallback_codec: &str,
12488) -> Result<()> {
12489 let codec = declared_payload_codec(payload, "payload_codec")?.unwrap_or(fallback_codec);
12490 validate_payload_codec(codec)?;
12491 for field in history_payload_fields(event_type) {
12492 validate_optional_inbound_payload(payload.get(*field), codec)?;
12493 }
12494 Ok(())
12495}
12496
12497const SIGNAL_HISTORY_PAYLOAD_FIELDS: &[&str] = &["value", "input", "arguments"];
12498
12499fn history_payload_fields(event_type: &str) -> &'static [&'static str] {
12500 match event_type {
12501 "ActivityCompleted" => &["result"],
12502 "SignalReceived" | "SignalApplied" => SIGNAL_HISTORY_PAYLOAD_FIELDS,
12503 "UpdateAccepted" | "UpdateRejected" | "UpdateApplied" => &["arguments"],
12504 "UpdateCompleted" | "SideEffectRecorded" => &["result"],
12505 "ChildRunCompleted" => &["result", "output"],
12506 "WorkflowCompleted" => &["output"],
12507 "ServiceCallStarted"
12508 | "ServiceCallCompleted"
12509 | "ServiceCallFailed"
12510 | "ServiceCallCancelled" => &["request_payload", "response_payload"],
12511 _ => &[],
12512 }
12513}
12514
12515fn signal_history_payload(payload: &Value) -> Option<&Value> {
12516 SIGNAL_HISTORY_PAYLOAD_FIELDS
12517 .iter()
12518 .find_map(|field| payload.get(*field))
12519}
12520
12521fn declared_payload_codec<'a>(value: &'a Value, field: &str) -> Result<Option<&'a str>> {
12522 match value.get(field) {
12523 None => Ok(None),
12524 Some(Value::String(codec)) => Ok(Some(codec)),
12525 Some(_) => Err(invalid_payload_envelope()),
12526 }
12527}
12528
12529fn validate_optional_inbound_payload(value: Option<&Value>, codec: &str) -> Result<()> {
12530 validate_payload_codec(codec)?;
12531 if let Some(value) = value.filter(|value| !value.is_null()) {
12532 decode_wire_avro_value(value, codec)?;
12533 }
12534 Ok(())
12535}
12536
12537fn recorded_parallel_group_entry(payload: &Value, sequence: u64) -> Result<ParallelGroupMetadata> {
12538 let group_id = payload_string(payload, "parallel_group_id").ok_or_else(|| {
12539 invalid_recorded_history(
12540 "parallel_group_metadata_invalid",
12541 sequence,
12542 "non-empty parallel_group_id",
12543 &payload.to_string(),
12544 "parallel-group history is missing its stable identity",
12545 )
12546 })?;
12547 let kind = payload_string(payload, "parallel_group_kind").ok_or_else(|| {
12548 invalid_recorded_history(
12549 "parallel_group_metadata_invalid",
12550 sequence,
12551 "activity, child, timer, signal, condition, or mixed group kind",
12552 &payload.to_string(),
12553 "parallel-group history is missing its group kind",
12554 )
12555 })?;
12556 if !matches!(
12557 kind.as_str(),
12558 "activity" | "child" | "timer" | "signal" | "condition" | "mixed"
12559 ) {
12560 return Err(invalid_recorded_history(
12561 "parallel_group_metadata_invalid",
12562 sequence,
12563 "activity, child, timer, signal, condition, or mixed group kind",
12564 &kind,
12565 "parallel-group history contains an unsupported group kind",
12566 ));
12567 }
12568 let base_sequence = payload
12569 .get("parallel_group_base_sequence")
12570 .and_then(value_as_u64)
12571 .filter(|value| *value > 0)
12572 .ok_or_else(|| {
12573 invalid_recorded_history(
12574 "parallel_group_metadata_invalid",
12575 sequence,
12576 "positive parallel_group_base_sequence",
12577 &payload.to_string(),
12578 "parallel-group history contains an invalid base sequence",
12579 )
12580 })?;
12581 let size = payload
12582 .get("parallel_group_size")
12583 .and_then(value_as_u64)
12584 .and_then(|value| usize::try_from(value).ok())
12585 .filter(|value| (1..=MAX_PARALLEL_OPERATIONS).contains(value))
12586 .ok_or_else(|| {
12587 invalid_recorded_history(
12588 "parallel_group_metadata_invalid",
12589 sequence,
12590 "bounded positive parallel_group_size",
12591 &payload.to_string(),
12592 "parallel-group history contains an invalid group size",
12593 )
12594 })?;
12595 let index = payload
12596 .get("parallel_group_index")
12597 .and_then(value_as_u64)
12598 .and_then(|value| usize::try_from(value).ok())
12599 .filter(|value| *value < size)
12600 .ok_or_else(|| {
12601 invalid_recorded_history(
12602 "parallel_group_metadata_invalid",
12603 sequence,
12604 "parallel_group_index within group bounds",
12605 &payload.to_string(),
12606 "parallel-group history contains an invalid member index",
12607 )
12608 })?;
12609 if base_sequence.checked_add(u64::try_from(index).unwrap_or(u64::MAX)) != Some(sequence) {
12610 return Err(invalid_recorded_history(
12611 "parallel_group_metadata_invalid",
12612 sequence,
12613 "base sequence plus member index equals workflow sequence",
12614 &payload.to_string(),
12615 "parallel-group path does not preserve durable workflow position",
12616 ));
12617 }
12618 let mode = payload
12619 .get("parallel_group_mode")
12620 .and_then(Value::as_str)
12621 .unwrap_or("all");
12622 if !matches!(mode, "all" | "select") {
12623 return Err(invalid_recorded_history(
12624 "parallel_group_metadata_invalid",
12625 sequence,
12626 "parallel group mode all or select",
12627 mode,
12628 "parallel-group history contains an unsupported group mode",
12629 ));
12630 }
12631 let expected_id = if mode == "select" {
12632 format!("select-calls:{base_sequence}:{size}")
12633 } else {
12634 format!("{}:{base_sequence}:{size}", parallel_group_prefix(&kind))
12635 };
12636 if group_id != expected_id {
12637 return Err(invalid_recorded_history(
12638 "parallel_group_metadata_invalid",
12639 sequence,
12640 &expected_id,
12641 &group_id,
12642 "parallel-group history contains an incompatible stable group ID",
12643 ));
12644 }
12645 let selection_member_key = if mode == "select" {
12646 Some(selection_key_from_value(
12647 payload.get("selection_member_key"),
12648 sequence,
12649 )?)
12650 } else {
12651 None
12652 };
12653 let selection_member_index = if mode == "select" {
12654 Some(required_parallel_usize(
12655 payload,
12656 "selection_member_index",
12657 sequence,
12658 )?)
12659 } else {
12660 None
12661 };
12662 let selection_member_base_sequence = if mode == "select" {
12663 Some(
12664 payload
12665 .get("selection_member_base_sequence")
12666 .and_then(value_as_u64)
12667 .filter(|value| *value >= base_sequence)
12668 .ok_or_else(|| {
12669 invalid_recorded_history(
12670 "parallel_group_metadata_invalid",
12671 sequence,
12672 "selection member base within its group",
12673 &payload.to_string(),
12674 "selection history contains an invalid member base sequence",
12675 )
12676 })?,
12677 )
12678 } else {
12679 None
12680 };
12681 let selection_member_size = if mode == "select" {
12682 let member_size = required_parallel_usize(payload, "selection_member_size", sequence)?;
12683 if member_size == 0 {
12684 return Err(invalid_recorded_history(
12685 "parallel_group_metadata_invalid",
12686 sequence,
12687 "positive selection member size",
12688 &payload.to_string(),
12689 "selection history contains an invalid member size",
12690 ));
12691 }
12692 Some(member_size)
12693 } else {
12694 None
12695 };
12696 let selection_member_kind = if mode == "select" {
12697 let kind = payload_string(payload, "selection_member_kind").ok_or_else(|| {
12698 invalid_recorded_history(
12699 "parallel_group_metadata_invalid",
12700 sequence,
12701 "selection member operation kind",
12702 &payload.to_string(),
12703 "selection history is missing its authored member kind",
12704 )
12705 })?;
12706 if !matches!(
12707 kind.as_str(),
12708 "activity" | "child" | "timer" | "signal" | "condition" | "group"
12709 ) {
12710 return Err(invalid_recorded_history(
12711 "parallel_group_metadata_invalid",
12712 sequence,
12713 "activity, child, timer, signal, condition, or group selection member kind",
12714 &kind,
12715 "selection history contains an unsupported member kind",
12716 ));
12717 }
12718 Some(kind)
12719 } else {
12720 None
12721 };
12722 if let (Some(member_base), Some(member_size)) =
12723 (selection_member_base_sequence, selection_member_size)
12724 {
12725 let member_end = member_base
12726 .checked_add(u64::try_from(member_size).unwrap_or(u64::MAX))
12727 .ok_or_else(|| {
12728 invalid_recorded_history(
12729 "parallel_group_metadata_invalid",
12730 sequence,
12731 "bounded selection member range",
12732 &payload.to_string(),
12733 "selection member range overflowed",
12734 )
12735 })?;
12736 let group_end = base_sequence
12737 .checked_add(u64::try_from(size).unwrap_or(u64::MAX))
12738 .unwrap_or(u64::MAX);
12739 if sequence < member_base || sequence >= member_end || member_end > group_end {
12740 return Err(invalid_recorded_history(
12741 "parallel_group_metadata_invalid",
12742 sequence,
12743 "workflow sequence within one bounded selection member",
12744 &payload.to_string(),
12745 "selection member range does not contain its durable leaf",
12746 ));
12747 }
12748 }
12749 Ok(ParallelGroupMetadata {
12750 parallel_group_id: group_id,
12751 parallel_group_kind: kind,
12752 parallel_group_base_sequence: base_sequence,
12753 parallel_group_size: size,
12754 parallel_group_index: index,
12755 parallel_group_mode: (mode == "select").then(|| "select".to_string()),
12756 selection_member_key,
12757 selection_member_index,
12758 selection_member_base_sequence,
12759 selection_member_size,
12760 selection_member_kind,
12761 })
12762}
12763
12764fn required_parallel_usize(payload: &Value, field: &str, sequence: u64) -> Result<usize> {
12765 payload
12766 .get(field)
12767 .and_then(value_as_u64)
12768 .and_then(|value| usize::try_from(value).ok())
12769 .ok_or_else(|| {
12770 invalid_recorded_history(
12771 "parallel_group_metadata_invalid",
12772 sequence,
12773 &format!("non-negative integer {field}"),
12774 &payload.to_string(),
12775 "selection history contains invalid member metadata",
12776 )
12777 })
12778}
12779
12780fn selection_key_from_value(value: Option<&Value>, sequence: u64) -> Result<SelectionKey> {
12781 match value {
12782 Some(Value::String(value)) if !value.is_empty() => Ok(SelectionKey::Name(value.clone())),
12783 Some(value) => value_as_u64(value)
12784 .and_then(|value| usize::try_from(value).ok())
12785 .map(SelectionKey::Index)
12786 .ok_or_else(|| {
12787 invalid_recorded_history(
12788 "selection_member_key_invalid",
12789 sequence,
12790 "non-empty string or non-negative integer member key",
12791 &value.to_string(),
12792 "selection history contains an invalid member key",
12793 )
12794 }),
12795 None => Err(invalid_recorded_history(
12796 "selection_member_key_missing",
12797 sequence,
12798 "selection_member_key",
12799 "<missing>",
12800 "selection history is missing its stable member key",
12801 )),
12802 }
12803}
12804
12805fn recorded_parallel_group_path(
12806 events: &[&HistoryEvent],
12807 sequence: u64,
12808) -> Result<Option<Vec<ParallelGroupMetadata>>> {
12809 let mut recorded: Option<Vec<ParallelGroupMetadata>> = None;
12810 for event in events {
12811 let payload = &event.payload;
12812 let has_metadata = payload.get("parallel_group_path").is_some()
12813 || payload.get("parallel_group_id").is_some()
12814 || payload.get("parallel_group_kind").is_some()
12815 || payload.get("parallel_group_base_sequence").is_some()
12816 || payload.get("parallel_group_size").is_some()
12817 || payload.get("parallel_group_index").is_some()
12818 || payload.get("parallel_group_mode").is_some()
12819 || payload.get("selection_member_key").is_some();
12820 if !has_metadata {
12821 continue;
12822 }
12823
12824 let top_level = recorded_parallel_group_entry(payload, sequence)?;
12825 let path = match payload.get("parallel_group_path") {
12826 None => vec![top_level.clone()],
12827 Some(Value::Array(entries)) if !entries.is_empty() => entries
12828 .iter()
12829 .map(|entry| recorded_parallel_group_entry(entry, sequence))
12830 .collect::<Result<Vec<_>>>()?,
12831 Some(value) => {
12832 return Err(invalid_recorded_history(
12833 "parallel_group_metadata_invalid",
12834 sequence,
12835 "non-empty parallel_group_path list",
12836 &value.to_string(),
12837 "parallel-group history contains an invalid group path",
12838 ));
12839 }
12840 };
12841 if path.last() != Some(&top_level) {
12842 return Err(invalid_recorded_history(
12843 "parallel_group_metadata_invalid",
12844 sequence,
12845 &serde_json::to_string(&path.last()).unwrap_or_default(),
12846 &serde_json::to_string(&top_level).unwrap_or_default(),
12847 "parallel-group top-level fields do not match the innermost path entry",
12848 ));
12849 }
12850 if recorded.as_ref().is_some_and(|existing| existing != &path) {
12851 return Err(invalid_recorded_history(
12852 "parallel_group_history_conflict",
12853 sequence,
12854 &serde_json::to_string(&recorded.as_ref()).unwrap_or_default(),
12855 &serde_json::to_string(&path).unwrap_or_default(),
12856 "parallel-group metadata changed between scheduling and resolution history",
12857 ));
12858 }
12859 recorded = Some(path);
12860 }
12861 Ok(recorded)
12862}
12863
12864fn recorded_commands(
12865 events: &[HistoryEvent],
12866 fallback_codec: &str,
12867 parent: WorkflowIdentity,
12868) -> Result<Vec<RecordedCommand>> {
12869 let mut events_by_sequence: BTreeMap<u64, Vec<&HistoryEvent>> = BTreeMap::new();
12870 let mut last_new_sequence = None;
12871
12872 for event in events {
12873 let is_activity = matches!(
12874 event.event_type.as_str(),
12875 "ActivityScheduled"
12876 | "ActivityStarted"
12877 | "ActivityHeartbeatRecorded"
12878 | "ActivityRetryScheduled"
12879 | "ActivityCompleted"
12880 | "ActivityFailed"
12881 | "ActivityCancelled"
12882 | "ActivityTimedOut"
12883 );
12884 let is_workflow_timer = matches!(
12885 event.event_type.as_str(),
12886 "TimerScheduled" | "TimerCancelled" | "TimerFired"
12887 ) && !is_internal_timer_event(event);
12888 let is_child_workflow = matches!(
12889 event.event_type.as_str(),
12890 "ChildWorkflowScheduled"
12891 | "ChildRunCompleted"
12892 | "ChildRunFailed"
12893 | "ChildRunCancelled"
12894 | "ChildRunTerminated"
12895 );
12896 let is_signal_wait = is_recorded_signal_wait_event(event);
12897 let is_condition_wait = is_recorded_condition_wait_event(event);
12898 let is_search_attributes = event.event_type == "SearchAttributesUpserted";
12899 let is_side_effect = event.event_type == "SideEffectRecorded";
12900 let is_version_marker = event.event_type == "VersionMarkerRecorded";
12901 let is_memo = event.event_type == "MemoUpserted";
12902 if !is_activity
12903 && !is_workflow_timer
12904 && !is_child_workflow
12905 && !is_signal_wait
12906 && !is_condition_wait
12907 && !is_search_attributes
12908 && !is_side_effect
12909 && !is_version_marker
12910 && !is_memo
12911 {
12912 continue;
12913 }
12914
12915 let sequence = durable_event_sequence(event).ok_or_else(|| {
12916 Error::NonDeterministicReplay(ReplayFailure::new(
12917 "durable_command_sequence_missing",
12918 None,
12919 Some("positive workflow sequence".to_string()),
12920 Some(event.event_type.clone()),
12921 "durable command history event has no workflow sequence",
12922 ))
12923 })?;
12924 if sequence == 0 {
12925 return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12926 "durable_command_sequence_invalid",
12927 Some(sequence),
12928 Some("positive workflow sequence".to_string()),
12929 Some(sequence.to_string()),
12930 "durable command history uses an invalid workflow sequence",
12931 )));
12932 }
12933 if !events_by_sequence.contains_key(&sequence) {
12934 if let Some(previous) = last_new_sequence {
12935 if sequence < previous {
12936 return Err(invalid_recorded_history(
12937 "durable_command_sequence_mismatch",
12938 sequence,
12939 &format!("workflow sequence greater than {previous}"),
12940 &sequence.to_string(),
12941 "durable commands are not strictly ordered by their recorded workflow sequence",
12942 ));
12943 }
12944 }
12945 last_new_sequence = Some(sequence);
12946 }
12947 events_by_sequence.entry(sequence).or_default().push(event);
12948 }
12949
12950 let commands: Vec<RecordedCommand> = events_by_sequence
12951 .into_iter()
12952 .map(|(sequence, sequence_events)| {
12953 let activity_events: Vec<_> = sequence_events
12954 .iter()
12955 .copied()
12956 .filter(|event| event.event_type.starts_with("Activity"))
12957 .collect();
12958 let timer_events: Vec<_> = sequence_events
12959 .iter()
12960 .copied()
12961 .filter(|event| event.event_type.starts_with("Timer"))
12962 .collect();
12963 let child_events: Vec<_> = sequence_events
12964 .iter()
12965 .copied()
12966 .filter(|event| {
12967 event.event_type == "ChildWorkflowScheduled"
12968 || event.event_type.starts_with("ChildRun")
12969 })
12970 .collect();
12971 let signal_wait_events: Vec<_> = sequence_events
12972 .iter()
12973 .copied()
12974 .filter(|event| is_recorded_signal_wait_event(event))
12975 .collect();
12976 let condition_wait_events: Vec<_> = sequence_events
12977 .iter()
12978 .copied()
12979 .filter(|event| is_recorded_condition_wait_event(event))
12980 .collect();
12981 let search_attribute_events: Vec<_> = sequence_events
12982 .iter()
12983 .copied()
12984 .filter(|event| event.event_type == "SearchAttributesUpserted")
12985 .collect();
12986 let side_effect_events: Vec<_> = sequence_events
12987 .iter()
12988 .copied()
12989 .filter(|event| event.event_type == "SideEffectRecorded")
12990 .collect();
12991 let version_marker_events: Vec<_> = sequence_events
12992 .iter()
12993 .copied()
12994 .filter(|event| event.event_type == "VersionMarkerRecorded")
12995 .collect();
12996 let memo_events: Vec<_> = sequence_events
12997 .iter()
12998 .copied()
12999 .filter(|event| event.event_type == "MemoUpserted")
13000 .collect();
13001
13002 let command_kind_count = usize::from(!activity_events.is_empty())
13003 + usize::from(!timer_events.is_empty())
13004 + usize::from(!child_events.is_empty())
13005 + usize::from(!signal_wait_events.is_empty())
13006 + usize::from(!condition_wait_events.is_empty())
13007 + usize::from(!search_attribute_events.is_empty())
13008 + usize::from(!side_effect_events.is_empty())
13009 + usize::from(!version_marker_events.is_empty())
13010 + usize::from(!memo_events.is_empty());
13011 if command_kind_count > 1 {
13012 let actual = [
13013 (!activity_events.is_empty()).then_some("activity"),
13014 (!timer_events.is_empty()).then_some("timer"),
13015 (!child_events.is_empty()).then_some("child workflow"),
13016 (!signal_wait_events.is_empty()).then_some("signal wait"),
13017 (!condition_wait_events.is_empty()).then_some("condition wait"),
13018 (!search_attribute_events.is_empty()).then_some("search-attribute update"),
13019 (!side_effect_events.is_empty()).then_some("side effect"),
13020 (!version_marker_events.is_empty()).then_some("version marker"),
13021 (!memo_events.is_empty()).then_some("memo upsert"),
13022 ]
13023 .into_iter()
13024 .flatten()
13025 .collect::<Vec<_>>()
13026 .join(" and ");
13027 return Err(invalid_recorded_history(
13028 "durable_command_sequence_collision",
13029 sequence,
13030 "one durable command kind",
13031 &actual,
13032 "one workflow sequence records more than one durable command kind",
13033 ));
13034 }
13035
13036 if !activity_events.is_empty() {
13037 let parallel_group_path =
13038 recorded_parallel_group_path(&activity_events, sequence)?;
13039 let scheduled_count = activity_events
13040 .iter()
13041 .filter(|event| event.event_type == "ActivityScheduled")
13042 .count();
13043 if scheduled_count > 1 {
13044 return Err(invalid_recorded_history(
13045 "duplicate_activity_schedule",
13046 sequence,
13047 "at most one ActivityScheduled event",
13048 "multiple ActivityScheduled events",
13049 "activity history schedules more than one command at one workflow sequence",
13050 ));
13051 }
13052 let activity_type = activity_events.iter().find_map(|event| {
13053 event
13054 .payload
13055 .get("activity_type")
13056 .or_else(|| event.payload.get("activity_name"))
13057 .and_then(Value::as_str)
13058 .map(str::to_string)
13059 });
13060 if activity_events.iter().filter_map(|event| {
13061 event
13062 .payload
13063 .get("activity_type")
13064 .or_else(|| event.payload.get("activity_name"))
13065 .and_then(Value::as_str)
13066 }).any(|candidate| Some(candidate) != activity_type.as_deref()) {
13067 return Err(invalid_recorded_history(
13068 "activity_identity_mismatch",
13069 sequence,
13070 activity_type.as_deref().unwrap_or("one activity identity"),
13071 "conflicting activity identities",
13072 "activity lifecycle events at one workflow sequence disagree on identity",
13073 ));
13074 }
13075 let terminal: Vec<_> = activity_events
13076 .iter()
13077 .copied()
13078 .filter(|event| {
13079 matches!(
13080 event.event_type.as_str(),
13081 "ActivityCompleted"
13082 | "ActivityFailed"
13083 | "ActivityCancelled"
13084 | "ActivityTimedOut"
13085 )
13086 })
13087 .collect();
13088 let duplicate_delivery = terminal.first().is_some_and(|first| {
13089 terminal.iter().all(|event| {
13090 event.event_type == first.event_type && event.payload == first.payload
13091 })
13092 });
13093 if terminal.len() > 1 && !duplicate_delivery {
13094 return Err(invalid_recorded_history(
13095 "duplicate_activity_terminal_event",
13096 sequence,
13097 "at most one terminal activity event",
13098 "multiple terminal activity events",
13099 "activity history settles one command more than once",
13100 ));
13101 }
13102 let outcome = terminal
13103 .first()
13104 .map(|event| activity_outcome(event, fallback_codec, activity_type.clone()))
13105 .transpose()?;
13106 let options = activity_events
13107 .iter()
13108 .find(|event| event.event_type == "ActivityScheduled")
13109 .and_then(|event| event.payload.get("activity"))
13110 .and_then(Value::as_object)
13111 .map(|activity| RecordedActivityOptions {
13112 task_queue: recorded_optional_string(activity, "queue"),
13113 execution_mode: recorded_optional_string(activity, "execution_mode"),
13114 retry_policy: recorded_activity_retry_snapshot(
13115 activity.get("retry_policy"),
13116 ),
13117 });
13118 return Ok(RecordedCommand::Activity {
13119 sequence,
13120 activity_type,
13121 options,
13122 outcome,
13123 parallel_group_path,
13124 });
13125 }
13126
13127 if !child_events.is_empty() {
13128 let parallel_group_path = recorded_parallel_group_path(&child_events, sequence)?;
13129 let scheduled: Vec<_> = child_events
13130 .iter()
13131 .copied()
13132 .filter(|event| event.event_type == "ChildWorkflowScheduled")
13133 .collect();
13134 if scheduled.len() != 1 {
13135 return Err(invalid_recorded_history(
13136 "child_workflow_schedule_missing_or_duplicate",
13137 sequence,
13138 "one ChildWorkflowScheduled event",
13139 &format!("{} ChildWorkflowScheduled events", scheduled.len()),
13140 "child workflow replay requires exactly one recorded schedule event",
13141 ));
13142 }
13143 let workflow_type = child_events.iter().find_map(|event| {
13144 event
13145 .payload
13146 .get("child_workflow_type")
13147 .or_else(|| event.payload.get("workflow_type"))
13148 .and_then(Value::as_str)
13149 .filter(|value| !value.is_empty())
13150 .map(str::to_string)
13151 });
13152 if child_events
13153 .iter()
13154 .filter_map(|event| {
13155 event
13156 .payload
13157 .get("child_workflow_type")
13158 .or_else(|| event.payload.get("workflow_type"))
13159 .and_then(Value::as_str)
13160 })
13161 .any(|candidate| Some(candidate) != workflow_type.as_deref())
13162 {
13163 return Err(invalid_recorded_history(
13164 "child_workflow_identity_mismatch",
13165 sequence,
13166 workflow_type
13167 .as_deref()
13168 .unwrap_or("one child workflow type"),
13169 "conflicting child workflow types",
13170 "child workflow lifecycle events at one sequence disagree on type",
13171 ));
13172 }
13173 let mut outcomes = child_workflow_outcomes(
13174 &child_events.iter().map(|event| (*event).clone()).collect::<Vec<_>>(),
13175 fallback_codec,
13176 parent.clone(),
13177 )?;
13178 let terminal_events = child_events
13179 .iter()
13180 .copied()
13181 .filter(|event| event.event_type.starts_with("ChildRun"))
13182 .collect::<Vec<_>>();
13183 let duplicate_delivery = terminal_events.first().is_some_and(|first| {
13184 terminal_events.iter().all(|event| {
13185 event.event_type == first.event_type && event.payload == first.payload
13186 })
13187 });
13188 if outcomes.len() > 1 && !duplicate_delivery {
13189 return Err(invalid_recorded_history(
13190 "duplicate_child_workflow_terminal_event",
13191 sequence,
13192 "at most one terminal child event",
13193 "multiple terminal child events",
13194 "child workflow history settles one command more than once",
13195 ));
13196 }
13197 return Ok(RecordedCommand::ChildWorkflow {
13198 sequence,
13199 workflow_type,
13200 outcome: outcomes.pop(),
13201 parallel_group_path,
13202 });
13203 }
13204
13205 if !signal_wait_events.is_empty() {
13206 let opened: Vec<_> = signal_wait_events
13207 .iter()
13208 .copied()
13209 .filter(|event| event.event_type == "SignalWaitOpened")
13210 .collect();
13211 if opened.len() != 1 {
13212 return Err(invalid_recorded_history(
13213 "signal_wait_open_missing_or_duplicate",
13214 sequence,
13215 "one SignalWaitOpened event",
13216 &format!("{} SignalWaitOpened events", opened.len()),
13217 "signal replay requires exactly one canonical wait-open event",
13218 ));
13219 }
13220
13221 let applied: Vec<_> = signal_wait_events
13222 .iter()
13223 .copied()
13224 .filter(|event| event.event_type == "SignalApplied")
13225 .collect();
13226 if applied.len() > 1 {
13227 return Err(invalid_recorded_history(
13228 "duplicate_signal_wait_apply",
13229 sequence,
13230 "at most one SignalApplied event",
13231 "multiple SignalApplied events",
13232 "signal history applies one durable wait more than once",
13233 ));
13234 }
13235
13236 let signal_names = signal_wait_events
13237 .iter()
13238 .map(|event| required_signal_wait_name(event, sequence))
13239 .collect::<Result<Vec<_>>>()?;
13240 let signal_name = signal_names
13241 .first()
13242 .expect("signal wait events are not empty")
13243 .clone();
13244 if signal_names.iter().any(|candidate| candidate != &signal_name) {
13245 return Err(invalid_recorded_history(
13246 "signal_wait_identity_mismatch",
13247 sequence,
13248 &signal_name,
13249 "conflicting signal names",
13250 "signal wait lifecycle events at one workflow sequence disagree on identity",
13251 ));
13252 }
13253 let value = applied
13254 .first()
13255 .map(|event| decode_signal_event_arguments(event, fallback_codec))
13256 .transpose()?;
13257 return Ok(RecordedCommand::SignalWait {
13258 sequence,
13259 signal_name,
13260 value,
13261 parallel_group_path: recorded_parallel_group_path(
13262 &signal_wait_events,
13263 sequence,
13264 )?,
13265 });
13266 }
13267
13268 if !condition_wait_events.is_empty() {
13269 return recorded_condition_wait(
13270 sequence,
13271 &condition_wait_events,
13272 events,
13273 );
13274 }
13275
13276 if !search_attribute_events.is_empty() {
13277 if search_attribute_events.len() != 1 {
13278 return Err(invalid_recorded_history(
13279 "duplicate_search_attribute_update",
13280 sequence,
13281 "one SearchAttributesUpserted event",
13282 &format!(
13283 "{} SearchAttributesUpserted events",
13284 search_attribute_events.len()
13285 ),
13286 "search-attribute history records one workflow command more than once",
13287 ));
13288 }
13289 let payload = &search_attribute_events[0].payload;
13290 let attributes = payload
13291 .get("attributes")
13292 .filter(|value| value.as_object().is_some_and(|values| !values.is_empty()))
13293 .cloned()
13294 .ok_or_else(|| {
13295 invalid_recorded_history(
13296 "search_attribute_update_missing",
13297 sequence,
13298 "non-empty attributes object",
13299 "missing or invalid attributes",
13300 "search-attribute history is missing its recorded mutation",
13301 )
13302 })?;
13303 let attribute_types =
13304 recorded_search_attribute_types(payload, &attributes, sequence)?;
13305 return Ok(RecordedCommand::SearchAttributes {
13306 sequence,
13307 attributes,
13308 attribute_types,
13309 });
13310 }
13311
13312 if !side_effect_events.is_empty() {
13313 if side_effect_events.len() != 1 {
13314 return Err(invalid_recorded_history(
13315 "duplicate_side_effect_record",
13316 sequence,
13317 "one SideEffectRecorded event",
13318 &format!("{} SideEffectRecorded events", side_effect_events.len()),
13319 "side-effect history records one workflow command more than once",
13320 ));
13321 }
13322 let event = side_effect_events[0];
13323 let result = event.payload.get("result").ok_or_else(|| {
13324 invalid_recorded_history(
13325 "side_effect_result_missing",
13326 sequence,
13327 "recorded result payload",
13328 "missing result",
13329 "side-effect history is missing its recorded value",
13330 )
13331 })?;
13332 let has_published_envelope = result.as_str().is_some()
13333 || result.as_object().is_some_and(|envelope| {
13334 envelope.get("codec").and_then(Value::as_str).is_some()
13335 && envelope.get("blob").and_then(Value::as_str).is_some()
13336 });
13337 if !has_published_envelope {
13338 return Err(invalid_recorded_history(
13339 "side_effect_payload_malformed",
13340 sequence,
13341 "payload blob or {codec, blob} envelope",
13342 &result.to_string(),
13343 "side-effect history result does not use a published payload envelope",
13344 ));
13345 }
13346 let codec = event
13347 .payload
13348 .get("payload_codec")
13349 .and_then(Value::as_str)
13350 .unwrap_or(fallback_codec);
13351 let value = decode_wire_avro_value(result, codec).map_err(|error| {
13352 if error.to_string().contains("unsupported_payload_codec") {
13353 return error;
13354 }
13355
13356 invalid_recorded_history(
13357 "side_effect_payload_incompatible",
13358 sequence,
13359 &format!("valid {codec} payload envelope"),
13360 &error.to_string(),
13361 "side-effect history payload cannot be decoded with its recorded codec",
13362 )
13363 })?;
13364 return Ok(RecordedCommand::SideEffect { sequence, value });
13365 }
13366
13367 if !version_marker_events.is_empty() {
13368 if version_marker_events.len() != 1 {
13369 return Err(invalid_recorded_history(
13370 "duplicate_version_marker_record",
13371 sequence,
13372 "one VersionMarkerRecorded event",
13373 &format!("{} VersionMarkerRecorded events", version_marker_events.len()),
13374 "version-marker history records one workflow command more than once",
13375 ));
13376 }
13377 let payload = &version_marker_events[0].payload;
13378 let change_id = payload
13379 .get("change_id")
13380 .and_then(Value::as_str)
13381 .filter(|value| !value.is_empty())
13382 .map(str::to_string)
13383 .ok_or_else(|| {
13384 invalid_recorded_history(
13385 "version_marker_field_missing",
13386 sequence,
13387 "non-empty change_id",
13388 "missing or invalid change_id",
13389 "version-marker history is missing its stable change ID",
13390 )
13391 })?;
13392 let version = required_version_i32(payload, "version", sequence)?;
13393 let min_supported = required_version_i32(payload, "min_supported", sequence)?;
13394 let max_supported = required_version_i32(payload, "max_supported", sequence)?;
13395 if min_supported > max_supported || version < min_supported || version > max_supported {
13396 return Err(invalid_recorded_history(
13397 "version_marker_history_range_invalid",
13398 sequence,
13399 "min_supported <= version <= max_supported",
13400 &format!("{min_supported} <= {version} <= {max_supported}"),
13401 "recorded version marker contains an internally incompatible range",
13402 ));
13403 }
13404 return Ok(RecordedCommand::VersionMarker {
13405 sequence,
13406 change_id,
13407 version,
13408 });
13409 }
13410
13411 if !memo_events.is_empty() {
13412 if memo_events.len() != 1 {
13413 return Err(invalid_recorded_history(
13414 "duplicate_memo_upsert_record",
13415 sequence,
13416 "one MemoUpserted event",
13417 &format!("{} MemoUpserted events", memo_events.len()),
13418 "memo history records one workflow update more than once",
13419 ));
13420 }
13421 let payload = &memo_events[0].payload;
13422 let entries = payload.get("entries").cloned().ok_or_else(|| {
13423 invalid_recorded_history(
13424 "memo_entries_missing",
13425 sequence,
13426 "memo entries object",
13427 "missing entries",
13428 "MemoUpserted history is missing replay identity entries",
13429 )
13430 })?;
13431 let entries = decode_memo_history_map(&entries, true).map_err(|error| {
13432 invalid_recorded_history(
13433 "memo_entries_invalid",
13434 sequence,
13435 "valid canonical memo entries",
13436 &error.to_string(),
13437 "MemoUpserted history contains invalid replay identity entries",
13438 )
13439 })?;
13440 let merged = payload.get("merged").cloned().ok_or_else(|| {
13441 invalid_recorded_history(
13442 "memo_merged_projection_missing",
13443 sequence,
13444 "merged memo projection",
13445 "missing merged",
13446 "MemoUpserted history is missing its merged projection",
13447 )
13448 })?;
13449 decode_memo_history_map(&merged, false).map_err(|error| {
13450 invalid_recorded_history(
13451 "memo_merged_projection_invalid",
13452 sequence,
13453 "valid merged memo projection",
13454 &error.to_string(),
13455 "MemoUpserted history contains an invalid merged projection",
13456 )
13457 })?;
13458
13459 return Ok(RecordedCommand::Memo { sequence, entries });
13460 }
13461 let scheduled: Vec<_> = timer_events
13462 .iter()
13463 .copied()
13464 .filter(|event| event.event_type == "TimerScheduled")
13465 .collect();
13466 let fired: Vec<_> = timer_events
13467 .iter()
13468 .copied()
13469 .filter(|event| event.event_type == "TimerFired")
13470 .collect();
13471 if scheduled.len() != 1 {
13472 return Err(invalid_recorded_history(
13473 "timer_schedule_missing_or_duplicate",
13474 sequence,
13475 "one TimerScheduled event",
13476 &format!("{} TimerScheduled events", scheduled.len()),
13477 "timer replay requires exactly one recorded schedule event",
13478 ));
13479 }
13480 if fired.len() > 1 {
13481 return Err(invalid_recorded_history(
13482 "duplicate_timer_fire",
13483 sequence,
13484 "at most one TimerFired event",
13485 "multiple TimerFired events",
13486 "timer history contains more than one fire event for a workflow sequence",
13487 ));
13488 }
13489
13490 let scheduled = scheduled[0];
13491 let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
13492 let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
13493 if let Some(fired) = fired.first() {
13494 let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
13495 if fired_timer_id != timer_id {
13496 return Err(invalid_recorded_history(
13497 "timer_identity_mismatch",
13498 sequence,
13499 &timer_id,
13500 &fired_timer_id,
13501 "TimerFired does not correspond to the recorded TimerScheduled event",
13502 ));
13503 }
13504 let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
13505 if fired_delay != delay_seconds {
13506 return Err(invalid_recorded_history(
13507 "timer_history_delay_mismatch",
13508 sequence,
13509 &delay_seconds.to_string(),
13510 &fired_delay.to_string(),
13511 "TimerScheduled and TimerFired record different delays",
13512 ));
13513 }
13514 }
13515
13516 Ok(RecordedCommand::Timer {
13517 sequence,
13518 delay_seconds,
13519 fired: !fired.is_empty(),
13520 parallel_group_path: recorded_parallel_group_path(&timer_events, sequence)?,
13521 })
13522 })
13523 .collect::<Result<_>>()?;
13524
13525 let mut marker_sequences = HashMap::new();
13526 for command in &commands {
13527 if let RecordedCommand::VersionMarker {
13528 sequence,
13529 change_id,
13530 ..
13531 } = command
13532 {
13533 if let Some(first_sequence) = marker_sequences.insert(change_id.clone(), *sequence) {
13534 return Err(invalid_recorded_history(
13535 "duplicate_version_marker",
13536 *sequence,
13537 &format!("one marker for change ID {change_id:?}"),
13538 &format!("markers at sequences {first_sequence} and {sequence}"),
13539 "workflow history contains duplicate markers for one stable change ID",
13540 ));
13541 }
13542 }
13543 }
13544
13545 Ok(commands)
13546}
13547
13548fn required_version_i32(payload: &Value, field: &str, sequence: u64) -> Result<i32> {
13549 payload
13550 .get(field)
13551 .and_then(Value::as_i64)
13552 .and_then(|value| i32::try_from(value).ok())
13553 .ok_or_else(|| {
13554 invalid_recorded_history(
13555 "version_marker_field_missing",
13556 sequence,
13557 &format!("integer {field}"),
13558 "missing or out-of-range integer",
13559 "version-marker history is missing a required integer field",
13560 )
13561 })
13562}
13563
13564fn durable_event_sequence(event: &HistoryEvent) -> Option<u64> {
13565 event
13566 .payload
13567 .get("sequence")
13568 .or_else(|| event.payload.get("workflow_sequence"))
13569 .or_else(|| event.raw.get("sequence"))
13570 .or_else(|| event.raw.get("workflow_sequence"))
13571 .and_then(value_as_u64)
13572}
13573
13574fn is_internal_timer_event(event: &HistoryEvent) -> bool {
13575 matches!(
13576 event
13577 .payload
13578 .get("timer_kind")
13579 .or_else(|| event.raw.get("timer_kind"))
13580 .and_then(Value::as_str),
13581 Some("condition_timeout" | "signal_timeout")
13582 )
13583}
13584
13585fn is_recorded_condition_wait_event(event: &HistoryEvent) -> bool {
13586 matches!(
13587 event.event_type.as_str(),
13588 "ConditionWaitOpened" | "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
13589 )
13590}
13591
13592fn recorded_condition_wait(
13593 sequence: u64,
13594 condition_events: &[&HistoryEvent],
13595 all_events: &[HistoryEvent],
13596) -> Result<RecordedCommand> {
13597 let opened = condition_events
13598 .iter()
13599 .copied()
13600 .filter(|event| event.event_type == "ConditionWaitOpened")
13601 .collect::<Vec<_>>();
13602 if opened.len() != 1 {
13603 return Err(invalid_recorded_history(
13604 "condition_wait_open_missing_or_duplicate",
13605 sequence,
13606 "one ConditionWaitOpened event",
13607 &format!("{} ConditionWaitOpened events", opened.len()),
13608 "condition replay requires exactly one canonical wait-open event",
13609 ));
13610 }
13611 let terminal = condition_events
13612 .iter()
13613 .copied()
13614 .filter(|event| {
13615 matches!(
13616 event.event_type.as_str(),
13617 "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
13618 )
13619 })
13620 .collect::<Vec<_>>();
13621 if terminal.len() > 1 {
13622 return Err(invalid_recorded_history(
13623 "duplicate_condition_wait_terminal_event",
13624 sequence,
13625 "at most one condition terminal event",
13626 "multiple condition terminal events",
13627 "condition history settles one durable wait more than once",
13628 ));
13629 }
13630
13631 let opened = opened[0];
13632 let condition_wait_id = required_condition_wait_id(opened, sequence)?;
13633 let occurrence_id = required_condition_wait_occurrence_id(opened, sequence)?;
13634 for event in condition_events
13635 .iter()
13636 .copied()
13637 .filter(|event| !std::ptr::eq(*event, opened))
13638 {
13639 let event_wait_id = required_condition_wait_id(event, sequence)?;
13640 if event_wait_id != condition_wait_id {
13641 return Err(invalid_recorded_history(
13642 "condition_wait_id_mismatch",
13643 sequence,
13644 &condition_wait_id,
13645 &event_wait_id,
13646 "condition lifecycle events at one sequence disagree on wait identity",
13647 ));
13648 }
13649 let event_occurrence_id = required_condition_wait_occurrence_id(event, sequence)?;
13650 if event_occurrence_id != occurrence_id {
13651 return Err(invalid_recorded_history(
13652 "condition_wait_occurrence_history_mismatch",
13653 sequence,
13654 &occurrence_id,
13655 &event_occurrence_id,
13656 "condition lifecycle events at one sequence disagree on authored occurrence identity",
13657 ));
13658 }
13659 }
13660
13661 let condition_key = optional_non_empty_history_string(opened, "condition_key");
13662 let predicate_identity = opened
13663 .payload
13664 .get("condition_definition_fingerprint")
13665 .and_then(Value::as_str)
13666 .filter(|value| !value.is_empty())
13667 .map(str::to_string)
13668 .ok_or_else(|| {
13669 invalid_recorded_history(
13670 "condition_wait_predicate_fingerprint_missing",
13671 sequence,
13672 "non-empty condition_definition_fingerprint",
13673 &opened.event_type,
13674 "canonical condition history is missing its predicate identity",
13675 )
13676 })?;
13677 let timeout_seconds = optional_history_u64(opened, "timeout_seconds", sequence)?;
13678 for event in condition_events
13679 .iter()
13680 .copied()
13681 .filter(|event| !std::ptr::eq(*event, opened))
13682 {
13683 for (field, opened_value) in [
13684 ("condition_key", condition_key.as_deref()),
13685 (
13686 "condition_definition_fingerprint",
13687 Some(predicate_identity.as_str()),
13688 ),
13689 ] {
13690 if let Some(value) = optional_non_empty_history_string(event, field) {
13691 if opened_value.is_some_and(|opened_value| opened_value != value) {
13692 return Err(invalid_recorded_history(
13693 "condition_wait_definition_history_mismatch",
13694 sequence,
13695 opened_value.unwrap_or_default(),
13696 &value,
13697 "condition lifecycle events disagree on the recorded definition",
13698 ));
13699 }
13700 }
13701 }
13702 if let Some(event_timeout) = optional_history_u64(event, "timeout_seconds", sequence)? {
13703 if timeout_seconds.is_some_and(|opened_timeout| opened_timeout != event_timeout) {
13704 return Err(invalid_recorded_history(
13705 "condition_wait_definition_history_mismatch",
13706 sequence,
13707 &format!("{}s", timeout_seconds.unwrap_or_default()),
13708 &format!("{event_timeout}s"),
13709 "condition lifecycle events disagree on the recorded timeout",
13710 ));
13711 }
13712 }
13713 }
13714
13715 let timeout_timer_events = all_events
13716 .iter()
13717 .filter(|event| {
13718 matches!(
13719 event.event_type.as_str(),
13720 "TimerScheduled" | "TimerCancelled" | "TimerFired"
13721 ) && event.payload.get("timer_kind").and_then(Value::as_str)
13722 == Some("condition_timeout")
13723 && event
13724 .payload
13725 .get("condition_wait_id")
13726 .and_then(Value::as_str)
13727 == Some(condition_wait_id.as_str())
13728 })
13729 .collect::<Vec<_>>();
13730 let scheduled = timeout_timer_events
13731 .iter()
13732 .copied()
13733 .filter(|event| event.event_type == "TimerScheduled")
13734 .collect::<Vec<_>>();
13735 let fired = timeout_timer_events
13736 .iter()
13737 .copied()
13738 .filter(|event| event.event_type == "TimerFired")
13739 .collect::<Vec<_>>();
13740 if scheduled.len() > 1 || fired.len() > 1 || (!fired.is_empty() && scheduled.len() != 1) {
13741 return Err(invalid_recorded_history(
13742 "condition_wait_timeout_history_invalid",
13743 sequence,
13744 "one timeout schedule and at most one fire",
13745 &format!("{} schedules and {} fires", scheduled.len(), fired.len()),
13746 "condition timeout history has a missing or duplicate lifecycle event",
13747 ));
13748 }
13749 if let Some(scheduled) = scheduled.first() {
13750 let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
13751 let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
13752 if timeout_seconds.is_some_and(|timeout| timeout != delay_seconds) {
13753 return Err(invalid_recorded_history(
13754 "condition_wait_timeout_delay_mismatch",
13755 sequence,
13756 &format!("{}s", timeout_seconds.unwrap_or_default()),
13757 &format!("{delay_seconds}s"),
13758 "condition timeout timer differs from the wait definition",
13759 ));
13760 }
13761 if let Some(fired) = fired.first() {
13762 let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
13763 let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
13764 if fired_timer_id != timer_id || fired_delay != delay_seconds {
13765 return Err(invalid_recorded_history(
13766 "condition_wait_timeout_identity_mismatch",
13767 sequence,
13768 &format!("{timer_id}:{delay_seconds}s"),
13769 &format!("{fired_timer_id}:{fired_delay}s"),
13770 "condition timeout fire does not match its durable schedule",
13771 ));
13772 }
13773 }
13774 }
13775
13776 let result = terminal.first().map(|event| {
13777 if event.event_type == "ConditionWaitTimedOut" {
13778 ConditionWaitResult::TimedOut
13779 } else {
13780 ConditionWaitResult::Satisfied
13781 }
13782 });
13783 let result = if !fired.is_empty() {
13784 if result == Some(ConditionWaitResult::Satisfied) {
13785 return Err(invalid_recorded_history(
13786 "condition_wait_terminal_conflict",
13787 sequence,
13788 "one satisfied or timed-out outcome",
13789 "satisfied event and fired timeout",
13790 "condition history records conflicting terminal outcomes",
13791 ));
13792 }
13793 Some(ConditionWaitResult::TimedOut)
13794 } else {
13795 result
13796 };
13797
13798 Ok(RecordedCommand::ConditionWait {
13799 sequence,
13800 occurrence_id,
13801 condition_key,
13802 predicate_identity,
13803 timeout_seconds,
13804 result,
13805 parallel_group_path: recorded_parallel_group_path(condition_events, sequence)?,
13806 })
13807}
13808
13809fn required_condition_wait_occurrence_id(event: &HistoryEvent, sequence: u64) -> Result<String> {
13810 event
13811 .payload
13812 .get("condition_wait_occurrence_id")
13813 .and_then(Value::as_str)
13814 .filter(|value| !value.is_empty())
13815 .map(str::to_string)
13816 .ok_or_else(|| {
13817 invalid_recorded_history(
13818 "condition_wait_occurrence_id_missing",
13819 sequence,
13820 "non-empty condition_wait_occurrence_id",
13821 &event.event_type,
13822 "condition history is missing authored occurrence identity",
13823 )
13824 })
13825}
13826
13827fn required_condition_wait_id(event: &HistoryEvent, sequence: u64) -> Result<String> {
13828 event
13829 .payload
13830 .get("condition_wait_id")
13831 .and_then(Value::as_str)
13832 .filter(|value| !value.is_empty())
13833 .map(str::to_string)
13834 .ok_or_else(|| {
13835 invalid_recorded_history(
13836 "condition_wait_id_missing",
13837 sequence,
13838 "non-empty condition_wait_id",
13839 &event.event_type,
13840 "canonical condition history is missing its durable wait identity",
13841 )
13842 })
13843}
13844
13845fn optional_non_empty_history_string(event: &HistoryEvent, field: &str) -> Option<String> {
13846 event
13847 .payload
13848 .get(field)
13849 .and_then(Value::as_str)
13850 .filter(|value| !value.is_empty())
13851 .map(str::to_string)
13852}
13853
13854fn optional_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<Option<u64>> {
13855 match event.payload.get(field) {
13856 None | Some(Value::Null) => Ok(None),
13857 Some(value) => value_as_u64(value).map(Some).ok_or_else(|| {
13858 invalid_recorded_history(
13859 "condition_wait_definition_invalid",
13860 sequence,
13861 &format!("non-negative integer {field}"),
13862 &value.to_string(),
13863 "condition history contains an invalid numeric definition field",
13864 )
13865 }),
13866 }
13867}
13868
13869fn required_signal_wait_name(event: &HistoryEvent, sequence: u64) -> Result<String> {
13870 event
13871 .payload
13872 .get("signal_name")
13873 .or_else(|| event.raw.get("signal_name"))
13874 .and_then(Value::as_str)
13875 .filter(|value| !value.is_empty())
13876 .map(str::to_string)
13877 .ok_or_else(|| {
13878 invalid_recorded_history(
13879 "signal_wait_name_missing",
13880 sequence,
13881 "non-empty signal_name",
13882 &event.event_type,
13883 "canonical signal-wait history is missing its signal identity",
13884 )
13885 })
13886}
13887
13888fn is_recorded_signal_wait_event(event: &HistoryEvent) -> bool {
13889 matches!(
13890 event.event_type.as_str(),
13891 "SignalWaitOpened" | "SignalApplied"
13892 )
13893}
13894
13895fn required_history_string(event: &HistoryEvent, field: &str, sequence: u64) -> Result<String> {
13896 event
13897 .payload
13898 .get(field)
13899 .and_then(Value::as_str)
13900 .filter(|value| !value.is_empty())
13901 .map(str::to_string)
13902 .ok_or_else(|| {
13903 invalid_recorded_history(
13904 "timer_history_field_missing",
13905 sequence,
13906 field,
13907 &event.event_type,
13908 "timer history is missing a required identity field",
13909 )
13910 })
13911}
13912
13913fn required_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<u64> {
13914 event
13915 .payload
13916 .get(field)
13917 .and_then(value_as_u64)
13918 .ok_or_else(|| {
13919 invalid_recorded_history(
13920 "timer_history_field_missing",
13921 sequence,
13922 field,
13923 &event.event_type,
13924 "timer history is missing a required numeric field",
13925 )
13926 })
13927}
13928
13929fn recorded_search_attribute_types(
13930 payload: &Value,
13931 attributes: &Value,
13932 sequence: u64,
13933) -> Result<RecordedSnapshotValue<BTreeMap<String, String>>> {
13934 let Some(raw_types) = payload.get("attribute_types") else {
13935 return Ok(RecordedSnapshotValue::Unknown);
13939 };
13940 let Some(raw_types) = raw_types.as_object() else {
13941 return Err(invalid_recorded_history(
13942 "search_attribute_types_malformed",
13943 sequence,
13944 "canonical attribute type map",
13945 &raw_types.to_string(),
13946 "search-attribute history contains malformed type identity",
13947 ));
13948 };
13949 let attribute_keys = attributes
13950 .as_object()
13951 .expect("recorded search attributes were validated as an object");
13952 let mut types = BTreeMap::new();
13953 for (key, value) in raw_types {
13954 let Some(attribute_type) = value.as_str() else {
13955 return Err(invalid_recorded_history(
13956 "search_attribute_types_malformed",
13957 sequence,
13958 "canonical string type name",
13959 &value.to_string(),
13960 "search-attribute history contains a non-string type identity",
13961 ));
13962 };
13963 if !attribute_keys.contains_key(key)
13964 || !matches!(
13965 attribute_type,
13966 "string" | "keyword" | "keyword_list" | "int" | "float" | "bool" | "datetime"
13967 )
13968 {
13969 return Err(invalid_recorded_history(
13970 "search_attribute_types_malformed",
13971 sequence,
13972 "canonical types for keys present in attributes",
13973 &format!("{key}:{attribute_type}"),
13974 "search-attribute history contains unsupported or orphaned type identity",
13975 ));
13976 }
13977 types.insert(key.clone(), attribute_type.to_string());
13978 }
13979 Ok(RecordedSnapshotValue::Known(types))
13980}
13981
13982fn invalid_recorded_history(
13983 reason: &str,
13984 sequence: u64,
13985 expected: &str,
13986 actual: &str,
13987 message: &str,
13988) -> Error {
13989 Error::NonDeterministicReplay(ReplayFailure::new(
13990 reason,
13991 Some(sequence),
13992 Some(expected.to_string()),
13993 Some(actual.to_string()),
13994 message,
13995 ))
13996}
13997
13998type ActivityOutcome = std::result::Result<AvroValue, ActivityFailure>;
13999
14000fn activity_outcome(
14001 event: &HistoryEvent,
14002 fallback_codec: &str,
14003 recorded_activity_type: Option<String>,
14004) -> Result<ActivityOutcome> {
14005 if event.event_type == "ActivityCompleted" {
14006 let codec = event
14007 .payload
14008 .get("payload_codec")
14009 .and_then(Value::as_str)
14010 .unwrap_or(fallback_codec);
14011 return Ok(Ok(decode_wire_avro_value(
14012 event.payload.get("result").unwrap_or(&Value::Null),
14013 codec,
14014 )?));
14015 }
14016
14017 let payload = &event.payload;
14018 let (kind, fallback_reason, fallback_message) = match event.event_type.as_str() {
14019 "ActivityFailed" => (ActivityFailureKind::Failed, "activity", "activity failed"),
14020 "ActivityCancelled" => (
14021 ActivityFailureKind::Cancelled,
14022 "cancelled",
14023 "activity was cancelled",
14024 ),
14025 "ActivityTimedOut" => (
14026 ActivityFailureKind::TimedOut,
14027 "timeout",
14028 "activity timed out",
14029 ),
14030 _ => unreachable!("activity_outcome is called only for terminal activity events"),
14031 };
14032 let exception = payload
14033 .get("exception")
14034 .filter(|value| !value.is_null())
14035 .cloned();
14036 let failure_category = payload_string(payload, "failure_category");
14037 let timeout_kind = payload_string(payload, "timeout_kind");
14038 let reason = payload_string(payload, "reason").unwrap_or_else(|| match kind {
14039 ActivityFailureKind::Failed => failure_category
14040 .clone()
14041 .unwrap_or_else(|| fallback_reason.to_string()),
14042 ActivityFailureKind::Cancelled => fallback_reason.to_string(),
14043 ActivityFailureKind::TimedOut => timeout_kind
14044 .clone()
14045 .unwrap_or_else(|| fallback_reason.to_string()),
14046 });
14047 let message = payload_string(payload, "message")
14048 .or_else(|| {
14049 exception
14050 .as_ref()
14051 .and_then(|value| payload_string(value, "message"))
14052 })
14053 .unwrap_or_else(|| fallback_message.to_string());
14054
14055 Ok(Err(ActivityFailure {
14056 kind,
14057 reason,
14058 message,
14059 activity_execution_id: payload_string(payload, "activity_execution_id"),
14060 activity_attempt_id: payload_string(payload, "activity_attempt_id"),
14061 activity_type: payload_string(payload, "activity_type")
14062 .or_else(|| payload_string(payload, "activity_name"))
14063 .or(recorded_activity_type),
14064 activity_class: payload_string(payload, "activity_class"),
14065 attempt_number: payload.get("attempt_number").and_then(value_as_u64),
14066 failure_id: payload_string(payload, "failure_id"),
14067 failure_category,
14068 timeout_kind,
14069 non_retryable: payload
14070 .get("non_retryable")
14071 .and_then(Value::as_bool)
14072 .unwrap_or(false),
14073 exception_type: payload_string(payload, "exception_type").or_else(|| {
14074 exception
14075 .as_ref()
14076 .and_then(|value| payload_string(value, "type"))
14077 }),
14078 exception_class: payload_string(payload, "exception_class").or_else(|| {
14079 exception
14080 .as_ref()
14081 .and_then(|value| payload_string(value, "class"))
14082 }),
14083 code: payload
14084 .get("code")
14085 .filter(|value| !value.is_null())
14086 .cloned(),
14087 exception,
14088 }))
14089}
14090
14091type ChildWorkflowOutcome = std::result::Result<ChildWorkflowAvroResult, ChildWorkflowFailure>;
14092
14093fn child_workflow_outcomes(
14094 events: &[HistoryEvent],
14095 fallback_codec: &str,
14096 parent: WorkflowIdentity,
14097) -> Result<Vec<ChildWorkflowOutcome>> {
14098 let mut outcomes = Vec::new();
14099
14100 for event in events {
14101 let kind = match event.event_type.as_str() {
14102 "ChildRunCompleted" => None,
14103 "ChildRunFailed" => Some((
14104 ChildWorkflowFailureKind::Failed,
14105 "child_workflow",
14106 "child workflow failed",
14107 )),
14108 "ChildRunCancelled" => Some((
14109 ChildWorkflowFailureKind::Cancelled,
14110 "cancelled",
14111 "child workflow was cancelled",
14112 )),
14113 "ChildRunTerminated" => Some((
14114 ChildWorkflowFailureKind::Terminated,
14115 "terminated",
14116 "child workflow was terminated",
14117 )),
14118 _ => continue,
14119 };
14120 let payload = &event.payload;
14121 let child_workflow_id = payload_string(payload, "child_workflow_instance_id");
14122 let child_workflow_run_id = payload_string(payload, "child_workflow_run_id");
14123 let child_workflow_type = payload_string(payload, "child_workflow_type");
14124
14125 if let Some((kind, reason, fallback_message)) = kind {
14126 let exception = payload
14127 .get("exception")
14128 .filter(|value| !value.is_null())
14129 .cloned();
14130 let message = payload_string(payload, "message")
14131 .or_else(|| {
14132 exception
14133 .as_ref()
14134 .and_then(|value| payload_string(value, "message"))
14135 })
14136 .unwrap_or_else(|| fallback_message.to_string());
14137 let exception_type = payload_string(payload, "exception_type").or_else(|| {
14138 exception
14139 .as_ref()
14140 .and_then(|value| payload_string(value, "type"))
14141 });
14142 let exception_class = payload_string(payload, "exception_class").or_else(|| {
14143 exception
14144 .as_ref()
14145 .and_then(|value| payload_string(value, "class"))
14146 });
14147 outcomes.push(Err(ChildWorkflowFailure {
14148 kind,
14149 reason: reason.to_string(),
14150 message,
14151 parent_workflow_id: parent.workflow_id.clone(),
14152 parent_workflow_run_id: parent.run_id.clone(),
14153 child_workflow_id,
14154 child_workflow_run_id,
14155 child_workflow_type,
14156 failure_id: payload_string(payload, "failure_id"),
14157 failure_category: payload_string(payload, "failure_category"),
14158 exception_type,
14159 exception_class,
14160 non_retryable: payload
14161 .get("non_retryable")
14162 .and_then(Value::as_bool)
14163 .unwrap_or(false),
14164 code: payload
14165 .get("code")
14166 .filter(|value| !value.is_null())
14167 .cloned(),
14168 exception,
14169 }));
14170 continue;
14171 }
14172
14173 let codec = payload
14174 .get("payload_codec")
14175 .and_then(Value::as_str)
14176 .unwrap_or(fallback_codec);
14177 let result = payload
14178 .get("result")
14179 .or_else(|| payload.get("output"))
14180 .unwrap_or(&Value::Null);
14181 outcomes.push(Ok(ChildWorkflowAvroResult {
14182 parent: parent.clone(),
14183 child: WorkflowIdentity {
14184 workflow_id: child_workflow_id,
14185 run_id: child_workflow_run_id,
14186 },
14187 child_workflow_type,
14188 result: decode_wire_avro_value(result, codec)?,
14189 }));
14190 }
14191
14192 Ok(outcomes)
14193}
14194
14195fn payload_string(payload: &Value, key: &str) -> Option<String> {
14196 payload
14197 .get(key)
14198 .and_then(Value::as_str)
14199 .filter(|value| !value.is_empty())
14200 .map(str::to_string)
14201}
14202
14203fn workflow_failure_command(error: &Error) -> Value {
14204 let (exception_type, exception_class, properties) = match error {
14205 Error::ActivityFailed(failure) => (
14206 match failure.kind {
14207 ActivityFailureKind::Failed => "ActivityFailed",
14208 ActivityFailureKind::Cancelled => "ActivityCancelled",
14209 ActivityFailureKind::TimedOut => "ActivityTimedOut",
14210 },
14211 "durable_workflow::ActivityFailure",
14212 json!({
14213 "reason": failure.reason,
14214 "activity_execution_id": failure.activity_execution_id,
14215 "activity_attempt_id": failure.activity_attempt_id,
14216 "activity_type": failure.activity_type,
14217 "activity_class": failure.activity_class,
14218 "attempt_number": failure.attempt_number,
14219 "failure_id": failure.failure_id,
14220 "failure_category": failure.failure_category,
14221 "timeout_kind": failure.timeout_kind,
14222 "activity_non_retryable": failure.non_retryable,
14223 "activity_exception_type": failure.exception_type,
14224 "activity_exception_class": failure.exception_class,
14225 "activity_code": failure.code,
14226 "activity_exception": failure.exception,
14227 }),
14228 ),
14229 Error::ChildWorkflowFailed(failure) => (
14230 match failure.kind {
14231 ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
14232 ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
14233 ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
14234 },
14235 "durable_workflow::ChildWorkflowFailure",
14236 json!({
14237 "reason": failure.reason,
14238 "parent_workflow_id": failure.parent_workflow_id,
14239 "parent_workflow_run_id": failure.parent_workflow_run_id,
14240 "child_workflow_id": failure.child_workflow_id,
14241 "child_workflow_run_id": failure.child_workflow_run_id,
14242 "child_workflow_type": failure.child_workflow_type,
14243 "failure_id": failure.failure_id,
14244 "failure_category": failure.failure_category,
14245 "child_exception_type": failure.exception_type,
14246 "child_exception_class": failure.exception_class,
14247 "child_non_retryable": failure.non_retryable,
14248 "child_code": failure.code,
14249 "child_exception": failure.exception,
14250 }),
14251 ),
14252 Error::ParallelFailed(failure) => (
14253 "ParallelFailed",
14254 "durable_workflow::ParallelFailure",
14255 json!({
14256 "parallel_group_id": failure.group_id,
14257 "parallel_member_path": failure.member_path,
14258 "parallel_group_path": failure.group_path,
14259 "completed_members": failure.completed.iter().map(|completion| &completion.member_path).collect::<Vec<_>>(),
14260 "cause_type": workflow_error_type(&failure.cause),
14261 "cause_message": failure.cause.to_string(),
14262 }),
14263 ),
14264 Error::SagaCompensationFailed(failure) => (
14265 "SagaCompensationFailed",
14266 "durable_workflow::SagaCompensationFailure",
14267 json!({
14268 "initiating_failure_type": workflow_error_type(&failure.initiating_failure),
14269 "initiating_failure_message": failure.initiating_failure.to_string(),
14270 "compensation_activity_type": failure.compensation_activity_type,
14271 "compensation_registration_order": failure.compensation_registration_order,
14272 "compensation_failure_type": workflow_error_type(&failure.compensation_failure),
14273 "compensation_failure_message": failure.compensation_failure.to_string(),
14274 }),
14275 ),
14276 Error::WorkflowCancellationRequested(_) => (
14277 "WorkflowCancellationRequested",
14278 "durable_workflow::WorkflowCancellationRequested",
14279 json!({"reason": "cancelled"}),
14280 ),
14281 Error::NonDeterministicReplay(_) => (
14282 "NonDeterministicReplay",
14283 "durable_workflow::Error",
14284 Value::Null,
14285 ),
14286 _ => ("RustWorkflowError", "durable_workflow::Error", Value::Null),
14287 };
14288 let non_retryable = match error {
14289 Error::ActivityFailed(failure) => failure.non_retryable,
14290 Error::ChildWorkflowFailed(failure) => failure.non_retryable,
14291 Error::ParallelFailed(failure) => workflow_error_non_retryable(&failure.cause),
14292 Error::SagaCompensationFailed(failure) => {
14293 workflow_error_non_retryable(&failure.compensation_failure)
14294 }
14295 Error::WorkflowCancellationRequested(_) => true,
14296 Error::NonDeterministicReplay(_) => true,
14297 _ => false,
14298 };
14299
14300 json!({
14301 "type": "fail_workflow",
14302 "message": error.to_string(),
14303 "exception_type": exception_type,
14304 "exception_class": exception_class,
14305 "non_retryable": non_retryable,
14306 "exception": {
14307 "type": exception_type,
14308 "class": exception_class,
14309 "message": error.to_string(),
14310 "properties": properties,
14311 }
14312 })
14313}
14314
14315fn workflow_error_type(error: &Error) -> &'static str {
14316 match error {
14317 Error::ActivityFailed(failure) => match failure.kind {
14318 ActivityFailureKind::Failed => "ActivityFailed",
14319 ActivityFailureKind::Cancelled => "ActivityCancelled",
14320 ActivityFailureKind::TimedOut => "ActivityTimedOut",
14321 },
14322 Error::ChildWorkflowFailed(failure) => match failure.kind {
14323 ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
14324 ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
14325 ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
14326 },
14327 Error::ParallelFailed(_) => "ParallelFailed",
14328 Error::SagaCompensationFailed(_) => "SagaCompensationFailed",
14329 Error::WorkflowCancellationRequested(_) => "WorkflowCancellationRequested",
14330 Error::NonDeterministicReplay(_) => "NonDeterministicReplay",
14331 _ => "RustWorkflowError",
14332 }
14333}
14334
14335fn workflow_error_non_retryable(error: &Error) -> bool {
14336 match error {
14337 Error::ActivityFailed(failure) => failure.non_retryable,
14338 Error::ChildWorkflowFailed(failure) => failure.non_retryable,
14339 Error::ParallelFailed(failure) => workflow_error_non_retryable(&failure.cause),
14340 Error::SagaCompensationFailed(failure) => {
14341 workflow_error_non_retryable(&failure.compensation_failure)
14342 }
14343 Error::WorkflowCancellationRequested(_) | Error::NonDeterministicReplay(_) => true,
14344 _ => false,
14345 }
14346}
14347
14348fn workflow_task_integrity_error(error: &Error) -> bool {
14349 matches!(
14350 error,
14351 Error::NonDeterministicReplay(_)
14352 | Error::Protocol(_)
14353 | Error::MissingWorkflowCommandIdentity
14354 | Error::WorkflowStatePoisoned
14355 )
14356}
14357
14358fn decode_signal_event_arguments(
14359 event: &HistoryEvent,
14360 fallback_codec: &str,
14361) -> Result<Vec<AvroValue>> {
14362 let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
14363 validate_payload_codec(codec)?;
14364 let raw = signal_history_payload(&event.payload);
14365 let decoded = match raw.filter(|value| !value.is_null()) {
14366 Some(value) => decode_wire_avro_value(value, codec)?,
14367 None => AvroValue::Array(Vec::new()),
14368 };
14369 let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
14370 unreachable!("normalize_avro_arguments always returns an array");
14371 };
14372 Ok(arguments)
14373}
14374
14375fn decode_update_event_arguments(
14376 event: &HistoryEvent,
14377 fallback_codec: &str,
14378) -> Result<Vec<AvroValue>> {
14379 let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
14380 validate_payload_codec(codec)?;
14381 let decoded = match event
14382 .payload
14383 .get("arguments")
14384 .filter(|value| !value.is_null())
14385 {
14386 Some(value) => decode_wire_avro_value(value, codec)?,
14387 None => AvroValue::Array(Vec::new()),
14388 };
14389 let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
14390 unreachable!("normalize_avro_arguments always returns an array");
14391 };
14392 Ok(arguments)
14393}
14394
14395fn hydrate_query_history_from_export(task: &mut QueryTask) -> Result<()> {
14396 let Some(export_events) = task
14397 .history_export
14398 .as_ref()
14399 .and_then(|export| export.get("history_events"))
14400 .and_then(Value::as_array)
14401 else {
14402 return Ok(());
14403 };
14404
14405 if export_events.len() > task.history_events.len() {
14406 task.history_events = serde_json::from_value(Value::Array(export_events.clone()))?;
14407 }
14408
14409 Ok(())
14410}
14411
14412fn enrich_query_history_from_export(task: &mut QueryTask) -> Result<()> {
14413 let Some(export) = task.history_export.as_ref() else {
14414 return Ok(());
14415 };
14416 let signals = export
14417 .get("signals")
14418 .and_then(Value::as_array)
14419 .cloned()
14420 .unwrap_or_default();
14421 let activities = export
14422 .get("activities")
14423 .and_then(Value::as_array)
14424 .cloned()
14425 .unwrap_or_default();
14426 let export_codec = export
14427 .get("payloads")
14428 .and_then(|payloads| payloads.get("codec"))
14429 .and_then(Value::as_str)
14430 .unwrap_or(&task.payload_codec)
14431 .to_string();
14432 let mut signal_name_offsets: HashMap<String, usize> = HashMap::new();
14433
14434 for event in &mut task.history_events {
14435 if event.event_type == "ActivityCompleted" {
14436 let sequence = event
14437 .payload
14438 .get("sequence")
14439 .or_else(|| event.payload.get("workflow_sequence"))
14440 .and_then(value_as_u64);
14441 let Some(activity) = sequence.and_then(|sequence| {
14442 activities.iter().find(|activity| {
14443 activity.get("sequence").and_then(value_as_u64) == Some(sequence)
14444 })
14445 }) else {
14446 continue;
14447 };
14448 let Some(payload) = event.payload.as_object_mut() else {
14449 continue;
14450 };
14451 if missing_payload(payload.get("result")) {
14452 if let Some(result) = activity
14453 .get("result")
14454 .filter(|value| !missing_payload(Some(value)))
14455 {
14456 payload.insert("result".to_string(), result.clone());
14457 }
14458 }
14459 for field in ["payload_codec", "activity_type"] {
14460 if payload
14461 .get(field)
14462 .and_then(Value::as_str)
14463 .unwrap_or_default()
14464 .is_empty()
14465 {
14466 if let Some(value) = activity.get(field) {
14467 payload.insert(field.to_string(), value.clone());
14468 }
14469 }
14470 }
14471 continue;
14472 }
14473
14474 if event.event_type != "SignalReceived" && event.event_type != "SignalApplied" {
14475 continue;
14476 }
14477 let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
14478 let command_id = event
14479 .payload
14480 .get("workflow_command_id")
14481 .or_else(|| event.raw.get("workflow_command_id"))
14482 .and_then(Value::as_str);
14483 let signal_name = event
14484 .payload
14485 .get("signal_name")
14486 .and_then(Value::as_str)
14487 .unwrap_or_default()
14488 .to_string();
14489 let matched = signals
14490 .iter()
14491 .find(|signal| {
14492 signal_id.is_some() && signal.get("id").and_then(Value::as_str) == signal_id
14493 })
14494 .or_else(|| {
14495 signals.iter().find(|signal| {
14496 command_id.is_some()
14497 && signal.get("command_id").and_then(Value::as_str) == command_id
14498 })
14499 })
14500 .or_else(|| {
14501 let offset = signal_name_offsets.entry(signal_name.clone()).or_default();
14502 let signal = signals
14503 .iter()
14504 .filter(|signal| {
14505 signal.get("name").and_then(Value::as_str) == Some(signal_name.as_str())
14506 })
14507 .nth(*offset);
14508 if signal.is_some() {
14509 *offset += 1;
14510 }
14511 signal
14512 });
14513 let Some(signal) = matched else {
14514 continue;
14515 };
14516 let signal_codec = signal
14517 .get("payload_codec")
14518 .and_then(Value::as_str)
14519 .unwrap_or(&export_codec);
14520 let Some(payload) = event.payload.as_object_mut() else {
14521 continue;
14522 };
14523 if missing_payload(payload.get("arguments")) {
14524 if let Some(arguments) = signal
14525 .get("arguments")
14526 .filter(|value| !missing_payload(Some(value)))
14527 {
14528 let envelope = match arguments {
14529 Value::String(blob) => json!({"codec": signal_codec, "blob": blob}),
14530 other => other.clone(),
14531 };
14532 payload.insert("arguments".to_string(), envelope);
14533 }
14534 }
14535 if payload
14536 .get("payload_codec")
14537 .and_then(Value::as_str)
14538 .unwrap_or_default()
14539 .is_empty()
14540 {
14541 payload.insert("payload_codec".to_string(), json!(signal_codec));
14542 }
14543 }
14544
14545 Ok(())
14546}
14547
14548fn missing_payload(value: Option<&Value>) -> bool {
14549 match value {
14550 None | Some(Value::Null) => true,
14551 Some(Value::String(value)) => value.is_empty(),
14552 Some(_) => false,
14553 }
14554}
14555
14556fn query_signal_events(task: &QueryTask) -> Result<Vec<QuerySignal>> {
14557 let export_signals = task
14558 .history_export
14559 .as_ref()
14560 .and_then(|export| export.get("signals"))
14561 .and_then(Value::as_array)
14562 .cloned()
14563 .unwrap_or_default();
14564 let export_codec = task
14565 .history_export
14566 .as_ref()
14567 .and_then(|export| export.get("payloads"))
14568 .and_then(|payloads| payloads.get("codec"))
14569 .and_then(Value::as_str)
14570 .unwrap_or(&task.payload_codec);
14571 let mut name_offsets: HashMap<String, usize> = HashMap::new();
14572 let mut signals = Vec::new();
14573
14574 for event in &task.history_events {
14575 if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
14576 continue;
14577 }
14578
14579 let name = event
14580 .payload
14581 .get("signal_name")
14582 .and_then(Value::as_str)
14583 .unwrap_or_default();
14584 if name.is_empty() {
14585 continue;
14586 }
14587 let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
14588 let command_id = event
14589 .payload
14590 .get("workflow_command_id")
14591 .or_else(|| event.raw.get("workflow_command_id"))
14592 .and_then(Value::as_str);
14593 let matched_export = export_signals
14594 .iter()
14595 .find(|candidate| {
14596 signal_id.is_some() && candidate.get("id").and_then(Value::as_str) == signal_id
14597 })
14598 .or_else(|| {
14599 export_signals.iter().find(|candidate| {
14600 command_id.is_some()
14601 && candidate.get("command_id").and_then(Value::as_str) == command_id
14602 })
14603 })
14604 .or_else(|| {
14605 let offset = name_offsets.entry(name.to_string()).or_default();
14606 let candidate = export_signals
14607 .iter()
14608 .filter(|candidate| candidate.get("name").and_then(Value::as_str) == Some(name))
14609 .nth(*offset);
14610 if candidate.is_some() {
14611 *offset += 1;
14612 }
14613 candidate
14614 });
14615 let codec = event
14616 .payload
14617 .get("payload_codec")
14618 .and_then(Value::as_str)
14619 .or_else(|| {
14620 matched_export
14621 .and_then(|signal| signal.get("payload_codec"))
14622 .and_then(Value::as_str)
14623 })
14624 .unwrap_or(export_codec);
14625 let raw_arguments = signal_history_payload(&event.payload)
14626 .filter(|value| !value.is_null())
14627 .or_else(|| matched_export.and_then(|signal| signal.get("arguments")));
14628 let (arguments, avro_arguments) = decode_query_signal_arguments(raw_arguments, codec)?;
14629 let workflow_sequence = event
14630 .payload
14631 .get("workflow_sequence")
14632 .and_then(value_as_u64)
14633 .or_else(|| {
14634 matched_export
14635 .and_then(|signal| signal.get("workflow_sequence"))
14636 .and_then(value_as_u64)
14637 });
14638
14639 signals.push(QuerySignal {
14640 id: signal_id.map(str::to_string).or_else(|| {
14641 matched_export
14642 .and_then(|signal| signal.get("id"))
14643 .and_then(Value::as_str)
14644 .map(str::to_string)
14645 }),
14646 name: name.to_string(),
14647 arguments,
14648 avro_arguments,
14649 workflow_sequence,
14650 });
14651 }
14652
14653 if signals.is_empty() {
14654 for signal in export_signals {
14655 if signal.get("status").and_then(Value::as_str) == Some("rejected") {
14656 continue;
14657 }
14658 let Some(name) = signal.get("name").and_then(Value::as_str) else {
14659 continue;
14660 };
14661 let codec = signal
14662 .get("payload_codec")
14663 .and_then(Value::as_str)
14664 .unwrap_or(export_codec);
14665 let (arguments, avro_arguments) =
14666 decode_query_signal_arguments(signal.get("arguments"), codec)?;
14667 signals.push(QuerySignal {
14668 id: signal.get("id").and_then(Value::as_str).map(str::to_string),
14669 name: name.to_string(),
14670 arguments,
14671 avro_arguments,
14672 workflow_sequence: signal.get("workflow_sequence").and_then(value_as_u64),
14673 });
14674 }
14675 signals.sort_by_key(|signal| signal.workflow_sequence.unwrap_or(u64::MAX));
14676 }
14677
14678 Ok(signals)
14679}
14680
14681fn decode_query_signal_arguments(
14682 raw: Option<&Value>,
14683 codec: &str,
14684) -> Result<(Vec<Value>, Vec<AvroValue>)> {
14685 validate_payload_codec(codec)?;
14686 let decoded = match raw.filter(|value| !value.is_null()) {
14687 Some(value) => decode_wire_avro_value(value, codec)?,
14688 None => AvroValue::Array(Vec::new()),
14689 };
14690 let AvroValue::Array(avro_arguments) = normalize_avro_arguments(decoded) else {
14691 unreachable!("normalize_avro_arguments always returns an array");
14692 };
14693 let arguments = avro_arguments
14694 .iter()
14695 .cloned()
14696 .map(AvroValue::into_json)
14697 .collect::<Result<Vec<_>>>()?;
14698 Ok((arguments, avro_arguments))
14699}
14700
14701fn value_as_u64(value: &Value) -> Option<u64> {
14702 value
14703 .as_u64()
14704 .or_else(|| value.as_str().and_then(|value| value.parse().ok()))
14705}
14706
14707#[cfg(test)]
14708mod tests {
14709 use super::*;
14710 use std::{
14711 fs,
14712 io::{Read, Write},
14713 net::{SocketAddr, TcpListener, TcpStream},
14714 process::Command as ProcessCommand,
14715 sync::atomic::AtomicUsize,
14716 thread,
14717 };
14718
14719 #[derive(Clone, Copy, Debug)]
14720 enum InvalidTaskPayloadCodec {
14721 Missing,
14722 Null,
14723 NonString,
14724 }
14725
14726 impl InvalidTaskPayloadCodec {
14727 fn label(self) -> &'static str {
14728 match self {
14729 Self::Missing => "missing",
14730 Self::Null => "null",
14731 Self::NonString => "non-string",
14732 }
14733 }
14734
14735 fn apply(self, task: &mut Value) {
14736 let task = task.as_object_mut().expect("task fixture object");
14737 match self {
14738 Self::Missing => {
14739 task.remove("payload_codec");
14740 }
14741 Self::Null => {
14742 task.insert("payload_codec".to_string(), Value::Null);
14743 }
14744 Self::NonString => {
14745 task.insert("payload_codec".to_string(), json!(42));
14746 }
14747 }
14748 }
14749 }
14750
14751 fn fixture_envelope(value: Value) -> Value {
14752 encode_value_envelope(&value, DEFAULT_CODEC).expect("encode Avro test fixture")
14753 }
14754
14755 fn fixture_blob(value: Value) -> String {
14756 encode_payload(&value, DEFAULT_CODEC)
14757 .expect("encode Avro test fixture")
14758 .blob
14759 }
14760
14761 #[test]
14762 fn client_builder_rejects_the_sdk_owned_api_suffix() {
14763 for base_url in [
14764 "http://127.0.0.1:8080/api",
14765 "http://localhost:8080/api/",
14766 "https://runtime.example.test/namespaces/orders/api",
14767 ] {
14768 let error = Client::builder(base_url)
14769 .build()
14770 .expect_err("SDK-owned /api suffix must be rejected during build");
14771
14772 assert!(matches!(error, Error::InvalidBaseUrl), "{base_url}");
14773 assert!(
14774 error.to_string().contains("SDK appends /api automatically"),
14775 "the validation error must explain how to fix the endpoint"
14776 );
14777 }
14778 }
14779
14780 #[test]
14781 fn client_builder_preserves_self_hosted_and_managed_runtime_prefixes() {
14782 for (base_url, expected) in [
14783 ("http://127.0.0.1:8080", "http://127.0.0.1:8080"),
14784 (
14785 "http://localhost:8080/durable-workflow/",
14786 "http://localhost:8080/durable-workflow",
14787 ),
14788 (
14789 "https://runtime.example.test/namespaces/orders",
14790 "https://runtime.example.test/namespaces/orders",
14791 ),
14792 (
14793 "https://runtime.example.test/gateway/api/namespaces/orders",
14794 "https://runtime.example.test/gateway/api/namespaces/orders",
14795 ),
14796 (
14797 "https://api.example.test/runtime/orders/",
14798 "https://api.example.test/runtime/orders",
14799 ),
14800 ] {
14801 let client = Client::builder(base_url)
14802 .build()
14803 .expect("Server and Cloud runtime base URL must remain valid");
14804
14805 assert_eq!(client.base_url, expected);
14806 }
14807 }
14808
14809 #[test]
14810 fn workflow_completion_uses_the_additive_command_protocol_floor() {
14811 assert_eq!(
14812 workflow_completion_protocol_version(&[json!({"type": "complete_workflow"})]),
14813 WORKER_PROTOCOL_VERSION
14814 );
14815 assert_eq!(
14816 workflow_completion_protocol_version(&[json!({
14817 "type": "upsert_search_attributes",
14818 "attributes": {"OrderStatus": "waiting"},
14819 })]),
14820 SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION
14821 );
14822 assert_eq!(
14823 workflow_completion_protocol_version(&[json!({
14824 "type": "upsert_search_attributes",
14825 "attributes": {"OrderStatus": "waiting"},
14826 "attribute_types": {"OrderStatus": "keyword"},
14827 })]),
14828 TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION
14829 );
14830 assert_eq!(
14831 workflow_completion_protocol_version(&[
14832 json!({"type": "upsert_memo", "entries": {"status": "waiting"}}),
14833 json!({"type": "open_condition_wait", "condition_key": "ready"}),
14834 ]),
14835 MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION
14836 );
14837 assert_eq!(
14838 workflow_completion_protocol_version(&[
14839 json!({"type": "upsert_search_attributes", "attributes": {"State": "waiting"}}),
14840 json!({"type": "open_condition_wait", "condition_key": "ready"}),
14841 ]),
14842 CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION
14843 );
14844 assert_eq!(
14845 workflow_completion_protocol_version(&[json!({
14846 "type": "open_condition_wait",
14847 "condition_wait_occurrence_id": "rust:condition-wait:0",
14848 "condition_key": "ready",
14849 })]),
14850 CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
14851 );
14852 assert_eq!(
14853 workflow_completion_protocol_version_with_message_streams(
14854 &[json!({"type": "upsert_memo", "entries": {"status": "waiting"}})],
14855 true,
14856 ),
14857 MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION
14858 );
14859 assert_eq!(
14860 workflow_completion_protocol_version_with_message_streams(
14861 &[json!({
14862 "type": "open_condition_wait",
14863 "condition_wait_occurrence_id": "rust:condition-wait:0",
14864 "condition_key": "ready",
14865 })],
14866 true,
14867 ),
14868 CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
14869 );
14870 }
14871
14872 #[test]
14873 fn portable_worker_affinity_manifest_explicitly_refuses_unimplemented_features() {
14874 let manifest = portable_worker_affinity_capability_manifest();
14875
14876 for capability in ["local_activities", "worker_sessions", "sticky_execution"] {
14877 assert_eq!(manifest[capability]["supported"], json!(false));
14878 assert_eq!(
14879 manifest[capability]["minimum_protocol_version"],
14880 json!(PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION)
14881 );
14882 assert!(manifest[capability]["reason"]
14883 .as_str()
14884 .is_some_and(|reason| !reason.is_empty()));
14885 }
14886 }
14887
14888 fn typed_fidelity_probe() -> AvroValue {
14889 AvroValue::Map(BTreeMap::from([
14890 ("bytes".to_string(), AvroValue::Bytes(vec![0, 0xff])),
14891 ("empty".to_string(), AvroValue::Map(BTreeMap::new())),
14892 (
14893 "numeric".to_string(),
14894 AvroValue::Map(BTreeMap::from([
14895 ("0".to_string(), AvroValue::String("zero".to_string())),
14896 ("1".to_string(), AvroValue::String("one".to_string())),
14897 ])),
14898 ),
14899 (
14900 "nested".to_string(),
14901 AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([(
14902 "enabled".to_string(),
14903 AvroValue::Boolean(true),
14904 )]))]),
14905 ),
14906 (
14907 "projection_collisions".to_string(),
14908 AvroValue::Array(projection_collision_probe()),
14909 ),
14910 ]))
14911 }
14912
14913 fn projection_collision_probe() -> Vec<AvroValue> {
14914 vec![
14915 AvroValue::Map(BTreeMap::from([
14916 ("$type".to_string(), AvroValue::String("bytes".to_string())),
14917 (
14918 "base64".to_string(),
14919 AvroValue::String("ordinary user text".to_string()),
14920 ),
14921 ])),
14922 AvroValue::Map(BTreeMap::from([
14923 ("$type".to_string(), AvroValue::String("map".to_string())),
14924 (
14925 "entries".to_string(),
14926 AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([
14927 ("key".to_string(), AvroValue::String("ordinary".to_string())),
14928 (
14929 "value".to_string(),
14930 AvroValue::String("user map".to_string()),
14931 ),
14932 ]))]),
14933 ),
14934 ])),
14935 ]
14936 }
14937
14938 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
14939 struct TypedContract {
14940 nested: TypedNested,
14941 mode: TypedMode,
14942 optional: Option<String>,
14943 absent: Option<String>,
14944 items: Vec<i64>,
14945 labels: BTreeMap<String, String>,
14946 bytes: serde_bytes::ByteBuf,
14947 signed: i64,
14948 finite: f64,
14949 }
14950
14951 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
14952 struct TypedNested {
14953 enabled: bool,
14954 }
14955
14956 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
14957 enum TypedMode {
14958 Detailed { label: String },
14959 }
14960
14961 fn typed_contract() -> TypedContract {
14962 TypedContract {
14963 nested: TypedNested { enabled: true },
14964 mode: TypedMode::Detailed {
14965 label: "compiler-checked".to_string(),
14966 },
14967 optional: Some("present".to_string()),
14968 absent: None,
14969 items: vec![i64::MIN, 0, i64::MAX],
14970 labels: BTreeMap::from([
14971 ("language".to_string(), "rust".to_string()),
14972 ("wire".to_string(), "avro".to_string()),
14973 ]),
14974 bytes: serde_bytes::ByteBuf::from(vec![0, 0xff, 7]),
14975 signed: -9_223_372_036_854_775_000,
14976 finite: 12.5,
14977 }
14978 }
14979
14980 #[derive(Clone, Debug, Default, PartialEq)]
14981 struct ReplayCounterState {
14982 loaded: Option<String>,
14983 count: i64,
14984 finished: bool,
14985 }
14986
14987 fn replay_counter_worker() -> Worker {
14988 let client = Client::new("http://127.0.0.1:8080").expect("client");
14989 let mut worker = Worker::new(client, "rust-workers");
14990 worker.register_replayed_workflow(
14991 "replay-counter",
14992 ReplayCounterState::default,
14993 |ctx, _input, state| async move {
14994 let loaded = ctx.activity("load-counter", json!([])).await?;
14995 state.update(|current| {
14996 current.loaded = loaded.as_str().map(str::to_string);
14997 })?;
14998 for _ in 0..2 {
14999 let signal = ctx.wait_signal("increment").await?;
15000 let amount = signal.first().and_then(Value::as_i64).unwrap_or_default();
15001 state.update(|current| current.count += amount)?;
15002 }
15003 state.update(|current| current.finished = true)?;
15004 state.read(|current| Ok(json!(current.count)))?
15005 },
15006 );
15007 worker.register_replayed_query::<ReplayCounterState, _, _>(
15008 "replay-counter",
15009 "current",
15010 |_ctx, state, _args| async move {
15011 Ok(json!({
15012 "loaded": state.loaded,
15013 "count": state.count,
15014 "finished": state.finished,
15015 }))
15016 },
15017 );
15018 worker.register_replayed_query::<ReplayCounterState, _, _>(
15019 "replay-counter",
15020 "detached-mutation",
15021 |_ctx, state, _args| async move {
15022 let mut detached = (*state).clone();
15023 detached.count = 999;
15024 Ok(json!(detached.count))
15025 },
15026 );
15027 worker.register_replayed_query::<ReplayCounterState, _, _>(
15028 "replay-counter",
15029 "failed-mutation",
15030 |_ctx, state, _args| async move {
15031 let mut detached = (*state).clone();
15032 detached.count = 999;
15033 Err(Error::WorkerLoop("query refused".to_string()))
15034 },
15035 );
15036 worker
15037 }
15038
15039 fn replay_counter_query(
15040 query_name: &str,
15041 history_events: Value,
15042 run_status: &str,
15043 ) -> QueryTask {
15044 let arguments = fixture_envelope(json!([]));
15045 serde_json::from_value(json!({
15046 "query_task_id": format!("query-{query_name}"),
15047 "workflow_type": "replay-counter",
15048 "query_name": query_name,
15049 "payload_codec": DEFAULT_CODEC,
15050 "workflow_arguments": arguments.clone(),
15051 "query_arguments": arguments,
15052 "history_events": history_events,
15053 "run_status": run_status,
15054 }))
15055 .expect("query task")
15056 }
15057
15058 fn workflow_context(history: Vec<HistoryEvent>) -> WorkflowContext {
15059 workflow_context_with_codec(history, DEFAULT_CODEC)
15060 }
15061
15062 fn workflow_context_with_codec(
15063 history: Vec<HistoryEvent>,
15064 payload_codec: &str,
15065 ) -> WorkflowContext {
15066 WorkflowContext {
15067 state: Arc::new(Mutex::new(
15068 WorkflowState::new_with_identity(
15069 history,
15070 None,
15071 None,
15072 "rust-workers".to_string(),
15073 payload_codec.to_string(),
15074 None,
15075 )
15076 .expect("valid workflow history"),
15077 )),
15078 }
15079 }
15080
15081 fn history_event(event_type: &str, payload: Value) -> HistoryEvent {
15082 HistoryEvent {
15083 event_type: event_type.to_string(),
15084 payload,
15085 raw: HashMap::new(),
15086 }
15087 }
15088
15089 fn parallel_path_entry(
15090 kind: &str,
15091 base: u64,
15092 size: usize,
15093 index: usize,
15094 ) -> ParallelGroupMetadata {
15095 parallel_group_entry(base, size, index, kind)
15096 }
15097
15098 fn parallel_history_event(
15099 event_type: &str,
15100 sequence: u64,
15101 identity_field: &str,
15102 identity: &str,
15103 path: Vec<ParallelGroupMetadata>,
15104 result: Option<Value>,
15105 ) -> HistoryEvent {
15106 let mut payload = serde_json::Map::from_iter([
15107 ("sequence".to_string(), json!(sequence)),
15108 (identity_field.to_string(), json!(identity)),
15109 ]);
15110 let inner = path.last().expect("parallel history path");
15111 apply_parallel_group_path(&mut payload, std::slice::from_ref(inner));
15112 payload.insert("parallel_group_path".to_string(), json!(path));
15113 if let Some(result) = result {
15114 let field = if event_type == "ChildRunCompleted" {
15115 "result"
15116 } else {
15117 "result"
15118 };
15119 payload.insert(field.to_string(), fixture_envelope(result));
15120 payload.insert("payload_codec".to_string(), json!(DEFAULT_CODEC));
15121 }
15122 history_event(event_type, Value::Object(payload))
15123 }
15124
15125 fn nested_parallel_operations() -> Vec<ParallelOperation> {
15126 vec![
15127 ParallelOperation::activity("first", json!([])),
15128 ParallelOperation::group(vec![
15129 ParallelOperation::child_workflow(
15130 "second",
15131 ChildWorkflowOptions::new("child-workers"),
15132 json!([]),
15133 ),
15134 ParallelOperation::activity("third", json!([])),
15135 ]),
15136 ]
15137 }
15138
15139 fn nested_parallel_paths() -> [Vec<ParallelGroupMetadata>; 3] {
15140 let outer = [
15141 parallel_path_entry("mixed", 1, 3, 0),
15142 parallel_path_entry("mixed", 1, 3, 1),
15143 parallel_path_entry("mixed", 1, 3, 2),
15144 ];
15145 [
15146 vec![outer[0].clone()],
15147 vec![outer[1].clone(), parallel_path_entry("mixed", 2, 2, 0)],
15148 vec![outer[2].clone(), parallel_path_entry("mixed", 2, 2, 1)],
15149 ]
15150 }
15151
15152 #[test]
15153 fn parallel_schedules_every_nested_mixed_leaf_with_stable_metadata() {
15154 let ctx = workflow_context(Vec::new());
15155 let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15156 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15157
15158 assert!(matches!(
15159 call.as_mut().poll(&mut task_context),
15160 Poll::Pending
15161 ));
15162 let commands = ctx.take_commands().expect("parallel commands");
15163 assert_eq!(
15164 commands
15165 .iter()
15166 .map(|command| command["type"].as_str().unwrap_or_default())
15167 .collect::<Vec<_>>(),
15168 [
15169 "schedule_activity",
15170 "start_child_workflow",
15171 "schedule_activity"
15172 ]
15173 );
15174 let paths = nested_parallel_paths();
15175 for (command, path) in commands.iter().zip(paths) {
15176 assert_eq!(command["parallel_group_path"], json!(path));
15177 assert_eq!(
15178 command["parallel_group_id"],
15179 json!(path.last().expect("inner group").parallel_group_id)
15180 );
15181 }
15182 }
15183
15184 fn completed_nested_parallel_history() -> Vec<HistoryEvent> {
15185 let paths = nested_parallel_paths();
15186 let third = parallel_history_event(
15187 "ActivityCompleted",
15188 3,
15189 "activity_type",
15190 "third",
15191 paths[2].clone(),
15192 Some(json!("three")),
15193 );
15194 vec![
15195 parallel_history_event(
15196 "ActivityCompleted",
15197 1,
15198 "activity_type",
15199 "first",
15200 paths[0].clone(),
15201 Some(json!("one")),
15202 ),
15203 parallel_history_event(
15204 "ChildWorkflowScheduled",
15205 2,
15206 "child_workflow_type",
15207 "second",
15208 paths[1].clone(),
15209 None,
15210 ),
15211 parallel_history_event(
15212 "ChildRunCompleted",
15213 2,
15214 "child_workflow_type",
15215 "second",
15216 paths[1].clone(),
15217 Some(json!("two")),
15218 ),
15219 third.clone(),
15220 third,
15221 ]
15222 }
15223
15224 #[test]
15225 fn parallel_replay_rebuilds_input_order_and_tolerates_duplicate_delivery() {
15226 for _restart_or_completed_replay in 0..2 {
15227 let ctx = workflow_context(completed_nested_parallel_history());
15228 let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15229 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15230 let Poll::Ready(Ok(results)) = call.as_mut().poll(&mut task_context) else {
15231 panic!("completed nested parallel history must replay");
15232 };
15233 assert_eq!(
15234 results,
15235 vec![
15236 ParallelResult::Activity(json!("one")),
15237 ParallelResult::Group(vec![
15238 ParallelResult::ChildWorkflow(ChildWorkflowResult {
15239 parent: WorkflowIdentity {
15240 workflow_id: None,
15241 run_id: None,
15242 },
15243 child: WorkflowIdentity {
15244 workflow_id: None,
15245 run_id: None,
15246 },
15247 child_workflow_type: Some("second".to_string()),
15248 result: json!("two"),
15249 }),
15250 ParallelResult::Activity(json!("three")),
15251 ]),
15252 ]
15253 );
15254 assert!(ctx.take_commands().expect("commands").is_empty());
15255 ctx.ensure_history_consumed().expect("history consumed");
15256 }
15257 }
15258
15259 #[test]
15260 fn parallel_failure_keeps_typed_cause_path_and_late_completions() {
15261 let paths = nested_parallel_paths();
15262 let history = vec![
15263 parallel_history_event(
15264 "ActivityCompleted",
15265 1,
15266 "activity_type",
15267 "first",
15268 paths[0].clone(),
15269 Some(json!("one")),
15270 ),
15271 parallel_history_event(
15272 "ChildWorkflowScheduled",
15273 2,
15274 "child_workflow_type",
15275 "second",
15276 paths[1].clone(),
15277 None,
15278 ),
15279 parallel_history_event(
15280 "ChildRunFailed",
15281 2,
15282 "child_workflow_type",
15283 "second",
15284 paths[1].clone(),
15285 None,
15286 ),
15287 parallel_history_event(
15288 "ActivityCompleted",
15289 3,
15290 "activity_type",
15291 "third",
15292 paths[2].clone(),
15293 Some(json!("late")),
15294 ),
15295 ];
15296 let ctx = workflow_context(history);
15297 let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15298 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15299 let outcome = call.as_mut().poll(&mut task_context);
15300 let Poll::Ready(Err(Error::ParallelFailed(failure))) = outcome else {
15301 panic!("one failed child must return a typed partial failure: {outcome:?}");
15302 };
15303 assert_eq!(failure.member_path, [1, 0]);
15304 assert_eq!(failure.group_id, "parallel-calls:1:3");
15305 assert!(matches!(*failure.cause, Error::ChildWorkflowFailed(_)));
15306 assert_eq!(
15307 failure
15308 .completed
15309 .iter()
15310 .map(|completion| completion.member_path.clone())
15311 .collect::<Vec<_>>(),
15312 [vec![0], vec![1, 1]]
15313 );
15314 }
15315
15316 #[test]
15317 fn pending_parallel_history_restarts_without_rescheduling_any_leaf() {
15318 let paths = nested_parallel_paths();
15319 let history = vec![
15320 parallel_history_event(
15321 "ActivityScheduled",
15322 1,
15323 "activity_type",
15324 "first",
15325 paths[0].clone(),
15326 None,
15327 ),
15328 parallel_history_event(
15329 "ChildWorkflowScheduled",
15330 2,
15331 "child_workflow_type",
15332 "second",
15333 paths[1].clone(),
15334 None,
15335 ),
15336 parallel_history_event(
15337 "ActivityScheduled",
15338 3,
15339 "activity_type",
15340 "third",
15341 paths[2].clone(),
15342 None,
15343 ),
15344 ];
15345 for _restart in 0..2 {
15346 let ctx = workflow_context(history.clone());
15347 let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15348 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15349 let outcome = call.as_mut().poll(&mut task_context);
15350 assert!(matches!(outcome, Poll::Pending), "{outcome:?}");
15351 assert!(ctx.take_commands().expect("commands").is_empty());
15352 }
15353 }
15354
15355 fn selection_path(index: usize, key: &str) -> Vec<ParallelGroupMetadata> {
15356 vec![selection_group_entry(
15357 1,
15358 2,
15359 index,
15360 "activity",
15361 &SelectionMemberMetadata {
15362 key: SelectionKey::Name(key.to_string()),
15363 index,
15364 base_sequence: index as u64 + 1,
15365 size: 1,
15366 kind: "activity".to_string(),
15367 },
15368 )]
15369 }
15370
15371 fn selection_activity_event(
15372 event_type: &str,
15373 index: usize,
15374 key: &str,
15375 result: Option<Value>,
15376 ) -> HistoryEvent {
15377 let sequence = index as u64 + 1;
15378 let mut event = parallel_history_event(
15379 event_type,
15380 sequence,
15381 "activity_type",
15382 &format!("{key}-activity"),
15383 selection_path(index, key),
15384 result,
15385 );
15386 event.payload["activity_execution_id"] = json!(format!("activity-{key}"));
15387 event.raw.insert(
15388 "id".to_string(),
15389 json!(if event_type == "ActivityCompleted" {
15390 format!("event-{key}")
15391 } else {
15392 format!("{event_type}-{key}")
15393 }),
15394 );
15395 event
15396 }
15397
15398 fn selection_winner_marker() -> HistoryEvent {
15399 history_event(
15400 "SelectionResolved",
15401 json!({
15402 "selection_group_id": "select-calls:1:2",
15403 "selection_group_base_sequence": 1,
15404 "selection_group_size": 2,
15405 "member_key": "fast",
15406 "member_index": 1,
15407 "member_base_sequence": 2,
15408 "member_size": 1,
15409 "operation_kind": "activity",
15410 "operation_identity": "activity-fast",
15411 "outcome": "completed",
15412 "resolution_event_id": "event-fast",
15413 "resolution_event_type": "ActivityCompleted",
15414 }),
15415 )
15416 }
15417
15418 fn keyed_activity_selection(ctx: &WorkflowContext) -> SelectCall {
15419 ctx.select_keyed(vec![
15420 (
15421 "slow",
15422 ParallelOperation::activity_with_options(
15423 "slow-activity",
15424 ActivityOptions::new().task_queue("default"),
15425 json!([]),
15426 ),
15427 ),
15428 (
15429 "fast",
15430 ParallelOperation::activity_with_options(
15431 "fast-activity",
15432 ActivityOptions::new().task_queue("default"),
15433 json!([]),
15434 ),
15435 ),
15436 ])
15437 }
15438
15439 fn assert_persisted_selection_replay(history: Vec<HistoryEvent>) {
15440 let ctx = workflow_context(history);
15441 let mut call = Box::pin(keyed_activity_selection(&ctx));
15442 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15443 let selected = match call.as_mut().poll(&mut task_context) {
15444 Poll::Ready(Ok(selected)) => selected,
15445 Poll::Ready(Err(error)) => panic!("persisted selection winner must replay: {error:?}"),
15446 Poll::Pending => panic!("persisted selection winner must replay without pending"),
15447 };
15448 assert_eq!(selected.key, SelectionKey::Name("fast".to_string()));
15449 assert_eq!(
15450 selected.value,
15451 Some(ParallelResult::Activity(json!("winner-value")))
15452 );
15453 let slow = selected
15454 .handle(&SelectionKey::Name("slow".to_string()))
15455 .expect("slow handle")
15456 .clone();
15457 let mut await_slow = Box::pin(slow.await_result());
15458 assert!(matches!(
15459 await_slow.as_mut().poll(&mut task_context),
15460 Poll::Ready(Ok(ParallelResult::Activity(value))) if value == json!("loser-value")
15461 ));
15462 assert!(ctx.take_commands().expect("commands").is_empty());
15463 }
15464
15465 const SELECTION_COLD_REPLAY_HISTORY: &str = "DURABLE_WORKFLOW_SELECTION_COLD_REPLAY_HISTORY";
15466
15467 fn canonical_selection_history() -> Vec<HistoryEvent> {
15468 const FIXTURE: &[u8] =
15469 include_bytes!("../tests/fixtures/durable_selection_runtime_history.json");
15470 assert_eq!(
15471 format!("{:x}", Sha256::digest(FIXTURE)),
15472 "51fd8b9c16e978dcef536a5c727b9fdc0ae724d9afc17d9a7837d219f41ee3ba",
15473 );
15474 let fixture: Value = serde_json::from_slice(FIXTURE).expect("canonical selection fixture");
15475
15476 serde_json::from_value(fixture["history"].clone()).expect("canonical selection history")
15477 }
15478
15479 #[test]
15480 fn selection_fresh_process_entrypoint() {
15481 let Ok(path) = std::env::var(SELECTION_COLD_REPLAY_HISTORY) else {
15482 return;
15483 };
15484 let persisted = fs::read(path).expect("persisted selection history");
15485 assert_eq!(
15486 format!("{:x}", Sha256::digest(&persisted)),
15487 "51fd8b9c16e978dcef536a5c727b9fdc0ae724d9afc17d9a7837d219f41ee3ba",
15488 );
15489 let fixture: Value =
15490 serde_json::from_slice(&persisted).expect("valid persisted selection fixture");
15491 let history: Vec<HistoryEvent> = serde_json::from_value(fixture["history"].clone())
15492 .expect("valid persisted selection history");
15493
15494 assert_persisted_selection_replay(history);
15495 }
15496
15497 #[test]
15498 fn selection_starts_every_member_with_stable_keys_and_group_identity() {
15499 let ctx = workflow_context(Vec::new());
15500 let mut call = Box::pin(keyed_activity_selection(&ctx));
15501 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15502
15503 assert!(matches!(
15504 call.as_mut().poll(&mut task_context),
15505 Poll::Pending
15506 ));
15507 let commands = ctx.take_commands().expect("selection commands");
15508 assert_eq!(commands.len(), 2);
15509 assert_eq!(commands[0]["selection_member_key"], json!("slow"));
15510 assert_eq!(commands[1]["selection_member_key"], json!("fast"));
15511 assert!(commands.iter().all(|command| {
15512 command["parallel_group_id"] == json!("select-calls:1:2")
15513 && command["parallel_group_mode"] == json!("select")
15514 }));
15515 }
15516
15517 #[test]
15518 fn selection_key_domain_rejects_empty_authoring_and_malformed_history() {
15519 let ctx = workflow_context(Vec::new());
15520 let mut invalid = Box::pin(ctx.select_keyed(vec![(
15521 "",
15522 ParallelOperation::activity("invalid", json!([])),
15523 )]));
15524 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15525 assert!(matches!(
15526 invalid.as_mut().poll(&mut task_context),
15527 Poll::Ready(Err(Error::InvalidParallelGroup(ParallelGroupError {
15528 reason: "selection_key_invalid",
15529 ..
15530 })))
15531 ));
15532
15533 for invalid_key in [json!(""), json!(-1)] {
15534 let mut event = selection_activity_event("ActivityScheduled", 0, "slow", None);
15535 event.payload["selection_member_key"] = invalid_key.clone();
15536 event.payload["parallel_group_path"][0]["selection_member_key"] = invalid_key;
15537 assert!(matches!(
15538 WorkflowState::new_with_identity(
15539 vec![event],
15540 None,
15541 None,
15542 "rust-workers".to_string(),
15543 DEFAULT_CODEC.to_string(),
15544 None,
15545 ),
15546 Err(Error::NonDeterministicReplay(_))
15547 ));
15548 }
15549 }
15550
15551 #[test]
15552 fn selection_preserves_valid_named_and_numeric_keys() {
15553 let ctx = workflow_context(Vec::new());
15554 let mut selection = Box::pin(ctx.select_keyed(vec![
15555 (
15556 SelectionKey::Index(0),
15557 ParallelOperation::activity("numeric", json!([])),
15558 ),
15559 (
15560 SelectionKey::Name("named".to_string()),
15561 ParallelOperation::timer(Duration::from_secs(1)),
15562 ),
15563 ]));
15564 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15565
15566 assert!(matches!(
15567 selection.as_mut().poll(&mut task_context),
15568 Poll::Pending
15569 ));
15570 let commands = ctx.take_commands().expect("selection commands");
15571 assert_eq!(commands[0]["selection_member_key"], json!(0));
15572 assert_eq!(commands[1]["selection_member_key"], json!("named"));
15573 }
15574
15575 #[test]
15576 fn selection_replays_persisted_winner_and_loser_can_be_awaited_later() {
15577 let history = canonical_selection_history();
15578 assert_persisted_selection_replay(history.clone());
15579
15580 let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
15581 .join("tests/fixtures/durable_selection_runtime_history.json");
15582 let output =
15583 ProcessCommand::new(std::env::current_exe().expect("current Rust test binary"))
15584 .args([
15585 "--exact",
15586 "tests::selection_fresh_process_entrypoint",
15587 "--nocapture",
15588 ])
15589 .env(SELECTION_COLD_REPLAY_HISTORY, &path)
15590 .output()
15591 .expect("run fresh selection replay process");
15592
15593 assert!(
15594 output.status.success(),
15595 "fresh selection replay failed:\nstdout:\n{}\nstderr:\n{}",
15596 String::from_utf8_lossy(&output.stdout),
15597 String::from_utf8_lossy(&output.stderr),
15598 );
15599 }
15600
15601 #[test]
15602 fn selection_waits_durably_when_terminal_members_precede_the_winner_marker() {
15603 let mut history = canonical_selection_history();
15604 history.retain(|event| event.event_type != "SelectionResolved");
15605 let ctx = workflow_context(history);
15606 let mut selection = Box::pin(keyed_activity_selection(&ctx));
15607 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15608
15609 assert!(matches!(
15610 selection.as_mut().poll(&mut task_context),
15611 Poll::Pending
15612 ));
15613 assert!(ctx.take_commands().expect("commands").is_empty());
15614 assert!(
15615 ctx.matched_recorded_pending()
15616 .expect("selection pending state"),
15617 "terminal member history must keep the workflow durably pending until SelectionResolved commits"
15618 );
15619 }
15620
15621 #[test]
15622 fn selection_terminal_condition_history_waits_durably_for_its_winner_marker() {
15623 for (terminal_event, predicate_satisfied, timeout_seconds) in [
15624 ("ConditionWaitSatisfied", true, None),
15625 ("ConditionWaitTimedOut", false, Some(0)),
15626 ] {
15627 let member = SelectionMemberMetadata {
15628 key: SelectionKey::Name("condition".to_string()),
15629 index: 0,
15630 base_sequence: 1,
15631 size: 1,
15632 kind: "condition".to_string(),
15633 };
15634 let path = vec![selection_group_entry(1, 1, 0, "condition", &member)];
15635 let mut payload = json!({
15636 "sequence": 1,
15637 "condition_wait_id": "condition-1",
15638 "condition_wait_occurrence_id": "rust:condition-wait:0",
15639 "condition_key": "ready",
15640 "condition_definition_fingerprint": "sha256:ready-v1",
15641 "parallel_group_path": path,
15642 });
15643 payload
15644 .as_object_mut()
15645 .expect("condition history payload")
15646 .extend(
15647 serde_json::to_value(&path[0])
15648 .expect("condition selection metadata")
15649 .as_object()
15650 .expect("condition selection metadata object")
15651 .clone(),
15652 );
15653 if let Some(timeout_seconds) = timeout_seconds {
15654 payload["timeout_seconds"] = json!(timeout_seconds);
15655 }
15656 let history = vec![
15657 history_event("ConditionWaitOpened", payload.clone()),
15658 history_event(terminal_event, payload),
15659 ];
15660 let ctx = workflow_context(history);
15661 let mut options = ConditionWaitOptions::new("ready", "sha256:ready-v1");
15662 if timeout_seconds.is_some() {
15663 options = options.timeout(Duration::ZERO);
15664 }
15665 let mut selection = Box::pin(ctx.select_keyed(vec![(
15666 "condition",
15667 ParallelOperation::condition(options, move || Ok(predicate_satisfied)),
15668 )]));
15669 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15670
15671 assert!(matches!(
15672 selection.as_mut().poll(&mut task_context),
15673 Poll::Pending
15674 ));
15675 assert!(ctx.take_commands().expect("commands").is_empty());
15676 assert!(
15677 ctx.matched_recorded_pending()
15678 .expect("condition selection pending state"),
15679 "{terminal_event} must keep the workflow durably pending until SelectionResolved commits"
15680 );
15681 }
15682 }
15683
15684 #[test]
15685 fn selection_immediate_condition_members_open_a_durable_wait() {
15686 for predicate_satisfied in [true, false] {
15687 let ctx = workflow_context(Vec::new());
15688 let mut selection = Box::pin(ctx.select_keyed(vec![(
15689 "condition",
15690 ParallelOperation::condition(
15691 ConditionWaitOptions::new("ready", "sha256:ready-v1").timeout(Duration::ZERO),
15692 move || Ok(predicate_satisfied),
15693 ),
15694 )]));
15695 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15696
15697 assert!(matches!(
15698 selection.as_mut().poll(&mut task_context),
15699 Poll::Pending
15700 ));
15701 let commands = ctx.take_commands().expect("condition selection command");
15702 assert_eq!(commands.len(), 1);
15703 assert_eq!(commands[0]["type"], json!("open_condition_wait"));
15704 assert_eq!(commands[0]["timeout_seconds"], json!(0));
15705 assert_eq!(
15706 commands[0]["parallel_group_path"][0]["parallel_group_mode"],
15707 json!("select")
15708 );
15709 }
15710 }
15711
15712 #[test]
15713 fn selection_loser_cancellation_is_explicit_and_idempotent() {
15714 let history = vec![
15715 selection_activity_event("ActivityScheduled", 0, "slow", None),
15716 selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
15717 selection_winner_marker(),
15718 ];
15719 let ctx = workflow_context(history.clone());
15720 let mut call = Box::pin(keyed_activity_selection(&ctx));
15721 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15722 let Poll::Ready(Ok(selected)) = call.as_mut().poll(&mut task_context) else {
15723 panic!("winner must replay");
15724 };
15725 let slow = selected
15726 .handle(&SelectionKey::Name("slow".to_string()))
15727 .expect("slow handle")
15728 .clone();
15729 let mut cancel = Box::pin(slow.cancel());
15730 assert!(matches!(
15731 cancel.as_mut().poll(&mut task_context),
15732 Poll::Pending
15733 ));
15734 assert!(matches!(
15735 cancel.as_mut().poll(&mut task_context),
15736 Poll::Pending
15737 ));
15738 let commands = ctx.take_commands().expect("cancel command");
15739 assert_eq!(commands.len(), 1);
15740 assert_eq!(commands[0]["type"], json!("cancel_selection_operation"));
15741 assert_eq!(commands[0]["member_key"], json!("slow"));
15742
15743 let mut cancelled_history = history;
15744 cancelled_history.push(history_event(
15745 "SelectionOperationCancelled",
15746 json!({
15747 "selection_group_id": "select-calls:1:2",
15748 "member_key": "slow",
15749 "member_index": 0,
15750 "member_base_sequence": 1,
15751 "member_size": 1,
15752 "operation_kind": "activity",
15753 "operation_identity": "activity-slow",
15754 "cancelled_at": "2026-08-27T00:00:00Z",
15755 }),
15756 ));
15757 let replayed = workflow_context(cancelled_history);
15758 let mut call = Box::pin(keyed_activity_selection(&replayed));
15759 let Poll::Ready(Ok(selected)) = call.as_mut().poll(&mut task_context) else {
15760 panic!("winner must replay after cancellation");
15761 };
15762 let slow = selected
15763 .handle(&SelectionKey::Name("slow".to_string()))
15764 .expect("slow handle")
15765 .clone();
15766 let mut cancel = Box::pin(slow.cancel());
15767 assert!(matches!(
15768 cancel.as_mut().poll(&mut task_context),
15769 Poll::Ready(Ok(()))
15770 ));
15771 assert!(replayed.take_commands().expect("commands").is_empty());
15772 }
15773
15774 #[test]
15775 fn selection_cancellation_marker_is_bound_to_every_authored_handle_field() {
15776 let base_history = vec![
15777 selection_activity_event("ActivityScheduled", 0, "slow", None),
15778 selection_activity_event("ActivityScheduled", 1, "fast", None),
15779 selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
15780 selection_winner_marker(),
15781 ];
15782 for (field, corrupt) in [
15783 ("member_key", json!("fast")),
15784 ("member_index", json!(1)),
15785 ("member_base_sequence", json!(3)),
15786 ("member_size", json!(2)),
15787 ("operation_kind", json!("timer")),
15788 ("operation_identity", json!("forged")),
15789 ] {
15790 let mut cancellation = json!({
15791 "selection_group_id": "select-calls:1:2",
15792 "member_key": "slow",
15793 "member_index": 0,
15794 "member_base_sequence": 1,
15795 "member_size": 1,
15796 "operation_kind": "activity",
15797 "operation_identity": "activity-slow",
15798 });
15799 cancellation[field] = corrupt;
15800 let mut history = base_history.clone();
15801 history.push(history_event("SelectionOperationCancelled", cancellation));
15802 let ctx = workflow_context(history);
15803 let mut selection = Box::pin(keyed_activity_selection(&ctx));
15804 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15805
15806 assert!(matches!(
15807 selection.as_mut().poll(&mut task_context),
15808 Poll::Ready(Err(Error::NonDeterministicReplay(_)))
15809 ));
15810 }
15811 }
15812
15813 #[test]
15814 fn selection_child_identity_prefers_the_durable_run_id() {
15815 let ctx = workflow_context(vec![history_event(
15816 "ChildWorkflowScheduled",
15817 json!({
15818 "sequence": 1,
15819 "child_workflow_type": "child",
15820 "child_workflow_instance_id": "child-instance",
15821 "child_workflow_run_id": "child-run",
15822 }),
15823 )]);
15824 let state = ctx.state.lock().expect("workflow state");
15825
15826 assert_eq!(
15827 selection_operation_identity(&state, "child", 1, 1),
15828 "child-run"
15829 );
15830 }
15831
15832 #[test]
15833 fn selection_activity_identity_requires_canonical_execution_id() {
15834 let slow = selection_activity_event("ActivityScheduled", 0, "slow", None);
15835 let mut fast_open = selection_activity_event("ActivityScheduled", 1, "fast", None);
15836 let mut fast_completed =
15837 selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner")));
15838 for event in [&mut fast_open, &mut fast_completed] {
15839 event
15840 .payload
15841 .as_object_mut()
15842 .expect("activity payload")
15843 .remove("activity_execution_id");
15844 event.payload["activity_id"] = json!("forged-activity-id");
15845 }
15846 let mut marker = selection_winner_marker();
15847 marker.payload["operation_identity"] = json!("forged-activity-id");
15848 let ctx = workflow_context(vec![slow, fast_open, fast_completed, marker]);
15849 let mut selection = Box::pin(keyed_activity_selection(&ctx));
15850 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15851
15852 assert!(matches!(
15853 selection.as_mut().poll(&mut task_context),
15854 Poll::Ready(Err(Error::NonDeterministicReplay(_)))
15855 ));
15856 }
15857
15858 #[test]
15859 fn selection_completion_before_cancellation_remains_awaitable() {
15860 let history = vec![
15861 selection_activity_event("ActivityScheduled", 0, "slow", None),
15862 selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
15863 selection_winner_marker(),
15864 selection_activity_event(
15865 "ActivityCompleted",
15866 0,
15867 "slow",
15868 Some(json!("completed-first")),
15869 ),
15870 ];
15871 let ctx = workflow_context(history);
15872 let mut selection = Box::pin(keyed_activity_selection(&ctx));
15873 let mut task_context = TaskContext::from_waker(noop_waker_ref());
15874 let Poll::Ready(Ok(selected)) = selection.as_mut().poll(&mut task_context) else {
15875 panic!("winner must replay");
15876 };
15877 let slow = selected
15878 .handle(&SelectionKey::Name("slow".to_string()))
15879 .expect("slow handle")
15880 .clone();
15881 let mut cancel = Box::pin(slow.cancel());
15882 assert!(matches!(
15883 cancel.as_mut().poll(&mut task_context),
15884 Poll::Ready(Ok(()))
15885 ));
15886 let mut await_slow = Box::pin(slow.await_result());
15887 assert!(matches!(
15888 await_slow.as_mut().poll(&mut task_context),
15889 Poll::Ready(Ok(ParallelResult::Activity(value))) if value == json!("completed-first")
15890 ));
15891 let commands = ctx.take_commands().expect("commands");
15892 assert!(commands.is_empty());
15893 }
15894
15895 #[test]
15896 fn selection_nested_later_failure_before_cancel_remains_the_awaited_failure() {
15897 let nested_member = SelectionMemberMetadata {
15898 key: SelectionKey::Name("nested".to_string()),
15899 index: 0,
15900 base_sequence: 1,
15901 size: 2,
15902 kind: "group".to_string(),
15903 };
15904 let deadline_member = SelectionMemberMetadata {
15905 key: SelectionKey::Name("deadline".to_string()),
15906 index: 1,
15907 base_sequence: 3,
15908 size: 1,
15909 kind: "timer".to_string(),
15910 };
15911 let nested_paths = [
15912 vec![
15913 selection_group_entry(1, 3, 0, "mixed", &nested_member),
15914 parallel_group_entry(1, 2, 0, "activity"),
15915 ],
15916 vec![
15917 selection_group_entry(1, 3, 1, "mixed", &nested_member),
15918 parallel_group_entry(1, 2, 1, "activity"),
15919 ],
15920 ];
15921 let deadline_path = vec![selection_group_entry(1, 3, 2, "mixed", &deadline_member)];
15922 let mut timer_fired = parallel_history_event(
15923 "TimerFired",
15924 3,
15925 "timer_id",
15926 "timer-3",
15927 deadline_path.clone(),
15928 None,
15929 );
15930 timer_fired.payload["delay_seconds"] = json!(0);
15931 timer_fired
15932 .raw
15933 .insert("id".to_string(), json!("timer-fired"));
15934 let mut timer_scheduled = parallel_history_event(
15935 "TimerScheduled",
15936 3,
15937 "timer_id",
15938 "timer-3",
15939 deadline_path,
15940 None,
15941 );
15942 timer_scheduled.payload["delay_seconds"] = json!(0);
15943 let history = vec![
15944 parallel_history_event(
15945 "ActivityScheduled",
15946 1,
15947 "activity_type",
15948 "nested-first",
15949 nested_paths[0].clone(),
15950 None,
15951 ),
15952 parallel_history_event(
15953 "ActivityScheduled",
15954 2,
15955 "activity_type",
15956 "nested-second",
15957 nested_paths[1].clone(),
15958 None,
15959 ),
15960 timer_scheduled,
15961 timer_fired,
15962 history_event(
15963 "SelectionResolved",
15964 json!({
15965 "selection_group_id": "select-calls:1:3",
15966 "selection_group_base_sequence": 1,
15967 "selection_group_size": 3,
15968 "member_key": "deadline",
15969 "member_index": 1,
15970 "member_base_sequence": 3,
15971 "member_size": 1,
15972 "operation_kind": "timer",
15973 "operation_identity": "timer-3",
15974 "outcome": "completed",
15975 "resolution_event_id": "timer-fired",
15976 "resolution_event_type": "TimerFired",
15977 }),
15978 ),
15979 parallel_history_event(
15980 "ActivityFailed",
15981 2,
15982 "activity_type",
15983 "nested-second",
15984 nested_paths[1].clone(),
15985 None,
15986 ),
15987 ];
15988 let ctx = workflow_context(history);
15989 let mut selection = Box::pin(ctx.select_keyed(vec![
15990 (
15991 "nested",
15992 ParallelOperation::group(vec![
15993 ParallelOperation::activity("nested-first", json!([])),
15994 ParallelOperation::activity("nested-second", json!([])),
15995 ]),
15996 ),
15997 ("deadline", ParallelOperation::timer(Duration::ZERO)),
15998 ]));
15999 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16000 let Poll::Ready(Ok(selected)) = selection.as_mut().poll(&mut task_context) else {
16001 panic!("deadline winner must replay");
16002 };
16003 let nested = selected
16004 .handle(&SelectionKey::Name("nested".to_string()))
16005 .expect("nested handle")
16006 .clone();
16007 let mut cancel = Box::pin(nested.cancel());
16008 assert!(matches!(
16009 cancel.as_mut().poll(&mut task_context),
16010 Poll::Ready(Ok(()))
16011 ));
16012 let mut await_nested = Box::pin(nested.await_result());
16013
16014 assert!(matches!(
16015 await_nested.as_mut().poll(&mut task_context),
16016 Poll::Ready(Err(Error::ActivityFailed(_)))
16017 ));
16018 assert!(ctx.take_commands().expect("commands").is_empty());
16019 }
16020
16021 #[test]
16022 fn selection_supports_child_timer_signal_condition_and_nested_groups() {
16023 let ctx = workflow_context(Vec::new());
16024 let mut call = Box::pin(ctx.select(vec![
16025 ParallelOperation::child_workflow(
16026 "child",
16027 ChildWorkflowOptions::new("children"),
16028 json!([]),
16029 ),
16030 ParallelOperation::timer(Duration::from_secs(30)),
16031 ParallelOperation::signal("approval"),
16032 ParallelOperation::condition(
16033 ConditionWaitOptions::new("ready", "sha256:ready"),
16034 || Ok(false),
16035 ),
16036 ParallelOperation::group(vec![
16037 ParallelOperation::activity("nested-one", json!([])),
16038 ParallelOperation::activity("nested-two", json!([])),
16039 ]),
16040 ]));
16041 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16042 assert!(matches!(
16043 call.as_mut().poll(&mut task_context),
16044 Poll::Pending
16045 ));
16046 let commands = ctx.take_commands().expect("selection commands");
16047 assert_eq!(
16048 commands
16049 .iter()
16050 .map(|command| command["type"].as_str().unwrap_or_default())
16051 .collect::<Vec<_>>(),
16052 [
16053 "start_child_workflow",
16054 "start_timer",
16055 "open_signal_wait",
16056 "open_condition_wait",
16057 "schedule_activity",
16058 "schedule_activity",
16059 ]
16060 );
16061 assert!(commands.iter().all(|command| {
16062 command["parallel_group_path"][0]["parallel_group_mode"] == json!("select")
16063 }));
16064 assert_eq!(
16065 commands[4]["parallel_group_path"].as_array().map(Vec::len),
16066 Some(2)
16067 );
16068 assert_eq!(
16069 commands[4]["parallel_group_path"][0]["selection_member_kind"],
16070 json!("group")
16071 );
16072 assert_eq!(
16073 commands[5]["parallel_group_path"][0]["selection_member_kind"],
16074 json!("group")
16075 );
16076
16077 let one_leaf_ctx = workflow_context(Vec::new());
16078 let mut one_leaf = Box::pin(one_leaf_ctx.select(vec![ParallelOperation::group(vec![
16079 ParallelOperation::activity("nested-only", json!([])),
16080 ])]));
16081 assert!(matches!(
16082 one_leaf.as_mut().poll(&mut task_context),
16083 Poll::Pending
16084 ));
16085 let one_leaf_commands = one_leaf_ctx.take_commands().expect("one-leaf commands");
16086 assert_eq!(one_leaf_commands.len(), 1);
16087 assert_eq!(
16088 one_leaf_commands[0]["parallel_group_path"][0]["selection_member_kind"],
16089 json!("group")
16090 );
16091 assert_eq!(
16092 one_leaf_commands[0]["parallel_group_path"][0]["selection_member_size"],
16093 json!(1)
16094 );
16095 }
16096
16097 async fn trip_saga(ctx: WorkflowContext) -> Result<Value> {
16098 let mut saga = ctx.saga();
16099 let outcome = async {
16100 let flight = ctx.activity("trip.reserve-flight", json!([])).await?;
16101 saga.add_compensation("trip.cancel-flight", json!([flight]))?;
16102 let hotel = ctx.activity("trip.reserve-hotel", json!([])).await?;
16103 saga.add_compensation("trip.cancel-hotel", json!([hotel]))?;
16104 ctx.activity("trip.charge", json!([])).await?;
16105 Ok(json!({"status": "booked"}))
16106 }
16107 .await;
16108 saga.finish(outcome).await
16109 }
16110
16111 fn saga_activity(
16112 event_type: &str,
16113 sequence: u64,
16114 activity_type: &str,
16115 result: Option<Value>,
16116 ) -> HistoryEvent {
16117 let mut payload = json!({
16118 "sequence": sequence,
16119 "activity_type": activity_type,
16120 "message": format!("{activity_type} failed"),
16121 "exception_type": "PlannedFailure",
16122 "non_retryable": true,
16123 });
16124 if let Some(result) = result {
16125 payload["result"] = fixture_envelope(result);
16126 }
16127 history_event(event_type, payload)
16128 }
16129
16130 #[test]
16131 fn saga_replays_reverse_compensation_across_restart_and_duplicate_delivery() {
16132 let completed_hotel_compensation = saga_activity(
16133 "ActivityCompleted",
16134 4,
16135 "trip.cancel-hotel",
16136 Some(Value::Null),
16137 );
16138 let history = vec![
16139 saga_activity(
16140 "ActivityCompleted",
16141 1,
16142 "trip.reserve-flight",
16143 Some(json!("flight-1")),
16144 ),
16145 saga_activity(
16146 "ActivityCompleted",
16147 2,
16148 "trip.reserve-hotel",
16149 Some(json!("hotel-1")),
16150 ),
16151 saga_activity("ActivityFailed", 3, "trip.charge", None),
16152 completed_hotel_compensation.clone(),
16153 completed_hotel_compensation,
16154 ];
16155
16156 for _restart in 0..2 {
16157 let ctx = workflow_context(history.clone());
16158 let mut future = Box::pin(trip_saga(ctx.clone()));
16159 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16160 assert!(matches!(
16161 future.as_mut().poll(&mut task_context),
16162 Poll::Pending
16163 ));
16164 let commands = ctx.take_commands().expect("compensation command");
16165 assert_eq!(commands.len(), 1);
16166 assert_eq!(commands[0]["activity_type"], "trip.cancel-flight");
16167 }
16168 }
16169
16170 #[test]
16171 fn saga_compensation_failure_preserves_both_typed_failures() {
16172 let history = vec![
16173 saga_activity(
16174 "ActivityCompleted",
16175 1,
16176 "trip.reserve-flight",
16177 Some(json!("flight-1")),
16178 ),
16179 saga_activity(
16180 "ActivityCompleted",
16181 2,
16182 "trip.reserve-hotel",
16183 Some(json!("hotel-1")),
16184 ),
16185 saga_activity("ActivityFailed", 3, "trip.charge", None),
16186 saga_activity("ActivityFailed", 4, "trip.cancel-hotel", None),
16187 ];
16188 let ctx = workflow_context(history);
16189 let mut future = Box::pin(trip_saga(ctx));
16190 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16191 let Poll::Ready(Err(Error::SagaCompensationFailed(failure))) =
16192 future.as_mut().poll(&mut task_context)
16193 else {
16194 panic!("compensation failure must remain structured");
16195 };
16196 assert!(matches!(
16197 *failure.initiating_failure,
16198 Error::ActivityFailed(_)
16199 ));
16200 assert!(matches!(
16201 *failure.compensation_failure,
16202 Error::ActivityFailed(_)
16203 ));
16204 assert_eq!(failure.compensation_activity_type, "trip.cancel-hotel");
16205 assert_eq!(failure.compensation_registration_order, 2);
16206 }
16207
16208 #[test]
16209 fn saga_compensates_cooperative_cancellation() {
16210 let ctx = workflow_context(vec![saga_activity(
16211 "ActivityCompleted",
16212 1,
16213 "trip.reserve-flight",
16214 Some(json!("flight-1")),
16215 )]);
16216 ctx.state.lock().expect("state").cancel_requested = true;
16217 let run = {
16218 let ctx = ctx.clone();
16219 async move {
16220 let mut saga = ctx.saga();
16221 let outcome = async {
16222 let flight = ctx.activity("trip.reserve-flight", json!([])).await?;
16223 saga.add_compensation("trip.cancel-flight", json!([flight]))?;
16224 ctx.throw_if_cancellation_requested()?;
16225 Ok(json!("unexpected"))
16226 }
16227 .await;
16228 saga.finish(outcome).await
16229 }
16230 };
16231 let mut future = Box::pin(run);
16232 let mut task_context = TaskContext::from_waker(noop_waker_ref());
16233 assert!(matches!(
16234 future.as_mut().poll(&mut task_context),
16235 Poll::Pending
16236 ));
16237 let commands = ctx.take_commands().expect("cancellation compensation");
16238 assert_eq!(commands[0]["activity_type"], "trip.cancel-flight");
16239 }
16240
16241 fn workflow_task(
16242 workflow_type: &str,
16243 history_events: Vec<HistoryEvent>,
16244 payload_codec: &str,
16245 ) -> WorkflowTask {
16246 WorkflowTask {
16247 task_id: format!("wft-{workflow_type}"),
16248 workflow_command_id: None,
16249 workflow_id: Some(format!("wf-{workflow_type}")),
16250 run_id: Some(format!("run-{workflow_type}")),
16251 workflow_type: workflow_type.to_string(),
16252 cancel_requested: false,
16253 payload_codec: payload_codec.to_string(),
16254 arguments: Some(
16255 encode_value_envelope(&json!([]), payload_codec).expect("workflow arguments"),
16256 ),
16257 total_history_events: Some(history_events.len() as u64),
16258 history_size_bytes: None,
16259 continue_as_new_recommended: None,
16260 history_budget_pressure: None,
16261 history_events,
16262 next_history_page_token: None,
16263 workflow_task_attempt: 1,
16264 workflow_signal_id: None,
16265 signal_name: None,
16266 signal_arguments: None,
16267 workflow_update_id: None,
16268 update_name: None,
16269 lease_owner: Some("rust-worker".to_string()),
16270 }
16271 }
16272
16273 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
16274 struct SideEffectProbe {
16275 request_id: String,
16276 attempt: u32,
16277 }
16278
16279 #[test]
16280 fn typed_side_effect_runs_callback_once_and_replay_skips_it() {
16281 let calls = AtomicUsize::new(0);
16282 let ctx = workflow_context(Vec::new());
16283 let value = ctx
16284 .side_effect(|| {
16285 calls.fetch_add(1, Ordering::SeqCst);
16286 SideEffectProbe {
16287 request_id: "request-42".to_string(),
16288 attempt: 3,
16289 }
16290 })
16291 .expect("first side effect");
16292 assert_eq!(value.attempt, 3);
16293 assert_eq!(calls.load(Ordering::SeqCst), 1);
16294 let commands = ctx.take_commands().expect("commands");
16295 assert_eq!(commands.len(), 1);
16296 assert_eq!(commands[0]["type"], "record_side_effect");
16297 assert_eq!(
16298 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
16299 serde_json::to_value(&value).expect("value")
16300 );
16301
16302 let replay = workflow_context(vec![history_event(
16303 "SideEffectRecorded",
16304 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16305 )]);
16306 let replayed: SideEffectProbe = replay
16307 .side_effect(|| {
16308 calls.fetch_add(1, Ordering::SeqCst);
16309 panic!("committed side-effect callbacks must not run during replay")
16310 })
16311 .expect("replayed side effect");
16312 assert_eq!(replayed, value);
16313 assert_eq!(calls.load(Ordering::SeqCst), 1);
16314 assert!(replay.take_commands().expect("commands").is_empty());
16315 replay.ensure_history_consumed().expect("history consumed");
16316 }
16317
16318 #[test]
16319 fn side_effect_uses_avro_envelope_and_uuid_is_replay_stable() {
16320 let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
16321 let value = ctx
16322 .side_effect(|| SideEffectProbe {
16323 request_id: "avro-request".to_string(),
16324 attempt: 1,
16325 })
16326 .expect("Avro side effect");
16327 let uuid = ctx.uuid_v4().expect("deterministic UUID");
16328 let commands = ctx.take_commands().expect("commands");
16329 assert_eq!(commands.len(), 2);
16330 assert_eq!(commands[0]["result"]["codec"], DEFAULT_CODEC);
16331 assert_eq!(commands[1]["result"]["codec"], DEFAULT_CODEC);
16332 assert_eq!(
16333 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
16334 serde_json::to_value(&value).expect("value")
16335 );
16336
16337 let replay = workflow_context_with_codec(
16338 vec![
16339 history_event(
16340 "SideEffectRecorded",
16341 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16342 ),
16343 history_event(
16344 "SideEffectRecorded",
16345 json!({"sequence": 2, "result": commands[1]["result"].clone()}),
16346 ),
16347 ],
16348 DEFAULT_CODEC,
16349 );
16350 let replayed: SideEffectProbe = replay
16351 .side_effect(|| panic!("Avro callback must not run"))
16352 .expect("replayed Avro value");
16353 let replayed_uuid = replay.uuid_v4().expect("replayed UUID");
16354 assert_eq!(replayed, value);
16355 assert_eq!(replayed_uuid, uuid);
16356 assert!(replay.take_commands().expect("commands").is_empty());
16357 }
16358
16359 #[test]
16360 fn typed_side_effect_replay_preserves_bytes_and_maps() {
16361 let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
16362 let value = ctx
16363 .side_effect_avro_value(typed_fidelity_probe)
16364 .expect("typed side effect");
16365 let commands = ctx.take_commands().expect("side-effect command");
16366 assert_eq!(
16367 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
16368 .expect("recorded side effect"),
16369 value
16370 );
16371
16372 let replay = workflow_context_with_codec(
16373 vec![history_event(
16374 "SideEffectRecorded",
16375 json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16376 )],
16377 DEFAULT_CODEC,
16378 );
16379 assert_eq!(
16380 replay
16381 .side_effect_avro_value(|| panic!("replay must not invoke callback"))
16382 .expect("replayed typed side effect"),
16383 value
16384 );
16385 }
16386
16387 #[test]
16388 fn ordered_side_effects_share_the_durable_command_stream() {
16389 let first = encode_value_envelope(&json!("first"), DEFAULT_CODEC).expect("first");
16390 let second = encode_value_envelope(&json!(29), DEFAULT_CODEC).expect("second");
16391 let ctx = workflow_context(vec![
16392 history_event(
16393 "SideEffectRecorded",
16394 json!({"sequence": 1, "result": first}),
16395 ),
16396 history_event(
16397 "SideEffectRecorded",
16398 json!({"sequence": 2, "result": second}),
16399 ),
16400 ]);
16401 let first: String = ctx
16402 .side_effect(|| panic!("first callback must not run"))
16403 .expect("first replay");
16404 let second: i32 = ctx
16405 .side_effect(|| panic!("second callback must not run"))
16406 .expect("second replay");
16407 assert_eq!(first, "first");
16408 assert_eq!(second, 29);
16409 ctx.ensure_history_consumed().expect("ordered history");
16410
16411 let reordered = workflow_context(vec![history_event(
16412 "VersionMarkerRecorded",
16413 json!({
16414 "sequence": 1,
16415 "change_id": "before-side-effect",
16416 "version": 1,
16417 "min_supported": 1,
16418 "max_supported": 1,
16419 }),
16420 )]);
16421 let error = reordered
16422 .side_effect(|| "new".to_string())
16423 .expect_err("command reordering must fail");
16424 assert!(matches!(
16425 error,
16426 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16427 if reason == "recorded_command_mismatch"
16428 ));
16429 }
16430
16431 #[test]
16432 fn version_markers_replay_across_upgrades_and_do_not_duplicate() {
16433 let ctx = workflow_context(Vec::new());
16434 assert_eq!(ctx.get_version("checkout-v2", 1, 2).expect("version"), 2);
16435 assert_eq!(ctx.get_version("checkout-v2", 1, 3).expect("cached"), 2);
16436 assert!(ctx.patched("new-search").expect("patch"));
16437 ctx.deprecate_patch("new-search").expect("deprecate patch");
16438 let commands = ctx.take_commands().expect("commands");
16439 assert_eq!(commands.len(), 2);
16440 assert_eq!(commands[0]["type"], "record_version_marker");
16441 assert_eq!(commands[0]["version"], 2);
16442 assert_eq!(commands[1]["change_id"], "new-search");
16443
16444 let replay = workflow_context(vec![history_event(
16445 "VersionMarkerRecorded",
16446 json!({
16447 "sequence": 1,
16448 "change_id": "checkout-v2",
16449 "version": 2,
16450 "min_supported": 1,
16451 "max_supported": 2,
16452 }),
16453 )]);
16454 assert_eq!(replay.get_version("checkout-v2", 1, 4).expect("upgrade"), 2);
16455 assert_eq!(replay.get_version("checkout-v2", 2, 5).expect("repeat"), 2);
16456 assert!(replay.take_commands().expect("commands").is_empty());
16457 replay.ensure_history_consumed().expect("history consumed");
16458 }
16459
16460 #[test]
16461 fn version_markers_reject_incompatible_or_malformed_history() {
16462 let incompatible = workflow_context(vec![history_event(
16463 "VersionMarkerRecorded",
16464 json!({
16465 "sequence": 1,
16466 "change_id": "checkout-v2",
16467 "version": 1,
16468 "min_supported": 1,
16469 "max_supported": 2,
16470 }),
16471 )]);
16472 let error = incompatible
16473 .get_version("checkout-v2", 2, 3)
16474 .expect_err("old version is unsupported");
16475 assert!(matches!(
16476 error,
16477 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16478 if reason == "version_marker_incompatible_range"
16479 ));
16480
16481 for (history, reason) in [
16482 (
16483 vec![history_event("SideEffectRecorded", json!({"sequence": 1}))],
16484 "side_effect_result_missing",
16485 ),
16486 (
16487 vec![history_event(
16488 "SideEffectRecorded",
16489 json!({
16490 "sequence": 1,
16491 "result": {"codec": "avro", "blob": "not-base64"},
16492 }),
16493 )],
16494 "side_effect_payload_incompatible",
16495 ),
16496 (
16497 vec![history_event(
16498 "SideEffectRecorded",
16499 json!({"sequence": 1, "result": {"unwrapped": true}}),
16500 )],
16501 "side_effect_payload_malformed",
16502 ),
16503 (
16504 vec![history_event(
16505 "VersionMarkerRecorded",
16506 json!({
16507 "sequence": 1,
16508 "change_id": "change",
16509 "version": 1,
16510 "min_supported": 2,
16511 "max_supported": 1,
16512 }),
16513 )],
16514 "version_marker_history_range_invalid",
16515 ),
16516 ] {
16517 let error = WorkflowState::new(
16518 history,
16519 "rust-workers".to_string(),
16520 DEFAULT_CODEC.to_string(),
16521 None,
16522 )
16523 .expect_err("malformed history must fail");
16524 assert!(matches!(
16525 error,
16526 Error::NonDeterministicReplay(ReplayFailure { reason: actual, .. })
16527 if actual == reason
16528 ));
16529 }
16530 }
16531
16532 #[test]
16533 fn typed_search_attributes_replay_value_and_type_identity_after_restart() {
16534 let history = vec![history_event(
16535 "SearchAttributesUpserted",
16536 json!({
16537 "sequence": 1,
16538 "attributes": {"customer_tier": "gold"},
16539 "attribute_types": {"customer_tier": "keyword"},
16540 "merged": {"customer_tier": "gold"}
16541 }),
16542 )];
16543
16544 let matching = workflow_context(history.clone());
16545 matching
16546 .upsert_search_attributes(
16547 SearchAttributeUpdate::new()
16548 .keyword("customer_tier", "gold")
16549 .expect("keyword update"),
16550 )
16551 .expect("matching typed update must replay");
16552 matching
16553 .ensure_history_consumed()
16554 .expect("history consumed");
16555
16556 let changed_type = workflow_context(history.clone());
16557 let error = changed_type
16558 .upsert_search_attributes(
16559 SearchAttributeUpdate::new()
16560 .string("customer_tier", "gold")
16561 .expect("string update"),
16562 )
16563 .expect_err("same JSON value with a different declaration must be nondeterministic");
16564 let Error::NonDeterministicReplay(failure) = error else {
16565 panic!("typed identity drift must be a replay failure");
16566 };
16567 assert_eq!(failure.reason, "search_attribute_type_mismatch");
16568 assert_eq!(failure.sequence, Some(1));
16569
16570 let changed_value = workflow_context(history);
16571 let error = changed_value
16572 .upsert_search_attributes(
16573 SearchAttributeUpdate::new()
16574 .keyword("customer_tier", "platinum")
16575 .expect("keyword update"),
16576 )
16577 .expect_err("changed values must be nondeterministic");
16578 let Error::NonDeterministicReplay(failure) = error else {
16579 panic!("value drift must be a replay failure");
16580 };
16581 assert_eq!(failure.reason, "search_attribute_value_mismatch");
16582 }
16583
16584 #[test]
16585 fn legacy_search_attribute_history_keeps_type_identity_unknown() {
16586 let history = vec![history_event(
16587 "SearchAttributesUpserted",
16588 json!({
16589 "sequence": 1,
16590 "attributes": {"customer_tier": "gold"},
16591 "merged": {"customer_tier": "gold"}
16592 }),
16593 )];
16594
16595 for update in [
16596 SearchAttributeUpdate::new()
16597 .keyword("customer_tier", "gold")
16598 .expect("keyword update"),
16599 SearchAttributeUpdate::new()
16600 .string("customer_tier", "gold")
16601 .expect("string update"),
16602 ] {
16603 let restarted = workflow_context(history.clone());
16604 restarted
16605 .upsert_search_attributes(update)
16606 .expect("legacy history constrains values but has unknown type identity");
16607 restarted
16608 .ensure_history_consumed()
16609 .expect("history consumed");
16610 }
16611 }
16612
16613 #[test]
16614 fn search_attribute_command_emits_canonical_types() {
16615 let ctx = workflow_context(Vec::new());
16616 ctx.upsert_search_attributes(
16617 SearchAttributeUpdate::new()
16618 .keyword("customer_tier", "gold")
16619 .expect("keyword update")
16620 .int("attempts", 3)
16621 .expect("int update")
16622 .delete("obsolete")
16623 .expect("delete update"),
16624 )
16625 .expect("valid search attributes");
16626
16627 assert_eq!(
16628 ctx.take_commands().expect("commands"),
16629 vec![json!({
16630 "type": "upsert_search_attributes",
16631 "attributes": {
16632 "attempts": 3,
16633 "customer_tier": "gold",
16634 "obsolete": null
16635 },
16636 "attribute_types": {
16637 "attempts": "int",
16638 "customer_tier": "keyword"
16639 }
16640 })]
16641 );
16642 }
16643
16644 #[test]
16645 fn duplicate_side_effects_and_version_markers_are_rejected() {
16646 let duplicate_side_effect = WorkflowState::new(
16647 vec![
16648 history_event(
16649 "SideEffectRecorded",
16650 json!({"sequence": 1, "result": fixture_envelope(json!(1))}),
16651 ),
16652 history_event(
16653 "SideEffectRecorded",
16654 json!({"sequence": 1, "result": fixture_envelope(json!(2))}),
16655 ),
16656 ],
16657 "rust-workers".to_string(),
16658 DEFAULT_CODEC.to_string(),
16659 None,
16660 )
16661 .expect_err("duplicate side effect");
16662 assert!(matches!(
16663 duplicate_side_effect,
16664 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16665 if reason == "duplicate_side_effect_record"
16666 ));
16667
16668 let marker = |sequence| {
16669 history_event(
16670 "VersionMarkerRecorded",
16671 json!({
16672 "sequence": sequence,
16673 "change_id": "same-change",
16674 "version": 1,
16675 "min_supported": 1,
16676 "max_supported": 1,
16677 }),
16678 )
16679 };
16680 let duplicate_marker = WorkflowState::new(
16681 vec![marker(1), marker(3)],
16682 "rust-workers".to_string(),
16683 DEFAULT_CODEC.to_string(),
16684 None,
16685 )
16686 .expect_err("duplicate marker");
16687 assert!(matches!(
16688 duplicate_marker,
16689 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16690 if reason == "duplicate_version_marker"
16691 ));
16692 }
16693
16694 #[test]
16695 fn workflow_stream_authoring_derives_identity_and_replay_skips_duplicate_append() {
16696 let mut state = WorkflowState::new(
16697 Vec::new(),
16698 "rust-workers".to_string(),
16699 DEFAULT_CODEC.to_string(),
16700 None,
16701 )
16702 .expect("workflow state");
16703 state.workflow_command_identity = "command-7".to_string();
16704 let context = WorkflowContext {
16705 state: Arc::new(Mutex::new(state)),
16706 };
16707 let item =
16708 WorkflowStreamAppendItem::from_reference("s3://bucket/item.avro").item_type("receipt");
16709
16710 context
16711 .append_workflow_stream("output", &[item], Some(10))
16712 .expect("append command");
16713 context
16714 .error_workflow_stream("output", "producer failed", None)
16715 .expect("error command");
16716 let commands = context.take_commands().expect("commands");
16717
16718 assert_eq!(commands[0]["type"], "record_side_effect");
16719 assert_eq!(
16720 commands[0]["workflow_stream"]["command_identity"],
16721 "command-7"
16722 );
16723 assert_eq!(commands[0]["workflow_stream"]["command_ordinal"], 0);
16724 assert_eq!(
16725 commands[0]["workflow_stream"]["items"][0]["idempotency_key"],
16726 "dw-stream:command-7:0:0"
16727 );
16728 assert_eq!(commands[1]["workflow_stream"]["operation"], "error");
16729
16730 let recorded = history_event(
16731 "SideEffectRecorded",
16732 json!({"sequence": 1, "result": fixture_envelope(Value::Null)}),
16733 );
16734 let mut replay_state = WorkflowState::new(
16735 vec![recorded],
16736 "rust-workers".to_string(),
16737 DEFAULT_CODEC.to_string(),
16738 None,
16739 )
16740 .expect("replay state");
16741 replay_state.workflow_command_identity = "command-7".to_string();
16742 let replay_context = WorkflowContext {
16743 state: Arc::new(Mutex::new(replay_state)),
16744 };
16745 replay_context
16746 .append_workflow_stream(
16747 "output",
16748 &[WorkflowStreamAppendItem::from_reference(
16749 "s3://bucket/item.avro",
16750 )],
16751 Some(10),
16752 )
16753 .expect("replayed append");
16754 assert!(replay_context
16755 .take_commands()
16756 .expect("replayed commands")
16757 .is_empty());
16758 }
16759
16760 #[test]
16761 fn workflow_stream_authoring_requires_server_durable_command_identity() {
16762 let context = workflow_context(Vec::new());
16763 let error = context
16764 .append_workflow_stream(
16765 "output",
16766 &[WorkflowStreamAppendItem::from_reference(
16767 "s3://bucket/item.avro",
16768 )],
16769 None,
16770 )
16771 .expect_err("stream append without durable command identity must fail closed");
16772
16773 assert!(matches!(error, Error::MissingWorkflowCommandIdentity));
16774 assert!(context.take_commands().expect("commands").is_empty());
16775 }
16776
16777 #[test]
16778 fn cold_worker_replay_does_not_repeat_committed_side_effects_or_markers() {
16779 fn worker(calls: Arc<AtomicUsize>) -> Worker {
16780 let client = Client::new("http://127.0.0.1:8080").expect("client");
16781 let mut worker = Worker::new(client, "rust-workers");
16782 worker.register_workflow("rust.side-effect-version", move |ctx, _input| {
16783 let calls = Arc::clone(&calls);
16784 async move {
16785 let captured = ctx.side_effect(|| {
16786 calls.fetch_add(1, Ordering::SeqCst);
16787 "captured-once".to_string()
16788 })?;
16789 let version = ctx.get_version("cold-restart", 1, 2)?;
16790 Ok(json!({"captured": captured, "version": version}))
16791 }
16792 });
16793 worker
16794 }
16795
16796 fn task(history_events: Vec<HistoryEvent>) -> WorkflowTask {
16797 WorkflowTask {
16798 task_id: "wft-side-effect-version".to_string(),
16799 workflow_command_id: None,
16800 workflow_id: Some("wf-side-effect-version".to_string()),
16801 run_id: Some("run-side-effect-version".to_string()),
16802 workflow_type: "rust.side-effect-version".to_string(),
16803 cancel_requested: false,
16804 payload_codec: DEFAULT_CODEC.to_string(),
16805 arguments: Some(
16806 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("arguments"),
16807 ),
16808 history_events,
16809 total_history_events: None,
16810 history_size_bytes: None,
16811 continue_as_new_recommended: None,
16812 history_budget_pressure: None,
16813 next_history_page_token: None,
16814 workflow_task_attempt: 1,
16815 workflow_signal_id: None,
16816 signal_name: None,
16817 signal_arguments: None,
16818 workflow_update_id: None,
16819 update_name: None,
16820 lease_owner: Some("rust-worker".to_string()),
16821 }
16822 }
16823
16824 let calls = Arc::new(AtomicUsize::new(0));
16825 let initial = worker(Arc::clone(&calls))
16826 .execute_workflow_task(task(Vec::new()))
16827 .expect("initial execution");
16828 assert_eq!(
16829 initial
16830 .iter()
16831 .map(|command| &command["type"])
16832 .collect::<Vec<_>>(),
16833 vec![
16834 "record_side_effect",
16835 "record_version_marker",
16836 "complete_workflow"
16837 ]
16838 );
16839 assert_eq!(calls.load(Ordering::SeqCst), 1);
16840
16841 let restarted = worker(Arc::clone(&calls));
16842 let replayed = restarted
16843 .execute_workflow_task(task(vec![
16844 history_event(
16845 "SideEffectRecorded",
16846 json!({"sequence": 1, "result": initial[0]["result"].clone()}),
16847 ),
16848 history_event(
16849 "VersionMarkerRecorded",
16850 json!({
16851 "sequence": 2,
16852 "change_id": "cold-restart",
16853 "version": 2,
16854 "min_supported": 1,
16855 "max_supported": 2,
16856 }),
16857 ),
16858 ]))
16859 .expect("cold replay");
16860 assert_eq!(replayed.len(), 1);
16861 assert_eq!(replayed[0]["type"], "complete_workflow");
16862 assert_eq!(calls.load(Ordering::SeqCst), 1);
16863 }
16864
16865 #[test]
16866 fn side_effect_replay_rejects_changed_rust_value_type() {
16867 let result = encode_value_envelope(&json!({"value": 42}), DEFAULT_CODEC).expect("result");
16868 let ctx = workflow_context(vec![history_event(
16869 "SideEffectRecorded",
16870 json!({"sequence": 1, "result": result}),
16871 )]);
16872 let error = ctx
16873 .side_effect::<Vec<String>, _>(|| panic!("callback must not run"))
16874 .expect_err("changed type must fail replay");
16875 assert!(matches!(
16876 error,
16877 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16878 if reason == "side_effect_type_mismatch"
16879 ));
16880 }
16881
16882 fn completed_retry_activity_history() -> Vec<HistoryEvent> {
16883 vec![
16884 history_event(
16885 "ActivityScheduled",
16886 json!({
16887 "sequence": 1,
16888 "activity_type": "flaky",
16889 "activity_execution_id": "act-1",
16890 "activity": {
16891 "id": "act-1",
16892 "sequence": 1,
16893 "type": "flaky",
16894 "queue": "critical-activities",
16895 "execution_mode": null,
16896 "retry_policy": {
16897 "snapshot_version": 1,
16898 "max_attempts": 3,
16899 "backoff_seconds": [2, 4],
16900 "start_to_close_timeout": 30,
16901 "schedule_to_start_timeout": 5,
16902 "schedule_to_close_timeout": 90,
16903 "heartbeat_timeout": 10,
16904 "non_retryable_error_types": ["PermanentError"]
16905 }
16906 }
16907 }),
16908 ),
16909 history_event(
16910 "ActivityStarted",
16911 json!({
16912 "sequence": 1,
16913 "activity_type": "flaky",
16914 "activity_execution_id": "act-1",
16915 "activity_attempt_id": "attempt-1",
16916 "attempt_number": 1
16917 }),
16918 ),
16919 history_event(
16920 "ActivityRetryScheduled",
16921 json!({
16922 "sequence": 1,
16923 "activity_type": "flaky",
16924 "activity_execution_id": "act-1",
16925 "activity_attempt_id": "attempt-1",
16926 "attempt_number": 1,
16927 "retry_after_attempt": 1,
16928 "retry_backoff_seconds": 2,
16929 "failure_category": "activity",
16930 "exception_type": "TransientError"
16931 }),
16932 ),
16933 history_event(
16934 "ActivityStarted",
16935 json!({
16936 "sequence": 1,
16937 "activity_type": "flaky",
16938 "activity_execution_id": "act-1",
16939 "activity_attempt_id": "attempt-2",
16940 "attempt_number": 2
16941 }),
16942 ),
16943 history_event(
16944 "ActivityCompleted",
16945 json!({
16946 "sequence": 1,
16947 "activity_type": "flaky",
16948 "activity_execution_id": "act-1",
16949 "activity_attempt_id": "attempt-2",
16950 "attempt_number": 2,
16951 "payload_codec": DEFAULT_CODEC,
16952 "result": fixture_envelope(json!({"status":"recovered"}))
16953 }),
16954 ),
16955 ]
16956 }
16957
16958 fn retry_activity_options() -> ActivityOptions {
16959 ActivityOptions::new()
16960 .task_queue("critical-activities")
16961 .retry_policy(
16962 ActivityRetryPolicy::new(3)
16963 .backoff_intervals([Duration::from_secs(2), Duration::from_secs(4)])
16964 .non_retryable_error_type("PermanentError"),
16965 )
16966 .start_to_close_timeout(Duration::from_secs(30))
16967 .schedule_to_start_timeout(Duration::from_secs(5))
16968 .schedule_to_close_timeout(Duration::from_secs(90))
16969 .heartbeat_timeout(Duration::from_secs(10))
16970 }
16971
16972 #[test]
16973 fn fixed_avro_value_round_trips_json_values() {
16974 let value = json!({"greeting": "hello", "count": 3, "ok": true});
16975 let envelope = PayloadEnvelope::avro(&value).expect("encode");
16976 assert_eq!(envelope.codec, DEFAULT_CODEC);
16977 assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
16978 }
16979
16980 #[tokio::test]
16981 async fn typed_handler_adapters_round_trip_serde_contracts_on_the_fixed_wire() {
16982 let client = Client::new("http://127.0.0.1:8080").expect("client");
16983 let mut worker = Worker::new(client, "rust-workers");
16984 worker.register_typed_workflow(
16985 "typed.contract.workflow",
16986 |_ctx, input: TypedContract| async move { Ok(input) },
16987 );
16988 worker.register_typed_activity(
16989 "typed.contract.activity",
16990 |_ctx, input: TypedContract| async move { Ok(input) },
16991 );
16992
16993 let expected = typed_contract();
16994 let arguments = AvroValue::Array(vec![
16995 AvroValue::from_serialize(&expected).expect("typed request")
16996 ]);
16997 let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("arguments");
16998 let mut workflow = workflow_task("typed.contract.workflow", Vec::new(), DEFAULT_CODEC);
16999 workflow.arguments = Some(envelope.clone());
17000 let commands = worker
17001 .execute_workflow_task(workflow)
17002 .expect("typed workflow task");
17003 let workflow_result: TypedContract =
17004 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17005 .expect("workflow result envelope")
17006 .deserialize()
17007 .expect("workflow result type");
17008 assert_eq!(workflow_result, expected);
17009
17010 let activity = ActivityTask {
17011 task_id: "typed-contract-activity".to_string(),
17012 activity_attempt_id: Some("typed-contract-attempt".to_string()),
17013 attempt_id: None,
17014 activity_type: "typed.contract.activity".to_string(),
17015 payload_codec: DEFAULT_CODEC.to_string(),
17016 arguments: Some(envelope),
17017 attempt_number: 1,
17018 lease_owner: Some("rust-worker".to_string()),
17019 };
17020 let activity_result: TypedContract = worker
17021 .execute_activity_task(activity)
17022 .await
17023 .expect("typed activity task")
17024 .deserialize()
17025 .expect("activity result type");
17026 assert_eq!(activity_result, expected);
17027 }
17028
17029 #[tokio::test]
17030 async fn typed_handler_errors_include_handler_name_direction_and_rust_type() {
17031 let client = Client::new("http://127.0.0.1:8080").expect("client");
17032 let mut worker = Worker::new(client, "rust-workers");
17033 worker.register_typed_workflow(
17034 "typed.shape.workflow",
17035 |_ctx, input: TypedContract| async move { Ok(input) },
17036 );
17037 worker.register_typed_activity("typed.unsupported.activity", |_ctx, (): ()| async move {
17038 Ok(f64::NAN)
17039 });
17040
17041 let mut workflow = workflow_task("typed.shape.workflow", Vec::new(), DEFAULT_CODEC);
17042 workflow.arguments = Some(
17043 encode_typed_envelope(
17044 &AvroValue::Array(vec![
17045 AvroValue::String("first".to_string()),
17046 AvroValue::String("second".to_string()),
17047 ]),
17048 DEFAULT_CODEC,
17049 )
17050 .expect("malformed typed arguments"),
17051 );
17052 let commands = worker
17053 .execute_workflow_task(workflow)
17054 .expect("shape mismatch becomes a workflow failure");
17055 let message = commands[0]["message"].as_str().expect("failure message");
17056 assert!(message.contains("workflow handler \"typed.shape.workflow\" input type"));
17057 assert!(message.contains(type_name::<TypedContract>()));
17058 assert!(message.contains("task carried 2 arguments"));
17059
17060 let activity = ActivityTask {
17061 task_id: "typed-unsupported-activity".to_string(),
17062 activity_attempt_id: Some("typed-unsupported-attempt".to_string()),
17063 attempt_id: None,
17064 activity_type: "typed.unsupported.activity".to_string(),
17065 payload_codec: DEFAULT_CODEC.to_string(),
17066 arguments: Some(
17067 encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
17068 .expect("unit arguments"),
17069 ),
17070 attempt_number: 1,
17071 lease_owner: Some("rust-worker".to_string()),
17072 };
17073 let Error::HandlerType {
17074 handler_kind,
17075 handler_name,
17076 value_kind,
17077 rust_type,
17078 message,
17079 } = worker
17080 .execute_activity_task(activity)
17081 .await
17082 .expect_err("non-finite handler output must fail")
17083 else {
17084 panic!("expected contextual handler type failure");
17085 };
17086 assert_eq!(handler_kind, HandlerKind::Activity);
17087 assert_eq!(handler_name, "typed.unsupported.activity");
17088 assert_eq!(value_kind, HandlerValueKind::Result);
17089 assert_eq!(rust_type, type_name::<f64>());
17090 assert!(message.contains("non_finite_float"));
17091 }
17092
17093 #[tokio::test]
17094 async fn typed_replayed_workflow_decodes_input_and_activity_result_losslessly() {
17095 #[derive(Clone, Default)]
17096 struct State {
17097 observed: Option<TypedContract>,
17098 }
17099
17100 let client = Client::new("http://127.0.0.1:8080").expect("client");
17101 let mut worker = Worker::new(client, "rust-workers");
17102 worker.register_typed_replayed_workflow(
17103 "typed.contract.replayed",
17104 State::default,
17105 |ctx, input: TypedContract, state| async move {
17106 let result: TypedContract =
17107 ctx.activity_typed("typed.contract.activity", input).await?;
17108 state.update(|current| current.observed = Some(result.clone()))?;
17109 Ok(result)
17110 },
17111 );
17112 worker.register_replayed_query::<State, _, _>(
17113 "typed.contract.replayed",
17114 "observed",
17115 |_ctx, state, _args| async move {
17116 Ok(json!(state.observed.as_ref().map(|value| value.signed)))
17117 },
17118 );
17119
17120 let expected = typed_contract();
17121 let typed_value = AvroValue::from_serialize(&expected).expect("typed value");
17122 let workflow_arguments =
17123 encode_typed_envelope(&AvroValue::Array(vec![typed_value.clone()]), DEFAULT_CODEC)
17124 .expect("workflow arguments");
17125 let result = encode_typed_envelope(&typed_value, DEFAULT_CODEC).expect("activity result");
17126 let task = QueryTask {
17127 query_task_id: "typed-replay-query".to_string(),
17128 query_task_attempt: 1,
17129 lease_owner: Some("rust-worker".to_string()),
17130 workflow_id: Some("typed-replay".to_string()),
17131 run_id: Some("typed-replay-run".to_string()),
17132 workflow_type: "typed.contract.replayed".to_string(),
17133 query_name: "observed".to_string(),
17134 payload_codec: DEFAULT_CODEC.to_string(),
17135 workflow_arguments: Some(workflow_arguments),
17136 query_arguments: Some(
17137 encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
17138 .expect("query arguments"),
17139 ),
17140 history_events: vec![
17141 history_event(
17142 "ActivityScheduled",
17143 json!({
17144 "sequence": 1,
17145 "activity_type": "typed.contract.activity"
17146 }),
17147 ),
17148 history_event(
17149 "ActivityCompleted",
17150 json!({
17151 "sequence": 1,
17152 "activity_type": "typed.contract.activity",
17153 "payload_codec": DEFAULT_CODEC,
17154 "result": result
17155 }),
17156 ),
17157 ],
17158 history_export: None,
17159 run_status: Some("completed".to_string()),
17160 };
17161
17162 assert_eq!(
17163 worker
17164 .execute_query_task(task)
17165 .await
17166 .expect("typed replay query")
17167 .deserialize::<i64>()
17168 .expect("query result"),
17169 expected.signed
17170 );
17171 }
17172
17173 #[tokio::test]
17174 async fn typed_worker_surfaces_preserve_bytes_and_map_list_identity() {
17175 let client = Client::new("http://127.0.0.1:8080").expect("client");
17176 let mut worker = Worker::new(client, "rust-workers");
17177 worker.register_workflow_avro_value("typed.echo", |_ctx, input| async move { Ok(input) });
17178 worker
17179 .register_activity_avro_value("typed.activity", |_ctx, input| async move { Ok(input) });
17180 worker.register_query_avro_value("typed.echo", "inspect", |_ctx, input| async move {
17181 Ok(input)
17182 });
17183 worker.register_update_avro_value("typed.echo", "replace", |_ctx, input| async move {
17184 Ok(input)
17185 });
17186 worker.register_workflow_avro_value("typed.signal", |ctx, _input| async move {
17187 Ok(AvroValue::Array(
17188 ctx.wait_signal_avro_value("changed").await?,
17189 ))
17190 });
17191
17192 let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
17193 let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed envelope");
17194
17195 let mut workflow = workflow_task("typed.echo", Vec::new(), DEFAULT_CODEC);
17196 workflow.arguments = Some(envelope.clone());
17197 let commands = worker
17198 .execute_workflow_task(workflow)
17199 .expect("typed workflow task");
17200 assert_eq!(commands[0]["type"], "complete_workflow");
17201 assert_eq!(
17202 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17203 .expect("typed workflow result"),
17204 arguments
17205 );
17206
17207 let activity = ActivityTask {
17208 task_id: "activity-typed".to_string(),
17209 activity_attempt_id: Some("attempt-typed".to_string()),
17210 attempt_id: None,
17211 activity_type: "typed.activity".to_string(),
17212 payload_codec: DEFAULT_CODEC.to_string(),
17213 arguments: Some(envelope.clone()),
17214 attempt_number: 1,
17215 lease_owner: Some("rust-worker".to_string()),
17216 };
17217 assert_eq!(
17218 worker
17219 .execute_activity_task(activity)
17220 .await
17221 .expect("typed activity result"),
17222 arguments
17223 );
17224
17225 let query = QueryTask {
17226 query_task_id: "query-typed".to_string(),
17227 query_task_attempt: 1,
17228 lease_owner: Some("rust-worker".to_string()),
17229 workflow_id: Some("typed-1".to_string()),
17230 run_id: Some("run-typed".to_string()),
17231 workflow_type: "typed.echo".to_string(),
17232 query_name: "inspect".to_string(),
17233 payload_codec: DEFAULT_CODEC.to_string(),
17234 workflow_arguments: Some(
17235 encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
17236 .expect("workflow input"),
17237 ),
17238 query_arguments: Some(envelope.clone()),
17239 history_events: Vec::new(),
17240 history_export: None,
17241 run_status: Some("running".to_string()),
17242 };
17243 assert_eq!(
17244 worker
17245 .execute_query_task(query)
17246 .await
17247 .expect("typed query result"),
17248 arguments
17249 );
17250
17251 let mut update = workflow_task(
17252 "typed.echo",
17253 vec![history_event(
17254 "UpdateAccepted",
17255 json!({
17256 "update_id": "update-typed",
17257 "update_name": "replace",
17258 "arguments": envelope.clone(),
17259 }),
17260 )],
17261 DEFAULT_CODEC,
17262 );
17263 update.workflow_update_id = Some("update-typed".to_string());
17264 update.update_name = Some("replace".to_string());
17265 let commands = worker
17266 .execute_workflow_task(update)
17267 .expect("typed update task");
17268 assert_eq!(commands[0]["type"], "complete_update");
17269 assert_eq!(
17270 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17271 .expect("typed update result"),
17272 arguments
17273 );
17274
17275 let mut signal = workflow_task(
17276 "typed.signal",
17277 vec![history_event(
17278 "SignalReceived",
17279 json!({
17280 "signal_id": "signal-typed",
17281 "signal_name": "changed",
17282 "arguments": envelope.clone(),
17283 }),
17284 )],
17285 DEFAULT_CODEC,
17286 );
17287 signal.workflow_signal_id = Some("signal-typed".to_string());
17288 signal.signal_name = Some("changed".to_string());
17289 signal.signal_arguments = Some(envelope);
17290 let commands = worker
17291 .execute_workflow_task(signal)
17292 .expect("typed signal resume");
17293 assert_eq!(
17294 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17295 .expect("typed signal result"),
17296 arguments
17297 );
17298 }
17299
17300 #[tokio::test]
17301 async fn typed_helpers_never_parse_json_inspection_projection() {
17302 let collision_values = projection_collision_probe();
17303 let expected = AvroValue::Array(collision_values.clone());
17304 let envelope = encode_typed_envelope(&expected, DEFAULT_CODEC).expect("collision envelope");
17305
17306 let activity_context = workflow_context_with_codec(
17307 vec![history_event(
17308 "ActivityCompleted",
17309 json!({
17310 "sequence": 1,
17311 "activity_type": "collision.activity",
17312 "payload_codec": DEFAULT_CODEC,
17313 "result": envelope.clone(),
17314 }),
17315 )],
17316 DEFAULT_CODEC,
17317 );
17318 assert_eq!(
17319 activity_context
17320 .activity_avro_value("collision.activity", AvroValue::Array(Vec::new()))
17321 .await
17322 .expect("typed activity collision result"),
17323 expected
17324 );
17325
17326 let signal_context = workflow_context_with_codec(
17327 vec![
17328 history_event(
17329 "SignalWaitOpened",
17330 json!({"sequence": 1, "signal_name": "collision"}),
17331 ),
17332 history_event(
17333 "SignalApplied",
17334 json!({
17335 "sequence": 1,
17336 "signal_name": "collision",
17337 "payload_codec": DEFAULT_CODEC,
17338 "value": envelope.clone(),
17339 }),
17340 ),
17341 ],
17342 DEFAULT_CODEC,
17343 );
17344 assert_eq!(
17345 signal_context
17346 .wait_signal_avro_value("collision")
17347 .await
17348 .expect("typed signal collision arguments"),
17349 collision_values
17350 );
17351
17352 let child_context = workflow_context_with_codec(
17353 vec![
17354 history_event(
17355 "ChildWorkflowScheduled",
17356 json!({
17357 "sequence": 1,
17358 "child_workflow_instance_id": "collision-child",
17359 "child_workflow_run_id": "collision-run",
17360 "child_workflow_type": "collision.child",
17361 }),
17362 ),
17363 history_event(
17364 "ChildRunCompleted",
17365 json!({
17366 "sequence": 1,
17367 "child_workflow_instance_id": "collision-child",
17368 "child_workflow_run_id": "collision-run",
17369 "child_workflow_type": "collision.child",
17370 "payload_codec": DEFAULT_CODEC,
17371 "result": envelope,
17372 }),
17373 ),
17374 ],
17375 DEFAULT_CODEC,
17376 );
17377 let child = child_context
17378 .start_child_workflow_avro_value(
17379 "collision.child",
17380 ChildWorkflowOptions::new("collision-workers"),
17381 AvroValue::Array(Vec::new()),
17382 )
17383 .await
17384 .expect("typed child collision result");
17385 assert_eq!(child.result, expected);
17386 }
17387
17388 #[tokio::test]
17389 async fn replayed_typed_query_keeps_lossless_workflow_and_query_inputs() {
17390 let client = Client::new("http://127.0.0.1:8080").expect("client");
17391 let mut worker = Worker::new(client, "rust-workers");
17392 worker.register_replayed_workflow_avro_value(
17393 "typed.replayed",
17394 || (),
17395 |_ctx, input, _state| async move { Ok(input) },
17396 );
17397 worker.register_replayed_query_avro_value::<(), _, _>(
17398 "typed.replayed",
17399 "inspect",
17400 |ctx, _state, args| async move {
17401 let mut signals = ctx.signals_avro_value("collision");
17402 let signal = signals
17403 .pop()
17404 .map(AvroValue::Array)
17405 .unwrap_or_else(|| AvroValue::Array(Vec::new()));
17406 Ok(AvroValue::Array(vec![
17407 ctx.workflow_input_avro_value().clone(),
17408 signal,
17409 args,
17410 ]))
17411 },
17412 );
17413 let arguments = AvroValue::Array(projection_collision_probe());
17414 let signal_arguments =
17415 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed query signal arguments");
17416 let task = QueryTask {
17417 query_task_id: "query-typed-replay".to_string(),
17418 query_task_attempt: 1,
17419 lease_owner: Some("rust-worker".to_string()),
17420 workflow_id: Some("typed-replay".to_string()),
17421 run_id: Some("run-typed-replay".to_string()),
17422 workflow_type: "typed.replayed".to_string(),
17423 query_name: "inspect".to_string(),
17424 payload_codec: DEFAULT_CODEC.to_string(),
17425 workflow_arguments: Some(
17426 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("workflow arguments"),
17427 ),
17428 query_arguments: Some(
17429 encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("query arguments"),
17430 ),
17431 history_events: vec![history_event(
17432 "SignalReceived",
17433 json!({
17434 "signal_id": "collision-signal",
17435 "signal_name": "collision",
17436 "workflow_sequence": 1,
17437 "payload_codec": DEFAULT_CODEC,
17438 "arguments": signal_arguments,
17439 }),
17440 )],
17441 history_export: None,
17442 run_status: Some("completed".to_string()),
17443 };
17444
17445 assert_eq!(
17446 worker
17447 .execute_query_task(task)
17448 .await
17449 .expect("typed replay query"),
17450 AvroValue::Array(vec![arguments.clone(), arguments.clone(), arguments])
17451 );
17452 }
17453
17454 #[test]
17455 fn public_avro_adapter_rejects_non_string_map_keys_before_json_conversion() {
17456 let value = BTreeMap::from([(1_i32, "integer key")]);
17457 let error = PayloadEnvelope::avro(&value)
17458 .expect_err("integer map keys must fail")
17459 .to_string();
17460
17461 assert!(error.contains("invalid_map_key"));
17462 }
17463
17464 #[test]
17465 fn json_tagged_payload_fails_closed_with_actionable_diagnostic() {
17466 let envelope = PayloadEnvelope {
17467 codec: "json".to_string(),
17468 blob: r#"{"greeting":"hello"}"#.to_string(),
17469 };
17470
17471 let error = decode_payload::<Value>(&envelope).expect_err("JSON payload must fail");
17472 let diagnostic = error.to_string();
17473 assert!(diagnostic.contains("unsupported_payload_codec"));
17474 assert!(diagnostic.contains("codec=\"avro\""));
17475 assert!(diagnostic.contains("HTTP document transport"));
17476 }
17477
17478 #[test]
17479 fn untagged_json_payload_value_fails_closed() {
17480 let error = decode_wire_value(&json!({"stale": true}), DEFAULT_CODEC)
17481 .expect_err("untagged JSON payload values must fail");
17482 let diagnostic = error.to_string();
17483 assert!(diagnostic.contains("unsupported_payload_codec"));
17484 assert!(diagnostic.contains("untagged durable payload"));
17485 assert!(diagnostic.contains("HTTP document transport"));
17486 }
17487
17488 #[test]
17489 fn prerelease_avro_payload_without_single_object_frame_is_rejected() {
17490 let envelope = PayloadEnvelope {
17491 codec: DEFAULT_CODEC.to_string(),
17492 blob: BASE64.encode([0x01]),
17493 };
17494
17495 let error = decode_payload::<Value>(&envelope).expect_err("prerelease payload must fail");
17496 assert!(error.to_string().contains("invalid_payload_framing"));
17497 }
17498
17499 #[tokio::test]
17500 async fn workflow_completion_rejects_invalid_payload_slots_without_transport() {
17501 let server = MockWorkerServer::start();
17502 let client = Client::builder(server.base_url())
17503 .timeout(Duration::from_secs(2))
17504 .build()
17505 .expect("client");
17506 let invalid_commands = [
17507 json!({
17508 "type": "complete_workflow",
17509 "result": {"codec": "json", "blob": null}
17510 }),
17511 json!({
17512 "type": "schedule_activity",
17513 "arguments": {"codec": "yaml", "blob": "ignored"}
17514 }),
17515 json!({
17516 "type": "start_child_workflow",
17517 "arguments": {"codec": DEFAULT_CODEC, "blob": null}
17518 }),
17519 json!({"type": "continue_as_new", "arguments": []}),
17520 json!({"type": "complete_update"}),
17521 json!({"type": "record_side_effect", "result": null}),
17522 json!({
17523 "type": "start_service_operation",
17524 "payload_codec": DEFAULT_CODEC,
17525 "request_payload": "raw-avro-bytes"
17526 }),
17527 ];
17528
17529 for command in invalid_commands {
17530 let error = client
17531 .complete_workflow_task("invalid-codec", "rust-worker", 1, vec![command])
17532 .await
17533 .expect_err("invalid durable payload must fail locally");
17534 let diagnostic = error.to_string();
17535 assert!(
17536 diagnostic.contains("unsupported_payload_codec")
17537 || diagnostic.contains("invalid_payload_envelope")
17538 || diagnostic.contains("untagged durable payload"),
17539 "unexpected validation diagnostic: {diagnostic}"
17540 );
17541 }
17542
17543 assert_eq!(
17544 server.request_count("/api/worker/workflow-tasks/invalid-codec/complete"),
17545 0,
17546 "invalid command payloads must not reach HTTP transport"
17547 );
17548 }
17549
17550 #[test]
17551 fn workflow_completion_validates_only_protocol_owned_payload_slots() {
17552 let envelope = fixture_envelope(json!({"codec": "customer-value"}));
17553 let commands = [
17554 json!({"type": "complete_workflow", "result": envelope.clone()}),
17555 json!({"type": "schedule_activity", "arguments": envelope.clone()}),
17556 json!({"type": "start_child_workflow", "arguments": envelope.clone()}),
17557 json!({"type": "continue_as_new", "arguments": envelope.clone()}),
17558 json!({"type": "complete_update", "result": envelope.clone()}),
17559 json!({"type": "record_side_effect", "result": envelope.clone()}),
17560 json!({
17561 "type": "start_service_operation",
17562 "payload_codec": DEFAULT_CODEC,
17563 "request_payload": envelope.clone()
17564 }),
17565 json!({
17566 "type": "complete_workflow",
17567 "result": envelope,
17568 "metadata": {
17569 "codec": "json",
17570 "payload_codec": "customer-codec",
17571 "result": {"codec": "yaml", "blob": null}
17572 }
17573 }),
17574 ];
17575
17576 validate_workflow_task_commands(&commands)
17577 .expect("customer metadata must not become a protocol codec declaration");
17578 }
17579
17580 #[test]
17581 fn valid_avro_tasks_normalize_absent_and_null_arguments_to_empty_lists() {
17582 assert_eq!(
17583 decode_task_avro_arguments(None, DEFAULT_CODEC).expect("absent arguments"),
17584 AvroValue::Array(Vec::new())
17585 );
17586 assert_eq!(
17587 decode_task_avro_arguments(Some(&Value::Null), DEFAULT_CODEC).expect("null arguments"),
17588 AvroValue::Array(Vec::new())
17589 );
17590
17591 let mut signal = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
17592 signal.signal_name = Some("empty-signal".to_string());
17593 signal.signal_arguments = None;
17594 let decoded = decode_resume_signal(&signal)
17595 .expect("valid Avro signal")
17596 .expect("named signal resumes the workflow");
17597 assert!(decoded.arguments.is_empty());
17598 }
17599
17600 #[tokio::test]
17601 async fn malformed_task_level_codecs_become_pre_handler_failures() {
17602 let client = Client::new("http://127.0.0.1:8080").expect("client");
17603 let mut worker = Worker::new(client, "rust-workers");
17604 let handler_calls = Arc::new(AtomicUsize::new(0));
17605
17606 let calls = Arc::clone(&handler_calls);
17607 worker.register_workflow("codec.workflow", move |_ctx, _args| {
17608 calls.fetch_add(1, Ordering::SeqCst);
17609 async move { Ok(Value::Null) }
17610 });
17611 let calls = Arc::clone(&handler_calls);
17612 worker.register_activity("codec.activity", move |_ctx, _args| {
17613 calls.fetch_add(1, Ordering::SeqCst);
17614 async move { Ok(Value::Null) }
17615 });
17616 let calls = Arc::clone(&handler_calls);
17617 worker.register_query("codec.workflow", "known", move |_ctx, _args| {
17618 calls.fetch_add(1, Ordering::SeqCst);
17619 async move { Ok(Value::Null) }
17620 });
17621
17622 let mut failures = Vec::new();
17623 for codec_case in [
17624 InvalidTaskPayloadCodec::Missing,
17625 InvalidTaskPayloadCodec::Null,
17626 InvalidTaskPayloadCodec::NonString,
17627 ] {
17628 let mut workflow = json!({
17629 "task_id": format!("workflow-{}", codec_case.label()),
17630 "workflow_type": "codec.workflow"
17631 });
17632 codec_case.apply(&mut workflow);
17633 match serde_json::from_value::<WorkflowTask>(workflow) {
17634 Ok(task) => match worker.execute_workflow_task(task) {
17635 Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
17636 outcome => failures.push(format!(
17637 "workflow {} codec returned {outcome:?}",
17638 codec_case.label()
17639 )),
17640 },
17641 Err(error) => failures.push(format!(
17642 "workflow {} codec failed transport deserialization: {error}",
17643 codec_case.label()
17644 )),
17645 }
17646
17647 let mut activity = json!({
17648 "task_id": format!("activity-{}", codec_case.label()),
17649 "activity_attempt_id": format!("attempt-{}", codec_case.label()),
17650 "activity_type": "codec.activity",
17651 "attempt_number": 1
17652 });
17653 codec_case.apply(&mut activity);
17654 match serde_json::from_value::<ActivityTask>(activity) {
17655 Ok(task) => match worker.execute_activity_task(task).await {
17656 Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
17657 outcome => failures.push(format!(
17658 "activity {} codec returned {outcome:?}",
17659 codec_case.label()
17660 )),
17661 },
17662 Err(error) => failures.push(format!(
17663 "activity {} codec failed transport deserialization: {error}",
17664 codec_case.label()
17665 )),
17666 }
17667
17668 let mut query = json!({
17669 "query_task_id": format!("query-{}", codec_case.label()),
17670 "workflow_type": "codec.workflow",
17671 "query_name": "known"
17672 });
17673 codec_case.apply(&mut query);
17674 match serde_json::from_value::<QueryTask>(query) {
17675 Ok(task) => match worker.execute_query_task(task).await {
17676 Err(failure) if failure.message.contains("unsupported_payload_codec") => {}
17677 outcome => failures.push(format!(
17678 "query {} codec returned {outcome:?}",
17679 codec_case.label()
17680 )),
17681 },
17682 Err(error) => failures.push(format!(
17683 "query {} codec failed transport deserialization: {error}",
17684 codec_case.label()
17685 )),
17686 }
17687 }
17688
17689 assert!(failures.is_empty(), "{}", failures.join("\n"));
17690 assert_eq!(
17691 handler_calls.load(Ordering::SeqCst),
17692 0,
17693 "invalid task codecs must not invoke a handler"
17694 );
17695 }
17696
17697 #[tokio::test]
17698 async fn polled_malformed_task_codecs_are_settled_without_handler_execution() {
17699 for codec_case in [
17700 InvalidTaskPayloadCodec::Missing,
17701 InvalidTaskPayloadCodec::Null,
17702 InvalidTaskPayloadCodec::NonString,
17703 ] {
17704 let server = MockWorkerServer::invalid_task_payload_codec(codec_case);
17705 let client = Client::builder(server.base_url())
17706 .timeout(Duration::from_secs(2))
17707 .build()
17708 .expect("client");
17709 let mut worker = Worker::new(client, "rust-workers")
17710 .worker_id("codec-worker")
17711 .poll_timeout(Duration::from_millis(10));
17712 let handler_calls = Arc::new(AtomicUsize::new(0));
17713
17714 let calls = Arc::clone(&handler_calls);
17715 worker.register_workflow("codec.workflow", move |_ctx, _args| {
17716 calls.fetch_add(1, Ordering::SeqCst);
17717 async move { Ok(Value::Null) }
17718 });
17719 let calls = Arc::clone(&handler_calls);
17720 worker.register_activity("codec.activity", move |_ctx, _args| {
17721 calls.fetch_add(1, Ordering::SeqCst);
17722 async move { Ok(Value::Null) }
17723 });
17724 let calls = Arc::clone(&handler_calls);
17725 worker.register_query("codec.workflow", "known", move |_ctx, _args| {
17726 calls.fetch_add(1, Ordering::SeqCst);
17727 async move { Ok(Value::Null) }
17728 });
17729
17730 assert_eq!(
17731 worker.run_once().await.expect("invalid tasks are settled"),
17732 3,
17733 "all {} codec tasks must be handled",
17734 codec_case.label()
17735 );
17736 assert_eq!(
17737 handler_calls.load(Ordering::SeqCst),
17738 0,
17739 "{} task codecs must fail before every handler",
17740 codec_case.label()
17741 );
17742
17743 for path in [
17744 "/api/worker/workflow-tasks/codec-workflow/fail",
17745 "/api/worker/activity-tasks/codec-activity/fail",
17746 "/api/worker/query-tasks/codec-query/fail",
17747 ] {
17748 let body = server.request_body(path);
17749 assert!(
17750 body["failure"]["message"]
17751 .as_str()
17752 .is_some_and(|message| message.contains("unsupported_payload_codec")),
17753 "{path} must receive the stable codec diagnostic for the {} case: {body}",
17754 codec_case.label()
17755 );
17756 }
17757 assert_eq!(
17758 server.request_body("/api/worker/query-tasks/codec-query/fail")["failure"]
17759 ["reason"],
17760 "query_payload_decode_failed"
17761 );
17762 for path in [
17763 "/api/worker/workflow-tasks/codec-workflow/complete",
17764 "/api/worker/activity-tasks/codec-activity/complete",
17765 "/api/worker/query-tasks/codec-query/complete",
17766 ] {
17767 assert_eq!(
17768 server.request_count(path),
17769 0,
17770 "invalid {} codec task reached {path}",
17771 codec_case.label()
17772 );
17773 }
17774 }
17775 }
17776
17777 #[tokio::test]
17778 async fn invalid_inbound_codecs_precede_handlers_and_unrelated_outcomes() {
17779 let client = Client::new("http://127.0.0.1:8080").expect("client");
17780 let mut worker = Worker::new(client, "rust-workers");
17781 let handler_calls = Arc::new(AtomicUsize::new(0));
17782
17783 let calls = Arc::clone(&handler_calls);
17784 worker.register_workflow("codec.workflow", move |_ctx, _args| {
17785 calls.fetch_add(1, Ordering::SeqCst);
17786 async move { Ok(Value::Null) }
17787 });
17788 let calls = Arc::clone(&handler_calls);
17789 worker.register_activity("codec.activity", move |_ctx, _args| {
17790 calls.fetch_add(1, Ordering::SeqCst);
17791 async move { Ok(Value::Null) }
17792 });
17793 let calls = Arc::clone(&handler_calls);
17794 worker.register_update("codec.workflow", "known", move |_ctx, _args| {
17795 calls.fetch_add(1, Ordering::SeqCst);
17796 async move { Ok(Value::Null) }
17797 });
17798 let calls = Arc::clone(&handler_calls);
17799 worker.register_query("codec.workflow", "known", move |_ctx, _args| {
17800 calls.fetch_add(1, Ordering::SeqCst);
17801 async move { Ok(Value::Null) }
17802 });
17803
17804 let mut workflow = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
17805 workflow.payload_codec = "json".to_string();
17806 workflow.arguments = None;
17807 let error = worker
17808 .execute_workflow_task(workflow)
17809 .expect_err("task codec must be checked before workflow invocation");
17810 assert!(error.to_string().contains("unsupported_payload_codec"));
17811
17812 let activity = ActivityTask {
17813 task_id: "activity-invalid-codec".to_string(),
17814 activity_attempt_id: None,
17815 attempt_id: None,
17816 activity_type: "codec.activity".to_string(),
17817 payload_codec: "unknown".to_string(),
17818 arguments: None,
17819 attempt_number: 1,
17820 lease_owner: None,
17821 };
17822 let error = worker
17823 .execute_activity_task(activity)
17824 .await
17825 .expect_err("task codec must be checked before activity invocation");
17826 assert!(error.to_string().contains("unsupported_payload_codec"));
17827
17828 let mut update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
17829 update.workflow_update_id = Some("update-invalid-codec".to_string());
17830 update.update_name = Some("known".to_string());
17831 update.history_events.push(history_event(
17832 "UpdateAccepted",
17833 json!({
17834 "update_id": "update-invalid-codec",
17835 "update_name": "known",
17836 "arguments": {"codec": "json", "blob": null}
17837 }),
17838 ));
17839 let error = worker
17840 .execute_workflow_task(update)
17841 .expect_err("nested update codec must be checked before handler lookup");
17842 assert!(error.to_string().contains("unsupported_payload_codec"));
17843
17844 let query: QueryTask = serde_json::from_value(json!({
17845 "query_task_id": "query-invalid-codec",
17846 "workflow_type": "codec.workflow",
17847 "query_name": "known",
17848 "payload_codec": DEFAULT_CODEC,
17849 "workflow_arguments": null,
17850 "query_arguments": null,
17851 "history_export": {
17852 "payloads": {"codec": DEFAULT_CODEC},
17853 "signals": [{
17854 "name": "empty",
17855 "payload_codec": "json",
17856 "arguments": null
17857 }]
17858 }
17859 }))
17860 .expect("query task");
17861 let failure = worker
17862 .execute_query_task(query)
17863 .await
17864 .expect_err("exported signal codec must be checked before query invocation");
17865 assert_eq!(failure.reason, "query_payload_decode_failed");
17866 assert!(failure.message.contains("unsupported_payload_codec"));
17867
17868 let exported_history: QueryTask = serde_json::from_value(json!({
17869 "query_task_id": "query-invalid-history-codec",
17870 "workflow_type": "codec.workflow",
17871 "query_name": "known",
17872 "payload_codec": DEFAULT_CODEC,
17873 "history_export": {
17874 "payloads": {"codec": DEFAULT_CODEC},
17875 "history_events": [{
17876 "type": "ActivityCompleted",
17877 "payload": {"payload_codec": "unknown", "result": null}
17878 }]
17879 }
17880 }))
17881 .expect("query task");
17882 let failure = worker
17883 .execute_query_task(exported_history)
17884 .await
17885 .expect_err("exported history codec must be checked before query invocation");
17886 assert_eq!(failure.reason, "query_payload_decode_failed");
17887 assert!(failure.message.contains("unsupported_payload_codec"));
17888 assert_eq!(handler_calls.load(Ordering::SeqCst), 0);
17889
17890 let mut unknown_workflow = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
17891 unknown_workflow.arguments = None;
17892 unknown_workflow.history_events.push(history_event(
17893 "SignalReceived",
17894 json!({
17895 "signal_name": "empty",
17896 "payload_codec": "json",
17897 "arguments": null
17898 }),
17899 ));
17900 let error = worker
17901 .execute_workflow_task(unknown_workflow)
17902 .expect_err("history codec must precede unknown workflow outcome");
17903 assert!(error.to_string().contains("unsupported_payload_codec"));
17904
17905 let unknown_activity = ActivityTask {
17906 task_id: "activity-unknown".to_string(),
17907 activity_attempt_id: None,
17908 attempt_id: None,
17909 activity_type: "missing".to_string(),
17910 payload_codec: "json".to_string(),
17911 arguments: None,
17912 attempt_number: 1,
17913 lease_owner: None,
17914 };
17915 let error = worker
17916 .execute_activity_task(unknown_activity)
17917 .await
17918 .expect_err("codec must precede unknown activity outcome");
17919 assert!(error.to_string().contains("unsupported_payload_codec"));
17920
17921 let mut unknown_update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
17922 unknown_update.payload_codec = "json".to_string();
17923 unknown_update.arguments = None;
17924 unknown_update.workflow_update_id = Some("update-unknown".to_string());
17925 unknown_update.update_name = Some("missing".to_string());
17926 let error = worker
17927 .execute_workflow_task(unknown_update)
17928 .expect_err("codec must precede fail_update shortcut");
17929 assert!(error.to_string().contains("unsupported_payload_codec"));
17930
17931 let unknown_query: QueryTask = serde_json::from_value(json!({
17932 "query_task_id": "query-unknown",
17933 "workflow_type": "missing",
17934 "query_name": "missing",
17935 "payload_codec": "json",
17936 "workflow_arguments": null,
17937 "query_arguments": null
17938 }))
17939 .expect("query task");
17940 let failure = worker
17941 .execute_query_task(unknown_query)
17942 .await
17943 .expect_err("codec must precede unknown query outcome");
17944 assert_eq!(failure.reason, "query_payload_decode_failed");
17945 assert!(failure.message.contains("unsupported_payload_codec"));
17946 }
17947
17948 #[tokio::test]
17949 async fn invalid_signal_history_payload_aliases_precede_shortcuts() {
17950 let client = Client::new("http://127.0.0.1:8080").expect("client");
17951 let worker = Worker::new(client, "rust-workers");
17952
17953 for event_type in ["SignalReceived", "SignalApplied"] {
17954 for (payload_field, codec) in [
17955 ("value", "json"),
17956 ("input", "unknown"),
17957 ("arguments", "json"),
17958 ] {
17959 let payload = json!({
17960 "signal_name": "empty",
17961 payload_field: {"codec": codec, "blob": null}
17962 });
17963 let workflow = workflow_task(
17964 "missing",
17965 vec![history_event(event_type, payload.clone())],
17966 DEFAULT_CODEC,
17967 );
17968 let error = worker
17969 .execute_workflow_task(workflow)
17970 .expect_err("signal payload codec must precede unknown workflow outcome");
17971 assert!(
17972 error.to_string().contains("unsupported_payload_codec"),
17973 "{event_type}.{payload_field} returned an unrelated workflow error: {error}"
17974 );
17975
17976 let query: QueryTask = serde_json::from_value(json!({
17977 "query_task_id": format!("query-{event_type}-{payload_field}"),
17978 "workflow_type": "missing",
17979 "query_name": "missing",
17980 "payload_codec": DEFAULT_CODEC,
17981 "workflow_arguments": null,
17982 "query_arguments": null,
17983 "history_events": [{
17984 "event_type": event_type,
17985 "payload": payload
17986 }]
17987 }))
17988 .expect("query task");
17989 let failure = worker
17990 .execute_query_task(query)
17991 .await
17992 .expect_err("signal payload codec must precede unknown query outcome");
17993 assert_eq!(
17994 failure.reason, "query_payload_decode_failed",
17995 "{event_type}.{payload_field} returned an unrelated query outcome"
17996 );
17997 assert!(
17998 failure.message.contains("unsupported_payload_codec"),
17999 "{event_type}.{payload_field} returned an unrelated query error: {}",
18000 failure.message
18001 );
18002 }
18003 }
18004 }
18005
18006 #[test]
18007 fn workflow_context_schedules_activity_until_completion_is_in_history() {
18008 let ctx = WorkflowContext {
18009 state: Arc::new(Mutex::new(
18010 WorkflowState::new_with_identity(
18011 Vec::new(),
18012 Some("wf-parent".to_string()),
18013 Some("run-parent".to_string()),
18014 "rust-workers".to_string(),
18015 DEFAULT_CODEC.to_string(),
18016 None,
18017 )
18018 .expect("workflow state"),
18019 )),
18020 };
18021
18022 let mut call = Box::pin(ctx.activity("hello.activity", json!(["Ada"])));
18023 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18024 assert!(matches!(
18025 call.as_mut().poll(&mut task_context),
18026 Poll::Pending
18027 ));
18028
18029 let commands = ctx.take_commands().expect("commands");
18030 assert_eq!(commands[0]["type"], "schedule_activity");
18031 assert_eq!(commands[0]["activity_type"], "hello.activity");
18032 }
18033
18034 #[test]
18035 fn activity_options_encode_retry_policy_queue_and_every_timeout() {
18036 let ctx = workflow_context(Vec::new());
18037 let options = ActivityOptions::new()
18038 .task_queue("payments")
18039 .retry_policy(
18040 ActivityRetryPolicy::new(4)
18041 .exponential_backoff(Duration::from_secs(1), 3, Some(Duration::from_secs(10)))
18042 .non_retryable_error_type("ValidationError"),
18043 )
18044 .start_to_close_timeout(Duration::from_secs(120))
18045 .schedule_to_start_timeout(Duration::from_secs(10))
18046 .schedule_to_close_timeout(Duration::from_secs(300))
18047 .heartbeat_timeout(Duration::from_secs(15));
18048 let mut call = Box::pin(ctx.activity_with_options(
18049 "charge-card",
18050 options,
18051 json!([{"order_id": "o-1"}]),
18052 ));
18053 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18054
18055 assert!(matches!(
18056 call.as_mut().poll(&mut task_context),
18057 Poll::Pending
18058 ));
18059 assert!(matches!(
18060 call.as_mut().poll(&mut task_context),
18061 Poll::Pending
18062 ));
18063
18064 let commands = ctx.take_commands().expect("activity command");
18065 assert_eq!(commands.len(), 1, "one future emits one logical schedule");
18066 assert_eq!(commands[0]["queue"], "payments");
18067 assert_eq!(
18068 commands[0]["retry_policy"],
18069 json!({
18070 "max_attempts": 4,
18071 "backoff_seconds": [1, 3, 9],
18072 "non_retryable_error_types": ["ValidationError"],
18073 })
18074 );
18075 assert_eq!(commands[0]["start_to_close_timeout"], 120);
18076 assert_eq!(commands[0]["schedule_to_start_timeout"], 10);
18077 assert_eq!(commands[0]["schedule_to_close_timeout"], 300);
18078 assert_eq!(commands[0]["heartbeat_timeout"], 15);
18079 }
18080
18081 #[test]
18082 fn activity_options_encode_explicit_and_rounded_backoff_intervals() {
18083 let ctx = workflow_context(Vec::new());
18084 let options = ActivityOptions::new().retry_policy(
18085 ActivityRetryPolicy::new(3)
18086 .backoff_intervals([Duration::from_millis(1), Duration::from_millis(1_001)]),
18087 );
18088 let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
18089 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18090
18091 assert!(matches!(
18092 call.as_mut().poll(&mut task_context),
18093 Poll::Pending
18094 ));
18095 assert_eq!(
18096 ctx.take_commands().expect("command")[0]["retry_policy"]["backoff_seconds"],
18097 json!([1, 2])
18098 );
18099 }
18100
18101 #[test]
18102 fn invalid_activity_options_return_typed_errors_before_emitting_commands() {
18103 let cases = [
18104 (
18105 ActivityOptions::new().task_queue(" "),
18106 ActivityOptionsErrorKind::EmptyTaskQueue,
18107 ),
18108 (
18109 ActivityOptions::new().retry_policy(ActivityRetryPolicy::default()),
18110 ActivityOptionsErrorKind::EmptyRetryPolicy,
18111 ),
18112 (
18113 ActivityOptions::new().retry_policy(ActivityRetryPolicy::new(0)),
18114 ActivityOptionsErrorKind::InvalidMaxAttempts,
18115 ),
18116 (
18117 ActivityOptions::new().retry_policy(ActivityRetryPolicy {
18118 max_attempts: None,
18119 backoff: Some(ActivityBackoff::Explicit(vec![Duration::from_secs(1)])),
18120 non_retryable_error_types: Vec::new(),
18121 }),
18122 ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
18123 ),
18124 (
18125 ActivityOptions::new().retry_policy(
18126 ActivityRetryPolicy::new(2)
18127 .backoff_intervals([Duration::from_secs(1), Duration::from_secs(2)]),
18128 ),
18129 ActivityOptionsErrorKind::TooManyBackoffIntervals,
18130 ),
18131 (
18132 ActivityOptions::new().retry_policy(
18133 ActivityRetryPolicy::new(2).exponential_backoff(
18134 Duration::from_secs(1),
18135 0,
18136 None,
18137 ),
18138 ),
18139 ActivityOptionsErrorKind::InvalidBackoffCoefficient,
18140 ),
18141 (
18142 ActivityOptions::new()
18143 .retry_policy(ActivityRetryPolicy::new(2).non_retryable_error_type(" ")),
18144 ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
18145 ),
18146 (
18147 ActivityOptions::new().retry_policy(
18148 ActivityRetryPolicy::new(10_002).exponential_backoff(
18149 Duration::from_secs(1),
18150 1,
18151 None,
18152 ),
18153 ),
18154 ActivityOptionsErrorKind::BackoffGenerationTooLarge,
18155 ),
18156 (
18157 ActivityOptions::new().retry_policy(
18158 ActivityRetryPolicy::new(2)
18159 .backoff_intervals([Duration::from_secs(i64::MAX as u64 + 1)]),
18160 ),
18161 ActivityOptionsErrorKind::BackoffOverflow,
18162 ),
18163 ];
18164
18165 for (options, expected_kind) in cases {
18166 let ctx = workflow_context(Vec::new());
18167 let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
18168 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18169 let Poll::Ready(Err(Error::InvalidActivityOptions(error))) =
18170 call.as_mut().poll(&mut task_context)
18171 else {
18172 panic!("expected typed activity validation error");
18173 };
18174 assert_eq!(error.kind, expected_kind);
18175 assert!(ctx.take_commands().expect("commands").is_empty());
18176 }
18177 }
18178
18179 #[test]
18180 fn activity_options_validate_positive_and_ordered_timeouts() {
18181 let zero_timeout_cases = [
18182 ActivityOptions::new().start_to_close_timeout(Duration::ZERO),
18183 ActivityOptions::new().schedule_to_start_timeout(Duration::ZERO),
18184 ActivityOptions::new().schedule_to_close_timeout(Duration::ZERO),
18185 ActivityOptions::new().heartbeat_timeout(Duration::ZERO),
18186 ];
18187 for options in zero_timeout_cases {
18188 assert_eq!(
18189 options.validate().expect_err("zero timeout").kind,
18190 ActivityOptionsErrorKind::TimeoutNotPositive
18191 );
18192 }
18193
18194 let ordering_cases = [
18195 ActivityOptions::new()
18196 .heartbeat_timeout(Duration::from_secs(11))
18197 .start_to_close_timeout(Duration::from_secs(10)),
18198 ActivityOptions::new()
18199 .start_to_close_timeout(Duration::from_secs(31))
18200 .schedule_to_close_timeout(Duration::from_secs(30)),
18201 ActivityOptions::new()
18202 .schedule_to_start_timeout(Duration::from_secs(31))
18203 .schedule_to_close_timeout(Duration::from_secs(30)),
18204 ];
18205 for options in ordering_cases {
18206 assert_eq!(
18207 options.validate().expect_err("timeout order").kind,
18208 ActivityOptionsErrorKind::TimeoutOrder
18209 );
18210 }
18211
18212 assert_eq!(
18213 ActivityOptions::new()
18214 .start_to_close_timeout(Duration::from_secs(i64::MAX as u64 + 1))
18215 .validate()
18216 .expect_err("protocol integer overflow")
18217 .kind,
18218 ActivityOptionsErrorKind::TimeoutOverflow
18219 );
18220 }
18221
18222 #[test]
18223 fn replayed_activity_retry_history_completes_without_duplicate_schedule() {
18224 let ctx = workflow_context(completed_retry_activity_history());
18225 let mut call =
18226 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18227 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18228
18229 assert!(matches!(
18230 call.as_mut().poll(&mut task_context),
18231 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18232 ));
18233 assert!(ctx.take_commands().expect("commands").is_empty());
18234 ctx.ensure_history_consumed().expect("history consumed");
18235 }
18236
18237 #[test]
18238 fn duplicate_non_retryable_types_use_one_command_and_replay_representation() {
18239 let mut options = retry_activity_options();
18240 options
18241 .retry_policy
18242 .as_mut()
18243 .expect("retry policy")
18244 .non_retryable_error_types
18245 .extend([" PermanentError ".to_string(), "PermanentError".to_string()]);
18246
18247 let new_ctx = workflow_context(Vec::new());
18248 let mut new_call =
18249 Box::pin(new_ctx.activity_with_options("flaky", options.clone(), json!([])));
18250 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18251 assert!(matches!(
18252 new_call.as_mut().poll(&mut task_context),
18253 Poll::Pending
18254 ));
18255 let commands = new_ctx.take_commands().expect("commands");
18256 assert_eq!(commands.len(), 1);
18257 assert_eq!(
18258 commands[0]["retry_policy"]["non_retryable_error_types"],
18259 json!(["PermanentError"])
18260 );
18261
18262 let replay_ctx = workflow_context(completed_retry_activity_history());
18263 let mut replay_call =
18264 Box::pin(replay_ctx.activity_with_options("flaky", options, json!([])));
18265 assert!(matches!(
18266 replay_call.as_mut().poll(&mut task_context),
18267 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18268 ));
18269 assert!(replay_ctx.take_commands().expect("commands").is_empty());
18270 replay_ctx
18271 .ensure_history_consumed()
18272 .expect("history consumed");
18273 }
18274
18275 #[test]
18276 fn replayed_intermediate_retry_remains_pending_across_restarts() {
18277 let history = completed_retry_activity_history()
18278 .into_iter()
18279 .take(3)
18280 .collect::<Vec<_>>();
18281
18282 for _restart in 0..2 {
18283 let ctx = workflow_context(history.clone());
18284 let mut call =
18285 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18286 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18287 assert!(matches!(
18288 call.as_mut().poll(&mut task_context),
18289 Poll::Pending
18290 ));
18291 assert!(ctx.take_commands().expect("commands").is_empty());
18292 }
18293 }
18294
18295 #[test]
18296 fn replayed_activity_rejects_changed_queue_retry_and_every_timeout_field() {
18297 let mut changed_queue = retry_activity_options();
18298 changed_queue.task_queue = Some("different-queue".to_string());
18299
18300 let mut changed_max_attempts = retry_activity_options();
18301 let retry_policy = changed_max_attempts
18302 .retry_policy
18303 .as_mut()
18304 .expect("retry policy");
18305 retry_policy.max_attempts = Some(4);
18306
18307 let mut changed_backoff = retry_activity_options();
18308 let retry_policy = changed_backoff.retry_policy.as_mut().expect("retry policy");
18309 retry_policy.backoff = Some(ActivityBackoff::Explicit(vec![
18310 Duration::from_secs(3),
18311 Duration::from_secs(4),
18312 ]));
18313
18314 let mut changed_non_retryable_types = retry_activity_options();
18315 let retry_policy = changed_non_retryable_types
18316 .retry_policy
18317 .as_mut()
18318 .expect("retry policy");
18319 retry_policy.non_retryable_error_types = vec!["AnotherPermanentError".to_string()];
18320
18321 let mut changed_start_to_close = retry_activity_options();
18322 changed_start_to_close.start_to_close_timeout = Some(Duration::from_secs(31));
18323 let mut changed_schedule_to_start = retry_activity_options();
18324 changed_schedule_to_start.schedule_to_start_timeout = Some(Duration::from_secs(6));
18325 let mut changed_schedule_to_close = retry_activity_options();
18326 changed_schedule_to_close.schedule_to_close_timeout = Some(Duration::from_secs(91));
18327 let mut changed_heartbeat = retry_activity_options();
18328 changed_heartbeat.heartbeat_timeout = Some(Duration::from_secs(11));
18329
18330 let cases = [
18331 (changed_queue, "activity_task_queue_mismatch"),
18332 (changed_max_attempts, "activity_retry_policy_mismatch"),
18333 (changed_backoff, "activity_retry_policy_mismatch"),
18334 (
18335 changed_non_retryable_types,
18336 "activity_retry_policy_mismatch",
18337 ),
18338 (changed_start_to_close, "activity_retry_policy_mismatch"),
18339 (changed_schedule_to_start, "activity_retry_policy_mismatch"),
18340 (changed_schedule_to_close, "activity_retry_policy_mismatch"),
18341 (changed_heartbeat, "activity_retry_policy_mismatch"),
18342 ];
18343
18344 for (options, expected_reason) in cases {
18345 let ctx = workflow_context(completed_retry_activity_history());
18346 let mut call = Box::pin(ctx.activity_with_options("flaky", options, json!([])));
18347 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18348 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18349 call.as_mut().poll(&mut task_context)
18350 else {
18351 panic!("changed activity options must fail replay");
18352 };
18353 assert_eq!(failure.reason, expected_reason);
18354 assert_eq!(failure.sequence, Some(1));
18355 assert!(ctx.take_commands().expect("commands").is_empty());
18356 }
18357 }
18358
18359 #[test]
18360 fn replayed_activity_rejects_changed_execution_mode_and_snapshot_version() {
18361 let cases = [
18362 (
18363 "execution_mode",
18364 json!("local"),
18365 "activity_execution_mode_mismatch",
18366 ),
18367 (
18368 "snapshot_version",
18369 json!(2),
18370 "activity_retry_policy_mismatch",
18371 ),
18372 ];
18373
18374 for (field, value, expected_reason) in cases {
18375 let mut history = completed_retry_activity_history();
18376 let activity = history[0].payload["activity"]
18377 .as_object_mut()
18378 .expect("activity snapshot");
18379 if field == "execution_mode" {
18380 activity.insert(field.to_string(), value);
18381 } else {
18382 activity["retry_policy"]
18383 .as_object_mut()
18384 .expect("retry snapshot")
18385 .insert(field.to_string(), value);
18386 }
18387
18388 let ctx = workflow_context(history);
18389 let mut call =
18390 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18391 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18392 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18393 call.as_mut().poll(&mut task_context)
18394 else {
18395 panic!("changed {field} must fail replay");
18396 };
18397 assert_eq!(failure.reason, expected_reason);
18398 assert_eq!(failure.sequence, Some(1));
18399 assert!(ctx.take_commands().expect("commands").is_empty());
18400 }
18401 }
18402
18403 #[test]
18404 fn replayed_legacy_activity_treats_missing_option_snapshot_as_unknown() {
18405 let mut history = completed_retry_activity_history();
18406 let activity = history[0].payload["activity"]
18407 .as_object_mut()
18408 .expect("activity snapshot");
18409 activity.remove("execution_mode");
18410 activity.remove("retry_policy");
18411
18412 let mut current = retry_activity_options();
18413 current.start_to_close_timeout = Some(Duration::from_secs(45));
18414 current.schedule_to_start_timeout = Some(Duration::from_secs(8));
18415 current.schedule_to_close_timeout = Some(Duration::from_secs(120));
18416 current.heartbeat_timeout = Some(Duration::from_secs(12));
18417
18418 let ctx = workflow_context(history);
18419 let mut call = Box::pin(ctx.activity_with_options("flaky", current, json!([])));
18420 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18421 assert!(matches!(
18422 call.as_mut().poll(&mut task_context),
18423 Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18424 ));
18425 assert!(ctx.take_commands().expect("commands").is_empty());
18426 ctx.ensure_history_consumed().expect("history consumed");
18427 }
18428
18429 #[test]
18430 fn terminal_activity_failed_after_start_returns_typed_failure() {
18431 let history = vec![
18432 history_event(
18433 "ActivityScheduled",
18434 json!({
18435 "sequence": 1,
18436 "activity_type": "flaky",
18437 "activity_execution_id": "act-terminal",
18438 "activity": {
18439 "id": "act-terminal",
18440 "sequence": 1,
18441 "type": "flaky",
18442 "queue": "critical-activities",
18443 "retry_policy": {
18444 "snapshot_version": 1,
18445 "max_attempts": 3,
18446 "backoff_seconds": [2, 4],
18447 "non_retryable_error_types": ["PermanentError"]
18448 }
18449 }
18450 }),
18451 ),
18452 history_event(
18453 "ActivityStarted",
18454 json!({
18455 "sequence": 1,
18456 "activity_type": "flaky",
18457 "activity_execution_id": "act-terminal",
18458 "activity_attempt_id": "attempt-1",
18459 "attempt_number": 1
18460 }),
18461 ),
18462 history_event(
18463 "ActivityFailed",
18464 json!({
18465 "sequence": 1,
18466 "activity_type": "flaky",
18467 "activity_execution_id": "act-terminal",
18468 "activity_attempt_id": "attempt-1",
18469 "attempt_number": 1,
18470 "failure_id": "failure-terminal",
18471 "failure_category": "activity",
18472 "exception_type": "PermanentError",
18473 "message": "cannot retry",
18474 "non_retryable": true
18475 }),
18476 ),
18477 ];
18478 let ctx = workflow_context(history);
18479 let mut call =
18480 Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18481 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18482
18483 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18484 call.as_mut().poll(&mut task_context)
18485 else {
18486 panic!("terminal ActivityFailed must settle the activity future");
18487 };
18488 assert_eq!(failure.kind, ActivityFailureKind::Failed);
18489 assert_eq!(
18490 failure.activity_execution_id.as_deref(),
18491 Some("act-terminal")
18492 );
18493 assert_eq!(failure.exception_type.as_deref(), Some("PermanentError"));
18494 assert!(failure.non_retryable);
18495 assert!(ctx.take_commands().expect("commands").is_empty());
18496 ctx.ensure_history_consumed().expect("history consumed");
18497 }
18498
18499 #[test]
18500 fn activity_terminal_events_return_machine_readable_failures() {
18501 let cases = [
18502 (
18503 "ActivityFailed",
18504 json!({
18505 "sequence": 1,
18506 "activity_type": "charge-card",
18507 "activity_execution_id": "act-1",
18508 "activity_attempt_id": "attempt-2",
18509 "attempt_number": 2,
18510 "failure_id": "failure-1",
18511 "failure_category": "activity",
18512 "exception_type": "PaymentDeclined",
18513 "exception_class": "payments.PaymentDeclined",
18514 "message": "card declined",
18515 "non_retryable": true
18516 }),
18517 ActivityFailureKind::Failed,
18518 "activity",
18519 ),
18520 (
18521 "ActivityCancelled",
18522 json!({
18523 "sequence": 1,
18524 "activity_type": "charge-card",
18525 "activity_execution_id": "act-1",
18526 "activity_attempt_id": "attempt-1"
18527 }),
18528 ActivityFailureKind::Cancelled,
18529 "cancelled",
18530 ),
18531 ];
18532
18533 for (event_type, payload, expected_kind, expected_reason) in cases {
18534 let ctx = workflow_context(vec![history_event(event_type, payload)]);
18535 let mut call = Box::pin(ctx.activity("charge-card", json!([])));
18536 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18537 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18538 call.as_mut().poll(&mut task_context)
18539 else {
18540 panic!("expected terminal activity failure");
18541 };
18542 assert_eq!(failure.kind, expected_kind);
18543 assert_eq!(failure.reason, expected_reason);
18544 assert_eq!(failure.activity_execution_id.as_deref(), Some("act-1"));
18545 assert_eq!(failure.activity_type.as_deref(), Some("charge-card"));
18546 }
18547 }
18548
18549 #[test]
18550 fn every_activity_timeout_class_is_typed() {
18551 for timeout_kind in [
18552 "start_to_close",
18553 "schedule_to_start",
18554 "schedule_to_close",
18555 "heartbeat",
18556 ] {
18557 let ctx = workflow_context(vec![history_event(
18558 "ActivityTimedOut",
18559 json!({
18560 "sequence": 1,
18561 "activity_type": "slow",
18562 "activity_execution_id": "act-timeout",
18563 "activity_attempt_id": "attempt-timeout",
18564 "failure_category": "timeout",
18565 "timeout_kind": timeout_kind,
18566 "message": "deadline expired"
18567 }),
18568 )]);
18569 let mut call = Box::pin(ctx.activity("slow", json!([])));
18570 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18571 let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18572 call.as_mut().poll(&mut task_context)
18573 else {
18574 panic!("expected timeout failure");
18575 };
18576 assert_eq!(failure.kind, ActivityFailureKind::TimedOut);
18577 assert_eq!(failure.reason, timeout_kind);
18578 assert_eq!(failure.timeout_kind.as_deref(), Some(timeout_kind));
18579 assert_eq!(failure.failure_category.as_deref(), Some("timeout"));
18580 }
18581 }
18582
18583 #[test]
18584 fn workflow_sleep_emits_one_durable_timer_and_rounds_up() {
18585 let ctx = workflow_context(Vec::new());
18586 let mut sleep = Box::pin(ctx.sleep(Duration::from_millis(1_001)));
18587 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18588
18589 assert!(matches!(
18590 sleep.as_mut().poll(&mut task_context),
18591 Poll::Pending
18592 ));
18593 assert!(matches!(
18594 sleep.as_mut().poll(&mut task_context),
18595 Poll::Pending
18596 ));
18597
18598 let commands = ctx.take_commands().expect("timer command");
18599 assert_eq!(
18600 commands,
18601 vec![json!({
18602 "type": "start_timer",
18603 "delay_seconds": 2,
18604 })]
18605 );
18606 }
18607
18608 #[test]
18609 fn workflow_sleep_replays_matching_schedule_and_fire_without_a_command() {
18610 let history = vec![
18611 history_event(
18612 "TimerScheduled",
18613 json!({
18614 "sequence": 1,
18615 "timer_id": "timer-1",
18616 "delay_seconds": 5,
18617 "fire_at": "2026-07-11T12:00:05Z",
18618 }),
18619 ),
18620 history_event(
18621 "TimerFired",
18622 json!({
18623 "sequence": 1,
18624 "timer_id": "timer-1",
18625 "delay_seconds": 5,
18626 "fire_at": "2026-07-11T12:00:05Z",
18627 "fired_at": "2026-07-11T12:00:05Z",
18628 }),
18629 ),
18630 ];
18631
18632 for _restart in 0..2 {
18633 let ctx = workflow_context(history.clone());
18634 let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(5)));
18635 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18636 assert!(matches!(
18637 sleep.as_mut().poll(&mut task_context),
18638 Poll::Ready(Ok(()))
18639 ));
18640 assert!(ctx.take_commands().expect("commands").is_empty());
18641 ctx.ensure_history_consumed().expect("history consumed");
18642 }
18643 }
18644
18645 #[test]
18646 fn workflow_sleep_rejects_changed_delay_during_replay() {
18647 let ctx = workflow_context(vec![
18648 history_event(
18649 "TimerScheduled",
18650 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
18651 ),
18652 history_event(
18653 "TimerFired",
18654 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
18655 ),
18656 ]);
18657 let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(500)));
18658 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18659
18660 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18661 sleep.as_mut().poll(&mut task_context)
18662 else {
18663 panic!("changed timer delay must be rejected");
18664 };
18665 assert_eq!(failure.reason, "timer_delay_mismatch");
18666 assert_eq!(failure.sequence, Some(1));
18667 }
18668
18669 #[test]
18670 fn workflow_condition_wait_emits_published_identity_and_timeout_contract() {
18671 let ctx = workflow_context(Vec::new());
18672 let mut wait = Box::pin(
18673 ctx.wait_condition(
18674 ConditionWaitOptions::new("approval.ready", "sha256:approval-v1")
18675 .timeout(Duration::from_millis(60_001)),
18676 || Ok(false),
18677 ),
18678 );
18679 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18680
18681 assert!(matches!(
18682 wait.as_mut().poll(&mut task_context),
18683 Poll::Pending
18684 ));
18685 assert!(matches!(
18686 wait.as_mut().poll(&mut task_context),
18687 Poll::Pending
18688 ));
18689 assert_eq!(
18690 ctx.take_commands().expect("condition command"),
18691 vec![json!({
18692 "type": "open_condition_wait",
18693 "condition_wait_occurrence_id": "rust:condition-wait:0",
18694 "condition_key": "approval.ready",
18695 "condition_definition_fingerprint": "sha256:approval-v1",
18696 "timeout_seconds": 61,
18697 })]
18698 );
18699 }
18700
18701 #[test]
18702 fn workflow_condition_wait_returns_explicit_immediate_results_without_commands() {
18703 let ctx = workflow_context(Vec::new());
18704 let mut satisfied = Box::pin(wait_condition!(ctx, "already-ready", || Ok(true)));
18705 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18706 assert!(matches!(
18707 satisfied.as_mut().poll(&mut task_context),
18708 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18709 ));
18710
18711 let mut timed_out = Box::pin(wait_condition!(
18712 ctx,
18713 "no-wait",
18714 timeout: Duration::ZERO,
18715 || Ok(false),
18716 ));
18717 assert!(matches!(
18718 timed_out.as_mut().poll(&mut task_context),
18719 Poll::Ready(Ok(ConditionWaitResult::TimedOut))
18720 ));
18721 assert!(ctx.take_commands().expect("commands").is_empty());
18722 }
18723
18724 #[test]
18725 fn signal_and_update_history_reevaluate_open_conditions_after_restart() {
18726 let signal_history = vec![
18727 history_event(
18728 "ConditionWaitOpened",
18729 json!({
18730 "sequence": 4,
18731 "condition_wait_id": "condition:4",
18732 "condition_wait_occurrence_id": "rust:condition-wait:0",
18733 "condition_key": "approval",
18734 "condition_definition_fingerprint": "sha256:approval-v1",
18735 "timeout_seconds": 30,
18736 }),
18737 ),
18738 history_event(
18739 "SignalReceived",
18740 json!({
18741 "workflow_sequence": 4,
18742 "signal_name": "approve",
18743 "arguments": fixture_envelope(json!(["Ada"])),
18744 }),
18745 ),
18746 ];
18747 for _worker_before_or_after_restart in 0..2 {
18748 let ctx = workflow_context(signal_history.clone());
18749 let predicate_ctx = ctx.clone();
18750 let mut wait = Box::pin(
18751 ctx.wait_condition(
18752 ConditionWaitOptions::new("approval", "sha256:approval-v1")
18753 .timeout(Duration::from_secs(30)),
18754 move || Ok(!predicate_ctx.signals("approve")?.is_empty()),
18755 ),
18756 );
18757 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18758 assert!(matches!(
18759 wait.as_mut().poll(&mut task_context),
18760 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18761 ));
18762 assert!(ctx.take_commands().expect("commands").is_empty());
18763 ctx.ensure_history_consumed().expect("condition consumed");
18764 }
18765
18766 let update_history = vec![
18767 history_event(
18768 "ConditionWaitOpened",
18769 json!({
18770 "sequence": 7,
18771 "condition_wait_id": "condition:7",
18772 "condition_wait_occurrence_id": "rust:condition-wait:0",
18773 "condition_key": "update-approval",
18774 "condition_definition_fingerprint": "sha256:update-approval-v1",
18775 }),
18776 ),
18777 history_event(
18778 "UpdateApplied",
18779 json!({
18780 "sequence": 7,
18781 "update_id": "update-1",
18782 "update_name": "approve",
18783 "arguments": fixture_envelope(json!([true])),
18784 }),
18785 ),
18786 ];
18787 let ctx = workflow_context(update_history);
18788 let predicate_ctx = ctx.clone();
18789 let mut wait = Box::pin(ctx.wait_condition(
18790 ConditionWaitOptions::new("update-approval", "sha256:update-approval-v1"),
18791 move || {
18792 Ok(predicate_ctx
18793 .updates("approve")?
18794 .first()
18795 .and_then(|arguments| arguments.first())
18796 .and_then(Value::as_bool)
18797 == Some(true))
18798 },
18799 ));
18800 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18801 assert!(matches!(
18802 wait.as_mut().poll(&mut task_context),
18803 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18804 ));
18805 assert!(ctx.take_commands().expect("commands").is_empty());
18806 ctx.ensure_history_consumed().expect("condition consumed");
18807 }
18808
18809 #[test]
18810 fn condition_wait_preserves_open_satisfied_and_timed_out_replay_states() {
18811 let open_history = vec![
18812 history_event(
18813 "ConditionWaitOpened",
18814 json!({
18815 "sequence": 3,
18816 "condition_wait_id": "condition:3",
18817 "condition_wait_occurrence_id": "rust:condition-wait:0",
18818 "condition_key": "two-votes",
18819 "condition_definition_fingerprint": "sha256:two-votes-v1",
18820 "timeout_seconds": 120,
18821 }),
18822 ),
18823 history_event(
18824 "SignalReceived",
18825 json!({
18826 "workflow_sequence": 3,
18827 "signal_name": "vote",
18828 "arguments": fixture_envelope(json!(["first"])),
18829 }),
18830 ),
18831 ];
18832 for _worker_before_or_after_restart in 0..2 {
18833 let ctx = workflow_context(open_history.clone());
18834 let predicate_ctx = ctx.clone();
18835 let mut wait = Box::pin(
18836 ctx.wait_condition(
18837 ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1")
18838 .timeout(Duration::from_secs(120)),
18839 move || Ok(predicate_ctx.signals("vote")?.len() >= 2),
18840 ),
18841 );
18842 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18843 assert!(matches!(
18844 wait.as_mut().poll(&mut task_context),
18845 Poll::Pending
18846 ));
18847 assert_eq!(
18848 ctx.take_commands().expect("reopened condition"),
18849 vec![json!({
18850 "type": "open_condition_wait",
18851 "condition_wait_occurrence_id": "rust:condition-wait:0",
18852 "condition_key": "two-votes",
18853 "condition_definition_fingerprint": "sha256:two-votes-v1",
18854 "timeout_seconds": 120,
18855 })]
18856 );
18857 }
18858
18859 let satisfied_ctx = workflow_context(vec![
18860 history_event(
18861 "ConditionWaitOpened",
18862 json!({
18863 "sequence": 5,
18864 "condition_wait_id": "condition:5",
18865 "condition_wait_occurrence_id": "rust:condition-wait:0",
18866 "condition_key": "approval",
18867 "condition_definition_fingerprint": "sha256:approval-v1",
18868 }),
18869 ),
18870 history_event(
18871 "ConditionWaitSatisfied",
18872 json!({
18873 "sequence": 5,
18874 "condition_wait_id": "condition:5",
18875 "condition_wait_occurrence_id": "rust:condition-wait:0",
18876 "condition_key": "approval",
18877 "condition_definition_fingerprint": "sha256:approval-v1",
18878 }),
18879 ),
18880 ]);
18881 let mut satisfied = Box::pin(satisfied_ctx.wait_condition(
18882 ConditionWaitOptions::new("approval", "sha256:approval-v1"),
18883 || Ok(false),
18884 ));
18885 let mut task_context = TaskContext::from_waker(noop_waker_ref());
18886 assert!(matches!(
18887 satisfied.as_mut().poll(&mut task_context),
18888 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18889 ));
18890
18891 let timed_out_ctx = workflow_context(vec![
18892 history_event(
18893 "ConditionWaitOpened",
18894 json!({
18895 "sequence": 8,
18896 "condition_wait_id": "condition:8",
18897 "condition_wait_occurrence_id": "rust:condition-wait:0",
18898 "condition_key": "approval-timeout",
18899 "condition_definition_fingerprint": "sha256:approval-timeout-v1",
18900 "timeout_seconds": 5,
18901 }),
18902 ),
18903 history_event(
18904 "TimerScheduled",
18905 json!({
18906 "sequence": 9,
18907 "timer_id": "condition-timer:9",
18908 "timer_kind": "condition_timeout",
18909 "condition_wait_id": "condition:8",
18910 "delay_seconds": 5,
18911 }),
18912 ),
18913 history_event(
18914 "TimerFired",
18915 json!({
18916 "sequence": 9,
18917 "timer_id": "condition-timer:9",
18918 "timer_kind": "condition_timeout",
18919 "condition_wait_id": "condition:8",
18920 "delay_seconds": 5,
18921 }),
18922 ),
18923 ]);
18924 let mut timed_out = Box::pin(
18925 timed_out_ctx.wait_condition(
18926 ConditionWaitOptions::new("approval-timeout", "sha256:approval-timeout-v1")
18927 .timeout(Duration::from_secs(5)),
18928 || Ok(true),
18929 ),
18930 );
18931 assert!(matches!(
18932 timed_out.as_mut().poll(&mut task_context),
18933 Poll::Ready(Ok(ConditionWaitResult::TimedOut))
18934 ));
18935 }
18936
18937 #[test]
18938 fn condition_wait_replays_repeated_physical_opens_as_one_logical_wait() {
18939 let history = vec![
18940 history_event(
18941 "ConditionWaitOpened",
18942 json!({
18943 "sequence": 3,
18944 "condition_wait_id": "condition:3",
18945 "condition_wait_occurrence_id": "rust:condition-wait:0",
18946 "condition_key": "two-votes",
18947 "condition_definition_fingerprint": "sha256:two-votes-v1",
18948 }),
18949 ),
18950 history_event(
18951 "SignalReceived",
18952 json!({
18953 "workflow_sequence": 3,
18954 "signal_name": "vote",
18955 "arguments": fixture_envelope(json!(["first"])),
18956 }),
18957 ),
18958 history_event(
18959 "ConditionWaitSatisfied",
18960 json!({
18961 "sequence": 3,
18962 "condition_wait_id": "condition:3",
18963 "condition_wait_occurrence_id": "rust:condition-wait:0",
18964 "condition_key": "two-votes",
18965 "condition_definition_fingerprint": "sha256:two-votes-v1",
18966 }),
18967 ),
18968 history_event(
18969 "ConditionWaitOpened",
18970 json!({
18971 "sequence": 5,
18972 "condition_wait_id": "condition:5",
18973 "condition_wait_occurrence_id": "rust:condition-wait:0",
18974 "condition_key": "two-votes",
18975 "condition_definition_fingerprint": "sha256:two-votes-v1",
18976 }),
18977 ),
18978 history_event(
18979 "SignalReceived",
18980 json!({
18981 "workflow_sequence": 5,
18982 "signal_name": "vote",
18983 "arguments": fixture_envelope(json!(["second"])),
18984 }),
18985 ),
18986 history_event(
18987 "ConditionWaitSatisfied",
18988 json!({
18989 "sequence": 5,
18990 "condition_wait_id": "condition:5",
18991 "condition_wait_occurrence_id": "rust:condition-wait:0",
18992 "condition_key": "two-votes",
18993 "condition_definition_fingerprint": "sha256:two-votes-v1",
18994 }),
18995 ),
18996 ];
18997 for _cold_worker_or_restart in 0..2 {
18998 let ctx = workflow_context(history.clone());
18999 let predicate_ctx = ctx.clone();
19000 let mut wait = Box::pin(ctx.wait_condition(
19001 ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1"),
19002 move || Ok(predicate_ctx.signals("vote")?.len() >= 2),
19003 ));
19004 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19005
19006 assert!(matches!(
19007 wait.as_mut().poll(&mut task_context),
19008 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19009 ));
19010 assert!(ctx.take_commands().expect("commands").is_empty());
19011 ctx.ensure_history_consumed()
19012 .expect("every physical wait-open is consumed");
19013 }
19014 }
19015
19016 #[test]
19017 fn condition_wait_replays_update_driven_physical_opens_as_one_occurrence() {
19018 let history = vec![
19019 history_event(
19020 "ConditionWaitOpened",
19021 json!({
19022 "sequence": 3,
19023 "condition_wait_id": "condition:3",
19024 "condition_wait_occurrence_id": "rust:condition-wait:0",
19025 "condition_key": "approved",
19026 "condition_definition_fingerprint": "sha256:approved-v1",
19027 }),
19028 ),
19029 history_event(
19030 "UpdateApplied",
19031 json!({
19032 "sequence": 3,
19033 "update_id": "update-1",
19034 "update_name": "approve",
19035 "arguments": fixture_envelope(json!([false])),
19036 }),
19037 ),
19038 history_event(
19039 "ConditionWaitOpened",
19040 json!({
19041 "sequence": 5,
19042 "condition_wait_id": "condition:5",
19043 "condition_wait_occurrence_id": "rust:condition-wait:0",
19044 "condition_key": "approved",
19045 "condition_definition_fingerprint": "sha256:approved-v1",
19046 }),
19047 ),
19048 history_event(
19049 "UpdateApplied",
19050 json!({
19051 "sequence": 5,
19052 "update_id": "update-2",
19053 "update_name": "approve",
19054 "arguments": fixture_envelope(json!([true])),
19055 }),
19056 ),
19057 ];
19058
19059 for _cold_worker_or_restart in 0..2 {
19060 let ctx = workflow_context(history.clone());
19061 let predicate_ctx = ctx.clone();
19062 let mut wait = Box::pin(ctx.wait_condition(
19063 ConditionWaitOptions::new("approved", "sha256:approved-v1"),
19064 move || {
19065 Ok(predicate_ctx
19066 .updates("approve")?
19067 .last()
19068 .and_then(|arguments| arguments.first())
19069 .and_then(Value::as_bool)
19070 == Some(true))
19071 },
19072 ));
19073 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19074
19075 assert!(matches!(
19076 wait.as_mut().poll(&mut task_context),
19077 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19078 ));
19079 assert!(ctx.take_commands().expect("commands").is_empty());
19080 ctx.ensure_history_consumed()
19081 .expect("every update-driven reopen is consumed");
19082 }
19083 }
19084
19085 #[test]
19086 fn condition_wait_replay_keeps_every_adjacent_authored_occurrence_distinct() {
19087 for (first_key, first_fingerprint, second_key, second_fingerprint) in [
19088 ("shared", "sha256:first", "shared", "sha256:second"),
19089 ("first", "sha256:shared", "second", "sha256:shared"),
19090 ("shared", "sha256:shared", "shared", "sha256:shared"),
19091 ("first", "sha256:first", "second", "sha256:second"),
19092 ] {
19093 let history = vec![
19094 history_event(
19095 "ConditionWaitOpened",
19096 json!({
19097 "sequence": 3,
19098 "condition_wait_id": "condition:3",
19099 "condition_wait_occurrence_id": "rust:condition-wait:0",
19100 "condition_key": first_key,
19101 "condition_definition_fingerprint": first_fingerprint,
19102 }),
19103 ),
19104 history_event(
19105 "ConditionWaitSatisfied",
19106 json!({
19107 "sequence": 3,
19108 "condition_wait_id": "condition:3",
19109 "condition_wait_occurrence_id": "rust:condition-wait:0",
19110 "condition_key": first_key,
19111 "condition_definition_fingerprint": first_fingerprint,
19112 }),
19113 ),
19114 history_event(
19115 "ConditionWaitOpened",
19116 json!({
19117 "sequence": 4,
19118 "condition_wait_id": "condition:4",
19119 "condition_wait_occurrence_id": "rust:condition-wait:1",
19120 "condition_key": second_key,
19121 "condition_definition_fingerprint": second_fingerprint,
19122 }),
19123 ),
19124 history_event(
19125 "ConditionWaitSatisfied",
19126 json!({
19127 "sequence": 4,
19128 "condition_wait_id": "condition:4",
19129 "condition_wait_occurrence_id": "rust:condition-wait:1",
19130 "condition_key": second_key,
19131 "condition_definition_fingerprint": second_fingerprint,
19132 }),
19133 ),
19134 ];
19135 for _cold_worker_or_restart in 0..2 {
19136 let ctx = workflow_context(history.clone());
19137 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19138 let mut first = Box::pin(ctx.wait_condition(
19139 ConditionWaitOptions::new(first_key, first_fingerprint),
19140 || Ok(false),
19141 ));
19142 assert!(matches!(
19143 first.as_mut().poll(&mut task_context),
19144 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19145 ));
19146
19147 let mut second = Box::pin(ctx.wait_condition(
19148 ConditionWaitOptions::new(second_key, second_fingerprint),
19149 || Ok(false),
19150 ));
19151 assert!(matches!(
19152 second.as_mut().poll(&mut task_context),
19153 Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19154 ));
19155 assert!(ctx.take_commands().expect("commands").is_empty());
19156 ctx.ensure_history_consumed()
19157 .expect("each authored wait consumes one occurrence");
19158 }
19159 }
19160 }
19161
19162 #[test]
19163 fn cold_workers_replay_adjacent_condition_waits_from_one_loop_call_site() {
19164 fn worker() -> Worker {
19165 let client = Client::new("http://127.0.0.1:8080").expect("client");
19166 let mut worker = Worker::new(client, "rust-workers");
19167 worker.register_workflow("rust.condition-loop", |ctx, _input| async move {
19168 let mut outcomes = Vec::new();
19169 for _ in 0..2 {
19170 outcomes.push(
19171 ctx.wait_condition(
19172 ConditionWaitOptions::new("shared", "sha256:shared"),
19173 || Ok(false),
19174 )
19175 .await?,
19176 );
19177 }
19178 Ok(json!(outcomes))
19179 });
19180 worker
19181 }
19182
19183 let task = workflow_task(
19184 "rust.condition-loop",
19185 vec![
19186 history_event(
19187 "ConditionWaitOpened",
19188 json!({
19189 "sequence": 1,
19190 "condition_wait_id": "condition:1",
19191 "condition_wait_occurrence_id": "rust:condition-wait:0",
19192 "condition_key": "shared",
19193 "condition_definition_fingerprint": "sha256:shared",
19194 }),
19195 ),
19196 history_event(
19197 "ConditionWaitSatisfied",
19198 json!({
19199 "sequence": 1,
19200 "condition_wait_id": "condition:1",
19201 "condition_wait_occurrence_id": "rust:condition-wait:0",
19202 "condition_key": "shared",
19203 "condition_definition_fingerprint": "sha256:shared",
19204 }),
19205 ),
19206 history_event(
19207 "ConditionWaitOpened",
19208 json!({
19209 "sequence": 2,
19210 "condition_wait_id": "condition:2",
19211 "condition_wait_occurrence_id": "rust:condition-wait:1",
19212 "condition_key": "shared",
19213 "condition_definition_fingerprint": "sha256:shared",
19214 }),
19215 ),
19216 history_event(
19217 "ConditionWaitSatisfied",
19218 json!({
19219 "sequence": 2,
19220 "condition_wait_id": "condition:2",
19221 "condition_wait_occurrence_id": "rust:condition-wait:1",
19222 "condition_key": "shared",
19223 "condition_definition_fingerprint": "sha256:shared",
19224 }),
19225 ),
19226 ],
19227 DEFAULT_CODEC,
19228 );
19229
19230 for _cold_worker_or_restart in 0..2 {
19231 let commands = worker()
19232 .execute_workflow_task(task.clone())
19233 .expect("adjacent loop waits replay deterministically");
19234 assert_eq!(commands.len(), 1);
19235 assert_eq!(commands[0]["type"], "complete_workflow");
19236 assert_eq!(
19237 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
19238 json!(["satisfied", "satisfied"])
19239 );
19240 }
19241 }
19242
19243 #[test]
19244 fn condition_wait_replay_rejects_identity_predicate_and_timeout_changes() {
19245 let history = vec![history_event(
19246 "ConditionWaitOpened",
19247 json!({
19248 "sequence": 12,
19249 "condition_wait_id": "condition:12",
19250 "condition_wait_occurrence_id": "rust:condition-wait:0",
19251 "condition_key": "approval",
19252 "condition_definition_fingerprint": "sha256:approval-v1",
19253 "timeout_seconds": 30,
19254 }),
19255 )];
19256 for (options, expected_reason) in [
19257 (
19258 ConditionWaitOptions::new("changed", "sha256:approval-v1")
19259 .timeout(Duration::from_secs(30)),
19260 "condition_wait_key_mismatch",
19261 ),
19262 (
19263 ConditionWaitOptions::new("approval", "sha256:approval-v2")
19264 .timeout(Duration::from_secs(30)),
19265 "condition_wait_predicate_mismatch",
19266 ),
19267 (
19268 ConditionWaitOptions::new("approval", "sha256:approval-v1")
19269 .timeout(Duration::from_secs(29)),
19270 "condition_wait_timeout_mismatch",
19271 ),
19272 ] {
19273 let ctx = workflow_context(history.clone());
19274 let mut wait = Box::pin(ctx.wait_condition(options, || Ok(false)));
19275 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19276 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
19277 wait.as_mut().poll(&mut task_context)
19278 else {
19279 panic!("changed condition definition must fail replay");
19280 };
19281 assert_eq!(failure.reason, expected_reason);
19282 assert_eq!(failure.sequence, Some(12));
19283 }
19284 }
19285
19286 #[test]
19287 fn condition_wait_history_requires_the_canonical_predicate_fingerprint() {
19288 let error = WorkflowState::new(
19289 vec![history_event(
19290 "ConditionWaitOpened",
19291 json!({
19292 "sequence": 12,
19293 "condition_wait_id": "condition:12",
19294 "condition_wait_occurrence_id": "rust:condition-wait:0",
19295 "condition_key": "approval",
19296 }),
19297 )],
19298 "rust-workers".to_string(),
19299 DEFAULT_CODEC.to_string(),
19300 None,
19301 )
19302 .expect_err("condition history without a predicate fingerprint must fail");
19303
19304 assert!(matches!(
19305 error,
19306 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19307 if reason == "condition_wait_predicate_fingerprint_missing"
19308 ));
19309 }
19310
19311 #[test]
19312 fn condition_wait_history_requires_authored_occurrence_identity() {
19313 let error = WorkflowState::new(
19314 vec![history_event(
19315 "ConditionWaitOpened",
19316 json!({
19317 "sequence": 12,
19318 "condition_wait_id": "condition:12",
19319 "condition_key": "approval",
19320 "condition_definition_fingerprint": "sha256:approval-v1",
19321 }),
19322 )],
19323 "rust-workers".to_string(),
19324 DEFAULT_CODEC.to_string(),
19325 None,
19326 )
19327 .expect_err("condition history without occurrence identity must fail");
19328
19329 assert!(matches!(
19330 error,
19331 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19332 if reason == "condition_wait_occurrence_id_missing"
19333 ));
19334 }
19335
19336 #[test]
19337 fn typed_search_attribute_updates_validate_emit_and_replay() {
19338 let update = SearchAttributeUpdate::new()
19339 .keyword("OrderStatus", " waiting ")
19340 .expect("keyword")
19341 .int("Attempt", 3)
19342 .expect("int")
19343 .bool("Escalated", false)
19344 .expect("bool")
19345 .keyword_list("Regions", ["us-east", "eu-west"])
19346 .expect("list")
19347 .datetime("UpdatedAt", "2026-08-22T04:00:00Z")
19348 .expect("datetime")
19349 .delete("LegacyStatus")
19350 .expect("delete");
19351 let ctx = workflow_context(Vec::new());
19352 ctx.upsert_search_attributes(update.clone())
19353 .expect("typed update");
19354 assert_eq!(
19355 ctx.take_commands().expect("search-attribute command"),
19356 vec![json!({
19357 "type": "upsert_search_attributes",
19358 "attributes": {
19359 "Attempt": 3,
19360 "Escalated": false,
19361 "LegacyStatus": null,
19362 "OrderStatus": "waiting",
19363 "Regions": ["us-east", "eu-west"],
19364 "UpdatedAt": "2026-08-22T04:00:00Z",
19365 },
19366 "attribute_types": {
19367 "Attempt": "int",
19368 "Escalated": "bool",
19369 "OrderStatus": "keyword",
19370 "Regions": "keyword_list",
19371 "UpdatedAt": "datetime",
19372 },
19373 })]
19374 );
19375
19376 let replay = workflow_context(vec![history_event(
19377 "SearchAttributesUpserted",
19378 json!({
19379 "sequence": 6,
19380 "attributes": {
19381 "Attempt": 3,
19382 "Escalated": false,
19383 "LegacyStatus": null,
19384 "OrderStatus": "waiting",
19385 "Regions": ["us-east", "eu-west"],
19386 "UpdatedAt": "2026-08-22T04:00:00Z",
19387 },
19388 "attribute_types": {
19389 "Attempt": "int",
19390 "Escalated": "bool",
19391 "OrderStatus": "keyword",
19392 "Regions": "keyword_list",
19393 "UpdatedAt": "datetime",
19394 },
19395 "merged": {},
19396 }),
19397 )]);
19398 replay
19399 .upsert_search_attributes(update)
19400 .expect("matching update replays");
19401 assert!(replay.take_commands().expect("commands").is_empty());
19402 replay.ensure_history_consumed().expect("history consumed");
19403
19404 let type_drift = workflow_context(vec![history_event(
19405 "SearchAttributesUpserted",
19406 json!({
19407 "sequence": 7,
19408 "attributes": {"OrderStatus": "waiting"},
19409 "attribute_types": {"OrderStatus": "keyword"},
19410 "merged": {"OrderStatus": "waiting"},
19411 }),
19412 )]);
19413 let error = type_drift
19414 .upsert_search_attributes(
19415 SearchAttributeUpdate::new()
19416 .string("OrderStatus", "waiting")
19417 .expect("string update"),
19418 )
19419 .expect_err("same JSON value with a changed type must fail replay");
19420 assert!(matches!(
19421 error,
19422 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19423 if reason == "search_attribute_type_mismatch"
19424 ));
19425
19426 let malformed_types = WorkflowState::new(
19427 vec![history_event(
19428 "SearchAttributesUpserted",
19429 json!({
19430 "sequence": 8,
19431 "attributes": {"OrderStatus": "waiting"},
19432 "attribute_types": {"OrderStatus": "unsupported"},
19433 "merged": {"OrderStatus": "waiting"},
19434 }),
19435 )],
19436 "rust-workers".to_string(),
19437 DEFAULT_CODEC.to_string(),
19438 None,
19439 )
19440 .expect_err("unsupported search-attribute type metadata must fail");
19441 assert!(matches!(
19442 malformed_types,
19443 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19444 if reason == "search_attribute_types_malformed"
19445 ));
19446
19447 assert!(matches!(
19448 SearchAttributeUpdate::new().keyword("bad key", "value"),
19449 Err(SearchAttributeUpdateError::InvalidKey(_))
19450 ));
19451 assert!(matches!(
19452 SearchAttributeUpdate::new().float("Ratio", f64::NAN),
19453 Err(SearchAttributeUpdateError::NonFiniteFloat(_))
19454 ));
19455 assert!(matches!(
19456 SearchAttributeUpdate::new().keyword("UnicodeKeyword", "é".repeat(128)),
19457 Err(SearchAttributeUpdateError::ValueTooLong { .. })
19458 ));
19459 assert!(matches!(
19460 SearchAttributeUpdate::new().datetime("UpdatedAt", "2026-02-30T04:00:00Z"),
19461 Err(SearchAttributeUpdateError::InvalidDateTime(_))
19462 ));
19463 assert!(matches!(
19464 workflow_context(Vec::new()).upsert_search_attributes(SearchAttributeUpdate::new()),
19465 Err(Error::InvalidSearchAttributeUpdate(
19466 SearchAttributeUpdateError::Empty
19467 ))
19468 ));
19469 }
19470
19471 #[test]
19472 fn typed_search_attribute_text_uses_the_runtime_byte_limit() {
19473 let ascii = "a".repeat(MAX_SEARCH_ATTRIBUTE_STRING_LENGTH);
19474 let utf8 = "é".repeat(MAX_SEARCH_ATTRIBUTE_STRING_LENGTH / 2);
19475
19476 assert!(SearchAttributeUpdate::new()
19477 .string("AsciiDescription", ascii)
19478 .is_ok());
19479 assert!(SearchAttributeUpdate::new()
19480 .string("Utf8Description", utf8)
19481 .is_ok());
19482 assert!(matches!(
19483 SearchAttributeUpdate::new().string(
19484 "TooLongDescription",
19485 "é".repeat((MAX_SEARCH_ATTRIBUTE_STRING_LENGTH / 2) + 1),
19486 ),
19487 Err(SearchAttributeUpdateError::ValueTooLong {
19488 kind: "string",
19489 limit: MAX_SEARCH_ATTRIBUTE_STRING_LENGTH,
19490 ..
19491 })
19492 ));
19493 }
19494
19495 #[test]
19496 fn workflow_history_rejects_unpaired_or_mismatched_timer_events() {
19497 let lone_fire = WorkflowState::new(
19498 vec![history_event(
19499 "TimerFired",
19500 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19501 )],
19502 "rust-workers".to_string(),
19503 DEFAULT_CODEC.to_string(),
19504 None,
19505 )
19506 .expect_err("TimerFired requires TimerScheduled");
19507 assert!(matches!(
19508 lone_fire,
19509 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19510 if reason == "timer_schedule_missing_or_duplicate"
19511 ));
19512
19513 let wrong_identity = WorkflowState::new(
19514 vec![
19515 history_event(
19516 "TimerScheduled",
19517 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19518 ),
19519 history_event(
19520 "TimerFired",
19521 json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
19522 ),
19523 ],
19524 "rust-workers".to_string(),
19525 DEFAULT_CODEC.to_string(),
19526 None,
19527 )
19528 .expect_err("fire must match scheduled timer identity");
19529 assert!(matches!(
19530 wrong_identity,
19531 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19532 if reason == "timer_identity_mismatch"
19533 ));
19534
19535 let duplicate_fire = WorkflowState::new(
19536 vec![
19537 history_event(
19538 "TimerScheduled",
19539 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19540 ),
19541 history_event(
19542 "TimerFired",
19543 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19544 ),
19545 history_event(
19546 "TimerFired",
19547 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19548 ),
19549 ],
19550 "rust-workers".to_string(),
19551 DEFAULT_CODEC.to_string(),
19552 None,
19553 )
19554 .expect_err("a durable timer cannot fire twice");
19555 assert!(matches!(
19556 duplicate_fire,
19557 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19558 if reason == "duplicate_timer_fire"
19559 ));
19560
19561 let wrong_fired_delay = WorkflowState::new(
19562 vec![
19563 history_event(
19564 "TimerScheduled",
19565 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19566 ),
19567 history_event(
19568 "TimerFired",
19569 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 6}),
19570 ),
19571 ],
19572 "rust-workers".to_string(),
19573 DEFAULT_CODEC.to_string(),
19574 None,
19575 )
19576 .expect_err("timer schedule and fire delays must agree");
19577 assert!(matches!(
19578 wrong_fired_delay,
19579 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19580 if reason == "timer_history_delay_mismatch"
19581 ));
19582 }
19583
19584 #[test]
19585 fn replay_rejects_activity_moved_before_recorded_timer() {
19586 let ctx = workflow_context(vec![
19587 history_event(
19588 "TimerScheduled",
19589 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19590 ),
19591 history_event(
19592 "TimerFired",
19593 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19594 ),
19595 history_event(
19596 "ActivityCompleted",
19597 json!({
19598 "sequence": 2,
19599 "activity_type": "after-timer",
19600 "payload_codec": DEFAULT_CODEC,
19601 "result": fixture_envelope(json!("done")),
19602 }),
19603 ),
19604 ]);
19605 let mut activity = Box::pin(ctx.activity("after-timer", json!([])));
19606 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19607
19608 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
19609 activity.as_mut().poll(&mut task_context)
19610 else {
19611 panic!("reordered durable command must be rejected");
19612 };
19613 assert_eq!(failure.reason, "recorded_command_mismatch");
19614 assert_eq!(failure.sequence, Some(1));
19615 assert_eq!(failure.expected.as_deref(), Some("timer"));
19616 assert_eq!(failure.actual.as_deref(), Some("activity:after-timer"));
19617 }
19618
19619 #[test]
19620 fn workflow_context_emits_a_typed_named_signal_wait() {
19621 let ctx = workflow_context(Vec::new());
19622 let mut signal = Box::pin(ctx.wait_signal("finish"));
19623 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19624
19625 assert!(matches!(
19626 signal.as_mut().poll(&mut task_context),
19627 Poll::Pending
19628 ));
19629 assert_eq!(
19630 ctx.take_commands().expect("signal-wait command"),
19631 vec![json!({
19632 "type": "open_signal_wait",
19633 "signal_name": "finish",
19634 })]
19635 );
19636 }
19637
19638 #[test]
19639 fn runtime_message_stream_transport_cannot_be_opened_as_a_user_signal() {
19640 let ctx = workflow_context(Vec::new());
19641 let mut signal = Box::pin(ctx.wait_signal(MESSAGE_STREAM_SIGNAL));
19642 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19643
19644 let Poll::Ready(Err(Error::Codec(message))) = signal.as_mut().poll(&mut task_context)
19645 else {
19646 panic!("runtime-reserved signal should be rejected");
19647 };
19648 assert!(message.contains("reserved by the workflow runtime"));
19649 assert!(ctx.take_commands().expect("commands").is_empty());
19650 }
19651
19652 #[tokio::test]
19653 async fn runtime_message_stream_transport_cannot_be_sent_as_a_user_signal() {
19654 let client = Client::builder("http://127.0.0.1:9")
19655 .build()
19656 .expect("client");
19657 let error = client
19658 .signal_workflow("workflow-1", MESSAGE_STREAM_SIGNAL, json!(["forged"]))
19659 .await
19660 .expect_err("runtime-reserved signal should be rejected before transport");
19661
19662 assert!(
19663 matches!(error, Error::Codec(ref message) if message.contains("reserved by the workflow runtime"))
19664 );
19665 }
19666
19667 #[test]
19668 fn message_stream_worker_task_consumes_current_contiguous_bounded_batch() {
19669 fn delivery(message_id: &str, position: u64, value: &str) -> Value {
19670 let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
19671 value.to_string(),
19672 )]))
19673 .expect("message payload");
19674 json!({
19675 "schema": MESSAGE_STREAM_SCHEMA,
19676 "stream_name": "orders",
19677 "message_id": message_id,
19678 "position": position,
19679 "payload_envelope": payload,
19680 })
19681 }
19682
19683 fn opened(sequence: u64) -> HistoryEvent {
19684 history_event(
19685 "SignalWaitOpened",
19686 json!({
19687 "sequence": sequence,
19688 "signal_name": MESSAGE_STREAM_SIGNAL,
19689 }),
19690 )
19691 }
19692
19693 fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
19694 history_event(
19695 "SignalApplied",
19696 json!({
19697 "sequence": sequence,
19698 "signal_name": MESSAGE_STREAM_SIGNAL,
19699 "value": fixture_envelope(json!([delivery])),
19700 }),
19701 )
19702 }
19703
19704 fn received(delivery: Value) -> HistoryEvent {
19705 history_event(
19706 "SignalReceived",
19707 json!({
19708 "signal_name": MESSAGE_STREAM_SIGNAL,
19709 "arguments": fixture_envelope(json!([delivery])),
19710 "payload_codec": DEFAULT_CODEC,
19711 }),
19712 )
19713 }
19714
19715 let client = Client::new("http://127.0.0.1:8080").expect("client");
19716 let mut worker = Worker::new(client, "rust-workers");
19717 worker.register_workflow("rust.message-stream-batch", |ctx, _input| async move {
19718 let messages = ctx.message_stream("orders")?.receive(2).await?;
19719 Ok(json!(messages
19720 .into_iter()
19721 .map(|message| message.message_id)
19722 .collect::<Vec<_>>()))
19723 });
19724
19725 let first = delivery("message-1", 1, "one");
19726 let second = delivery("message-2", 2, "two");
19727 let batch = worker
19728 .execute_workflow_task_decision(workflow_task(
19729 "rust.message-stream-batch",
19730 vec![
19731 opened(1),
19732 received(first.clone()),
19733 applied(1, first.clone()),
19734 received(first.clone()),
19735 received(second),
19736 ],
19737 DEFAULT_CODEC,
19738 ))
19739 .expect("worker task consumes the available batch");
19740
19741 assert_eq!(batch.commands.len(), 1);
19742 assert_eq!(batch.commands[0]["type"], "complete_workflow");
19743 assert_eq!(
19744 decode_wire_value(&batch.commands[0]["result"], DEFAULT_CODEC)
19745 .expect("workflow result"),
19746 json!(["message-1", "message-2"])
19747 );
19748 assert_eq!(
19749 batch.message_stream_cursors,
19750 vec![json!({"stream_name": "orders", "through_position": 2})]
19751 );
19752 assert!(batch.message_stream_waits.is_empty());
19753
19754 let partial = worker
19755 .execute_workflow_task_decision(workflow_task(
19756 "rust.message-stream-batch",
19757 vec![opened(1), received(first.clone()), applied(1, first)],
19758 DEFAULT_CODEC,
19759 ))
19760 .expect("worker task returns without waiting for a missing second item");
19761 assert_eq!(partial.commands.len(), 1);
19762 assert_eq!(partial.commands[0]["type"], "complete_workflow");
19763 assert_eq!(
19764 decode_wire_value(&partial.commands[0]["result"], DEFAULT_CODEC)
19765 .expect("workflow result"),
19766 json!(["message-1"])
19767 );
19768 assert_eq!(
19769 partial.message_stream_cursors,
19770 vec![json!({"stream_name": "orders", "through_position": 1})]
19771 );
19772 assert!(partial.message_stream_waits.is_empty());
19773 }
19774
19775 #[test]
19776 fn message_stream_replay_preserves_partial_batch_boundary_before_later_wait() {
19777 fn delivery(message_id: &str, position: u64, value: &str) -> Value {
19778 let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
19779 value.to_string(),
19780 )]))
19781 .expect("message payload");
19782 json!({
19783 "schema": MESSAGE_STREAM_SCHEMA,
19784 "stream_name": "orders",
19785 "message_id": message_id,
19786 "position": position,
19787 "payload_envelope": payload,
19788 })
19789 }
19790
19791 fn opened(sequence: u64) -> HistoryEvent {
19792 history_event(
19793 "SignalWaitOpened",
19794 json!({
19795 "sequence": sequence,
19796 "signal_name": MESSAGE_STREAM_SIGNAL,
19797 }),
19798 )
19799 }
19800
19801 fn received(delivery: Value) -> HistoryEvent {
19802 history_event(
19803 "SignalReceived",
19804 json!({
19805 "signal_name": MESSAGE_STREAM_SIGNAL,
19806 "arguments": fixture_envelope(json!([delivery])),
19807 "payload_codec": DEFAULT_CODEC,
19808 }),
19809 )
19810 }
19811
19812 fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
19813 history_event(
19814 "SignalApplied",
19815 json!({
19816 "sequence": sequence,
19817 "signal_name": MESSAGE_STREAM_SIGNAL,
19818 "value": fixture_envelope(json!([delivery])),
19819 }),
19820 )
19821 }
19822
19823 let client = Client::new("http://127.0.0.1:8080").expect("client");
19824 let mut worker = Worker::new(client, "rust-workers");
19825 worker.register_workflow(
19826 "rust.message-stream-partial-batches",
19827 |ctx, _input| async move {
19828 let stream = ctx.message_stream("orders")?;
19829 let first = stream.receive(10).await?;
19830 let second = stream.receive(10).await?;
19831 Ok(json!([
19832 first
19833 .into_iter()
19834 .map(|message| message.message_id)
19835 .collect::<Vec<_>>(),
19836 second
19837 .into_iter()
19838 .map(|message| message.message_id)
19839 .collect::<Vec<_>>(),
19840 ]))
19841 },
19842 );
19843
19844 let first = delivery("message-1", 1, "one");
19845 let second = delivery("message-2", 2, "two");
19846 let decision = worker
19847 .execute_workflow_task_decision(workflow_task(
19848 "rust.message-stream-partial-batches",
19849 vec![
19850 opened(1),
19851 received(first.clone()),
19852 applied(1, first),
19853 opened(2),
19854 received(second.clone()),
19855 applied(2, second),
19856 ],
19857 DEFAULT_CODEC,
19858 ))
19859 .expect("cold replay preserves both authored receive boundaries");
19860
19861 assert_eq!(decision.commands.len(), 1);
19862 assert_eq!(decision.commands[0]["type"], "complete_workflow");
19863 assert_eq!(
19864 decode_wire_value(&decision.commands[0]["result"], DEFAULT_CODEC)
19865 .expect("workflow result"),
19866 json!([["message-1"], ["message-2"]])
19867 );
19868 assert_eq!(
19869 decision.message_stream_cursors,
19870 vec![json!({"stream_name": "orders", "through_position": 2})]
19871 );
19872 assert!(decision.message_stream_waits.is_empty());
19873 }
19874
19875 #[test]
19876 fn empty_message_stream_opens_internal_signal_wait_and_reports_position() {
19877 let ctx = workflow_context(Vec::new());
19878 let stream = ctx.message_stream("orders").expect("message stream");
19879 let mut receive = Box::pin(stream.receive(10));
19880 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19881
19882 assert!(matches!(
19883 receive.as_mut().poll(&mut task_context),
19884 Poll::Pending
19885 ));
19886 assert_eq!(
19887 ctx.take_commands().expect("message-stream wait command"),
19888 vec![json!({
19889 "type": "open_signal_wait",
19890 "signal_name": MESSAGE_STREAM_SIGNAL,
19891 })]
19892 );
19893 let (cursors, waits) = ctx.message_stream_metadata().expect("stream metadata");
19894 assert!(cursors.is_empty());
19895 assert_eq!(
19896 waits,
19897 vec![json!({"stream_name": "orders", "after_position": 0})]
19898 );
19899 }
19900
19901 #[test]
19902 fn continue_as_new_cursor_checkpoint_preserves_global_pending_position() {
19903 let ctx = workflow_context(vec![history_event(
19904 "SignalReceived",
19905 json!({
19906 "signal_name": MESSAGE_STREAM_SIGNAL,
19907 "arguments": fixture_envelope(json!([{
19908 "schema": MESSAGE_STREAM_CURSOR_SCHEMA,
19909 "stream_name": "orders",
19910 "through_position": 2,
19911 }])),
19912 "payload_codec": DEFAULT_CODEC,
19913 }),
19914 )]);
19915 let stream = ctx.message_stream("orders").expect("message stream");
19916 let mut receive = Box::pin(stream.receive(10));
19917 let mut task_context = TaskContext::from_waker(noop_waker_ref());
19918
19919 assert!(matches!(
19920 receive.as_mut().poll(&mut task_context),
19921 Poll::Pending
19922 ));
19923 let (cursors, waits) = ctx.message_stream_metadata().expect("stream metadata");
19924 assert_eq!(
19925 cursors,
19926 vec![json!({"stream_name": "orders", "through_position": 2})]
19927 );
19928 assert_eq!(
19929 waits,
19930 vec![json!({"stream_name": "orders", "after_position": 2})]
19931 );
19932 }
19933
19934 #[test]
19935 fn message_stream_delivery_preserves_typed_avro_arguments_across_replay() {
19936 let mut empty_map = BTreeMap::new();
19937 let mut nested = BTreeMap::new();
19938 nested.insert(
19939 "value".to_string(),
19940 AvroValue::Array(vec![AvroValue::Bytes(b"nested".to_vec())]),
19941 );
19942 let values = vec![
19943 AvroValue::Bytes(vec![0, 255]),
19944 AvroValue::Long(1),
19945 AvroValue::Double(1.0),
19946 AvroValue::Array(Vec::new()),
19947 AvroValue::Map(std::mem::take(&mut empty_map)),
19948 AvroValue::Map(nested),
19949 ];
19950 let payload = encode_avro_value(&AvroValue::Array(values.clone())).expect("payload");
19951 let transport = vec![json!({
19952 "schema": MESSAGE_STREAM_SCHEMA,
19953 "stream_name": "orders",
19954 "message_id": "message-1",
19955 "position": 1,
19956 "payload_envelope": payload,
19957 })];
19958
19959 for _ in 0..2 {
19960 let Some(MessageStreamDelivery::Message(message)) =
19961 decode_message_stream_delivery(transport.clone()).expect("delivery")
19962 else {
19963 panic!("message delivery expected");
19964 };
19965 assert_eq!(message.arguments, values);
19966 assert!(matches!(message.arguments[1], AvroValue::Long(1)));
19967 assert!(matches!(message.arguments[2], AvroValue::Double(1.0)));
19968 }
19969 }
19970
19971 #[test]
19972 fn cold_worker_replacement_consumes_message_stream_wait_arrivals_once_in_order() {
19973 fn delivery(message_id: &str, position: u64, value: &str) -> Value {
19974 let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
19975 value.to_string(),
19976 )]))
19977 .expect("message payload");
19978 json!({
19979 "schema": MESSAGE_STREAM_SCHEMA,
19980 "stream_name": "orders",
19981 "message_id": message_id,
19982 "position": position,
19983 "payload_envelope": payload,
19984 })
19985 }
19986
19987 fn opened(sequence: u64) -> HistoryEvent {
19988 history_event(
19989 "SignalWaitOpened",
19990 json!({
19991 "sequence": sequence,
19992 "signal_name": MESSAGE_STREAM_SIGNAL,
19993 }),
19994 )
19995 }
19996
19997 fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
19998 history_event(
19999 "SignalApplied",
20000 json!({
20001 "sequence": sequence,
20002 "signal_name": MESSAGE_STREAM_SIGNAL,
20003 "value": fixture_envelope(json!([delivery])),
20004 }),
20005 )
20006 }
20007
20008 fn worker() -> Worker {
20009 let client = Client::new("http://127.0.0.1:8080").expect("client");
20010 let mut worker = Worker::new(client, "rust-workers");
20011 worker.register_workflow("rust.message-stream", |ctx, _input| async move {
20012 let stream = ctx.message_stream("orders")?;
20013 let first = stream.receive_one().await?;
20014 let second = stream.receive_one().await?;
20015 Ok(json!([first.message_id, second.message_id]))
20016 });
20017 worker
20018 }
20019
20020 fn task_with_resume(history: Vec<HistoryEvent>, delivery: Value) -> WorkflowTask {
20021 let mut task = workflow_task("rust.message-stream", history, DEFAULT_CODEC);
20022 task.signal_name = Some(MESSAGE_STREAM_SIGNAL.to_string());
20023 task.signal_arguments = Some(fixture_envelope(json!([delivery])));
20024 task
20025 }
20026
20027 let waiting = worker()
20028 .execute_workflow_task_decision(workflow_task(
20029 "rust.message-stream",
20030 Vec::new(),
20031 DEFAULT_CODEC,
20032 ))
20033 .expect("first worker opens the stream wait");
20034 assert_eq!(
20035 waiting.commands,
20036 vec![json!({
20037 "type": "open_signal_wait",
20038 "signal_name": MESSAGE_STREAM_SIGNAL,
20039 })]
20040 );
20041 assert!(waiting.message_stream_cursors.is_empty());
20042 assert_eq!(
20043 waiting.message_stream_waits,
20044 vec![json!({"stream_name": "orders", "after_position": 0})]
20045 );
20046
20047 let first_delivery = delivery("message-1", 1, "one");
20048 let first_arrival = worker()
20049 .execute_workflow_task_decision(task_with_resume(
20050 vec![opened(1)],
20051 first_delivery.clone(),
20052 ))
20053 .expect("replacement worker consumes the first arrival");
20054 assert_eq!(
20055 first_arrival.commands,
20056 vec![json!({
20057 "type": "open_signal_wait",
20058 "signal_name": MESSAGE_STREAM_SIGNAL,
20059 })]
20060 );
20061 assert_eq!(
20062 first_arrival.message_stream_cursors,
20063 vec![json!({"stream_name": "orders", "through_position": 1})]
20064 );
20065 assert_eq!(
20066 first_arrival.message_stream_waits,
20067 vec![json!({"stream_name": "orders", "after_position": 1})]
20068 );
20069
20070 let second_delivery = delivery("message-2", 2, "two");
20071 let first_applied = applied(1, first_delivery);
20072 let completed = worker()
20073 .execute_workflow_task_decision(task_with_resume(
20074 vec![opened(1), first_applied.clone(), opened(2)],
20075 second_delivery.clone(),
20076 ))
20077 .expect("next replacement worker consumes the second arrival");
20078 assert_eq!(completed.commands.len(), 1);
20079 assert_eq!(completed.commands[0]["type"], "complete_workflow");
20080 assert_eq!(
20081 decode_wire_value(&completed.commands[0]["result"], DEFAULT_CODEC)
20082 .expect("workflow result"),
20083 json!(["message-1", "message-2"])
20084 );
20085 assert_eq!(
20086 completed.message_stream_cursors,
20087 vec![json!({"stream_name": "orders", "through_position": 2})]
20088 );
20089 assert!(completed.message_stream_waits.is_empty());
20090
20091 let replay_history = vec![
20092 opened(1),
20093 first_applied,
20094 opened(2),
20095 applied(2, second_delivery),
20096 ];
20097 for _cold_worker_or_restart in 0..2 {
20098 let replayed = worker()
20099 .execute_workflow_task_decision(workflow_task(
20100 "rust.message-stream",
20101 replay_history.clone(),
20102 DEFAULT_CODEC,
20103 ))
20104 .expect("cold worker replays each logical message exactly once");
20105 assert_eq!(replayed.commands.len(), 1);
20106 assert_eq!(
20107 decode_wire_value(&replayed.commands[0]["result"], DEFAULT_CODEC)
20108 .expect("replayed workflow result"),
20109 json!(["message-1", "message-2"])
20110 );
20111 assert_eq!(
20112 replayed.message_stream_cursors,
20113 vec![json!({"stream_name": "orders", "through_position": 2})]
20114 );
20115 assert!(replayed.message_stream_waits.is_empty());
20116 }
20117 }
20118
20119 #[test]
20120 fn message_stream_capability_and_completion_require_protocol_one_fifteen() {
20121 assert!(!worker_protocol_supports_message_streams("1.14"));
20122 assert!(worker_protocol_supports_message_streams("1.15"));
20123 assert!(worker_protocol_supports_message_streams("1.16"));
20124 assert!(worker_protocol_supports_message_streams(
20125 WORKER_PROTOCOL_VERSION
20126 ));
20127 assert_eq!(MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION, "1.15");
20128 }
20129
20130 #[test]
20131 fn condition_wait_history_cannot_be_consumed_as_a_typed_signal_wait() {
20132 let ctx = workflow_context(vec![
20133 history_event(
20134 "ConditionWaitOpened",
20135 json!({
20136 "sequence": 1,
20137 "condition_wait_id": "condition:1",
20138 "condition_wait_occurrence_id": "rust:condition-wait:0",
20139 "condition_key": "signal:finish",
20140 "condition_definition_fingerprint": "sha256:signal-finish-v1",
20141 }),
20142 ),
20143 history_event(
20144 "ConditionWaitSatisfied",
20145 json!({
20146 "sequence": 1,
20147 "condition_wait_id": "condition:1",
20148 "condition_wait_occurrence_id": "rust:condition-wait:0",
20149 "condition_key": "signal:finish",
20150 "condition_definition_fingerprint": "sha256:signal-finish-v1",
20151 }),
20152 ),
20153 history_event(
20154 "SignalReceived",
20155 json!({"signal_name": "finish", "arguments": []}),
20156 ),
20157 ]);
20158 let mut signal = Box::pin(ctx.wait_signal("finish"));
20159 let mut task_context = TaskContext::from_waker(noop_waker_ref());
20160
20161 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
20162 signal.as_mut().poll(&mut task_context)
20163 else {
20164 panic!("condition history must not resolve as a typed signal wait");
20165 };
20166 assert_eq!(failure.reason, "recorded_command_mismatch");
20167 assert_eq!(failure.expected.as_deref(), Some("condition wait"));
20168 }
20169
20170 #[test]
20171 fn replay_orders_signal_waits_and_timers_in_one_command_stream() {
20172 let signal_then_timer = vec![
20173 history_event(
20174 "SignalWaitOpened",
20175 json!({"sequence": 1, "signal_name": "go"}),
20176 ),
20177 history_event(
20178 "SignalApplied",
20179 json!({
20180 "sequence": 1,
20181 "signal_name": "go",
20182 "value": fixture_envelope(json!(["now"])),
20183 }),
20184 ),
20185 history_event(
20186 "TimerScheduled",
20187 json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
20188 ),
20189 history_event(
20190 "TimerFired",
20191 json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
20192 ),
20193 ];
20194
20195 let ctx = workflow_context(signal_then_timer.clone());
20196 let mut signal = Box::pin(ctx.wait_signal("go"));
20197 let mut task_context = TaskContext::from_waker(noop_waker_ref());
20198 assert!(matches!(
20199 signal.as_mut().poll(&mut task_context),
20200 Poll::Ready(Ok(arguments)) if arguments == vec![json!("now")]
20201 ));
20202 let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5)));
20203 assert!(matches!(
20204 timer.as_mut().poll(&mut task_context),
20205 Poll::Ready(Ok(()))
20206 ));
20207 ctx.ensure_history_consumed()
20208 .expect("signal and timer history consumed in order");
20209
20210 let reordered = workflow_context(signal_then_timer);
20211 let mut timer_first = Box::pin(reordered.sleep(Duration::from_secs(5)));
20212 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
20213 timer_first.as_mut().poll(&mut task_context)
20214 else {
20215 panic!("timer cannot consume signal-wait-first history");
20216 };
20217 assert_eq!(failure.reason, "recorded_command_mismatch");
20218 assert_eq!(failure.sequence, Some(1));
20219 assert_eq!(failure.expected.as_deref(), Some("signal wait"));
20220
20221 let timer_then_signal = vec![
20222 history_event(
20223 "TimerScheduled",
20224 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20225 ),
20226 history_event(
20227 "TimerFired",
20228 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20229 ),
20230 history_event(
20231 "SignalWaitOpened",
20232 json!({"sequence": 2, "signal_name": "go"}),
20233 ),
20234 history_event(
20235 "SignalApplied",
20236 json!({
20237 "sequence": 2,
20238 "signal_name": "go",
20239 "value": fixture_envelope(json!([])),
20240 }),
20241 ),
20242 ];
20243 let reordered = workflow_context(timer_then_signal);
20244 let mut signal_first = Box::pin(reordered.wait_signal("go"));
20245 let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
20246 signal_first.as_mut().poll(&mut task_context)
20247 else {
20248 panic!("signal wait cannot consume timer-first history");
20249 };
20250 assert_eq!(failure.reason, "recorded_command_mismatch");
20251 assert_eq!(failure.sequence, Some(1));
20252 assert_eq!(failure.expected.as_deref(), Some("timer"));
20253 }
20254
20255 #[test]
20256 fn workflow_history_rejects_duplicate_or_colliding_command_sequences() {
20257 let duplicate_timer = WorkflowState::new(
20258 vec![
20259 history_event(
20260 "TimerScheduled",
20261 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20262 ),
20263 history_event(
20264 "TimerScheduled",
20265 json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
20266 ),
20267 ],
20268 "rust-workers".to_string(),
20269 DEFAULT_CODEC.to_string(),
20270 None,
20271 )
20272 .expect_err("one workflow sequence cannot schedule two timers");
20273 assert!(matches!(
20274 duplicate_timer,
20275 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20276 if reason == "timer_schedule_missing_or_duplicate"
20277 ));
20278
20279 let colliding_kinds = WorkflowState::new(
20280 vec![
20281 history_event(
20282 "TimerScheduled",
20283 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20284 ),
20285 history_event(
20286 "ActivityCompleted",
20287 json!({"sequence": 1, "activity_type": "same-sequence"}),
20288 ),
20289 ],
20290 "rust-workers".to_string(),
20291 DEFAULT_CODEC.to_string(),
20292 None,
20293 )
20294 .expect_err("one workflow sequence cannot identify two command kinds");
20295 assert!(matches!(
20296 colliding_kinds,
20297 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20298 if reason == "durable_command_sequence_collision"
20299 ));
20300
20301 let duplicate_signal_wait = WorkflowState::new(
20302 vec![
20303 history_event(
20304 "SignalWaitOpened",
20305 json!({"sequence": 1, "signal_name": "go"}),
20306 ),
20307 history_event(
20308 "SignalWaitOpened",
20309 json!({"sequence": 1, "signal_name": "go"}),
20310 ),
20311 ],
20312 "rust-workers".to_string(),
20313 DEFAULT_CODEC.to_string(),
20314 None,
20315 )
20316 .expect_err("one workflow sequence cannot open two signal waits");
20317 assert!(matches!(
20318 duplicate_signal_wait,
20319 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20320 if reason == "signal_wait_open_missing_or_duplicate"
20321 ));
20322 }
20323
20324 #[test]
20325 fn workflow_history_accepts_a_first_command_after_global_sequence_gaps() {
20326 let result = encode_value_envelope(&json!({"captured": true}), DEFAULT_CODEC)
20327 .expect("side-effect result");
20328 let ctx = workflow_context(vec![history_event(
20329 "SideEffectRecorded",
20330 json!({"sequence": 99, "result": result}),
20331 )]);
20332
20333 let replayed: Value = ctx
20334 .side_effect(|| panic!("recorded side effect must not run"))
20335 .expect("positive global workflow sequence is valid");
20336 assert_eq!(replayed, json!({"captured": true}));
20337 ctx.ensure_history_consumed().expect("history consumed");
20338 }
20339
20340 #[test]
20341 fn workflow_history_rejects_zero_and_descending_command_sequences() {
20342 let result =
20343 encode_value_envelope(&json!("captured"), DEFAULT_CODEC).expect("side-effect result");
20344 let zero = WorkflowState::new(
20345 vec![history_event(
20346 "SideEffectRecorded",
20347 json!({"sequence": 0, "result": result.clone()}),
20348 )],
20349 "rust-workers".to_string(),
20350 DEFAULT_CODEC.to_string(),
20351 None,
20352 )
20353 .expect_err("durable command sequences must be positive");
20354 assert!(matches!(
20355 zero,
20356 Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20357 if reason == "durable_command_sequence_invalid"
20358 ));
20359
20360 let descending = WorkflowState::new(
20361 vec![
20362 history_event(
20363 "SideEffectRecorded",
20364 json!({"sequence": 3, "result": result}),
20365 ),
20366 history_event(
20367 "VersionMarkerRecorded",
20368 json!({
20369 "sequence": 2,
20370 "change_id": "descending-marker",
20371 "version": 1,
20372 "min_supported": 1,
20373 "max_supported": 1,
20374 }),
20375 ),
20376 ],
20377 "rust-workers".to_string(),
20378 DEFAULT_CODEC.to_string(),
20379 None,
20380 )
20381 .expect_err("new durable commands must remain strictly ordered");
20382 let Error::NonDeterministicReplay(failure) = descending else {
20383 panic!("expected typed replay failure");
20384 };
20385 assert_eq!(failure.reason, "durable_command_sequence_mismatch");
20386 assert_eq!(failure.sequence, Some(2));
20387 assert_eq!(
20388 failure.expected.as_deref(),
20389 Some("workflow sequence greater than 3")
20390 );
20391 assert_eq!(failure.actual.as_deref(), Some("2"));
20392 }
20393
20394 #[test]
20395 fn workflow_task_replay_completes_after_signals_create_sequence_gaps() {
20396 fn worker() -> Worker {
20397 let client = Client::new("http://127.0.0.1:8080").expect("client");
20398 let mut worker = Worker::new(client, "rust-workers");
20399 worker.register_workflow("rust.finish-after-gaps", |ctx, _input| async move {
20400 ctx.wait_signal("finish").await?;
20401 let marker: String =
20402 ctx.side_effect(|| panic!("recorded side effect must not run"))?;
20403 assert_eq!(marker, "after-finish");
20404 Ok(json!("finished"))
20405 });
20406 worker
20407 }
20408
20409 let marker = encode_value_envelope(&json!("after-finish"), DEFAULT_CODEC)
20410 .expect("side-effect result");
20411 let task = workflow_task(
20412 "rust.finish-after-gaps",
20413 vec![
20414 history_event(
20415 "SignalWaitOpened",
20416 json!({"sequence": 1, "signal_name": "finish"}),
20417 ),
20418 history_event(
20419 "SignalReceived",
20420 json!({
20421 "signal_id": "increment-3",
20422 "signal_name": "increment",
20423 "workflow_sequence": 2,
20424 "payload_codec": DEFAULT_CODEC,
20425 "arguments": fixture_envelope(json!([3])),
20426 }),
20427 ),
20428 history_event(
20429 "SignalReceived",
20430 json!({
20431 "signal_id": "increment-5",
20432 "signal_name": "increment",
20433 "workflow_sequence": 3,
20434 "payload_codec": DEFAULT_CODEC,
20435 "arguments": fixture_envelope(json!([5])),
20436 }),
20437 ),
20438 history_event(
20439 "SignalReceived",
20440 json!({
20441 "signal_id": "finish",
20442 "signal_name": "finish",
20443 "workflow_sequence": 4,
20444 "payload_codec": DEFAULT_CODEC,
20445 "arguments": fixture_envelope(json!([])),
20446 }),
20447 ),
20448 history_event(
20449 "SignalApplied",
20450 json!({
20451 "sequence": 1,
20452 "signal_id": "finish",
20453 "signal_name": "finish",
20454 "payload_codec": DEFAULT_CODEC,
20455 "value": fixture_envelope(json!([])),
20456 }),
20457 ),
20458 history_event(
20459 "SideEffectRecorded",
20460 json!({"sequence": 5, "result": marker}),
20461 ),
20462 ],
20463 DEFAULT_CODEC,
20464 );
20465
20466 for _original_or_cold_worker in 0..2 {
20467 let commands = worker()
20468 .execute_workflow_task(task.clone())
20469 .expect("signal gaps preserve deterministic replay");
20470 assert_eq!(commands.len(), 1, "replay emits only terminal completion");
20471 assert_eq!(commands[0]["type"], "complete_workflow");
20472 assert_eq!(
20473 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
20474 json!("finished")
20475 );
20476 }
20477 }
20478
20479 #[test]
20480 fn workflow_sleep_rejects_unrepresentable_rounded_duration() {
20481 let ctx = workflow_context(Vec::new());
20482 let mut sleep = Box::pin(ctx.start_timer(Duration::new(u64::MAX, 1)));
20483 let mut task_context = TaskContext::from_waker(noop_waker_ref());
20484 assert!(matches!(
20485 sleep.as_mut().poll(&mut task_context),
20486 Poll::Ready(Err(Error::TimerDurationOverflow))
20487 ));
20488 assert!(ctx.take_commands().expect("commands").is_empty());
20489 }
20490
20491 #[test]
20492 fn workflow_memo_update_emits_canonical_command_and_replays_once() {
20493 let entries = AvroValue::Map(BTreeMap::from([
20494 ("text".to_string(), AvroValue::String("same".to_string())),
20495 (
20496 "nested".to_string(),
20497 AvroValue::Map(BTreeMap::from([
20498 ("beta".to_string(), AvroValue::Long(2)),
20499 ("alpha".to_string(), AvroValue::Long(1)),
20500 ])),
20501 ),
20502 ("long".to_string(), AvroValue::Long(7)),
20503 ("double".to_string(), AvroValue::Double(7.0)),
20504 ("binary".to_string(), AvroValue::Bytes(b"same".to_vec())),
20505 ]));
20506 let ctx = workflow_context(Vec::new());
20507 ctx.upsert_memo(entries.clone()).expect("valid memo update");
20508 let commands = ctx.take_commands().expect("commands");
20509
20510 assert_eq!(commands.len(), 1);
20511 assert_eq!(commands[0]["type"], "upsert_memo");
20512 let server_entries = json!({
20513 "codec": "avro",
20514 "blob": "wwHioz3/VYAiNw4KDGJpbmFyeQgIc2FtZQxkb3VibGUGAAAAAAAAHEAIbG9uZwQODG5lc3RlZA4ECmFscGhhBAIIYmV0YQQEAAh0ZXh0CghzYW1lAA==",
20515 });
20516 assert_eq!(
20517 commands[0]["entries"]
20518 .as_object()
20519 .expect("entries envelope")
20520 .keys()
20521 .collect::<Vec<_>>(),
20522 vec!["blob", "codec"]
20523 );
20524 assert_eq!(commands[0]["entries"], server_entries);
20525 let wire_entries =
20526 decode_wire_avro_value(&commands[0]["entries"], DEFAULT_CODEC).expect("memo entries");
20527 assert_eq!(wire_entries, entries);
20528
20529 let history = vec![history_event(
20530 "MemoUpserted",
20531 json!({
20532 "sequence": 1,
20533 "entries": server_entries.clone(),
20534 "merged": server_entries,
20535 }),
20536 )];
20537 let replay = workflow_context(history.clone());
20538 replay
20539 .upsert_memo(entries.clone())
20540 .expect("matching replay identity");
20541 assert!(replay.take_commands().expect("replay commands").is_empty());
20542
20543 let changed_types = AvroValue::Map(BTreeMap::from([
20544 ("text".to_string(), AvroValue::Bytes(b"same".to_vec())),
20545 (
20546 "nested".to_string(),
20547 AvroValue::Map(BTreeMap::from([
20548 ("alpha".to_string(), AvroValue::Long(1)),
20549 ("beta".to_string(), AvroValue::Long(2)),
20550 ])),
20551 ),
20552 ("long".to_string(), AvroValue::Double(7.0)),
20553 ("double".to_string(), AvroValue::Long(7)),
20554 ("binary".to_string(), AvroValue::String("same".to_string())),
20555 ]));
20556 let error = workflow_context(history)
20557 .upsert_memo(changed_types)
20558 .expect_err("memo replay identity must preserve Avro value types");
20559 assert!(matches!(
20560 error,
20561 Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20562 ));
20563 }
20564
20565 #[test]
20566 fn workflow_memo_update_rejects_changed_replay_identity_and_invalid_keys() {
20567 let original = encode_value_envelope(&json!({"stage": "original"}), DEFAULT_CODEC)
20568 .expect("memo envelope");
20569 let replay = workflow_context(vec![history_event(
20570 "MemoUpserted",
20571 json!({
20572 "sequence": 1,
20573 "entries": original.clone(),
20574 "merged": original
20575 }),
20576 )]);
20577 let error = replay
20578 .upsert_memo(json!({"stage": "changed"}))
20579 .expect_err("changed memo update must fail replay");
20580 assert!(matches!(
20581 error,
20582 Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20583 ));
20584
20585 let invalid = workflow_context(Vec::new())
20586 .upsert_memo(
20587 json!({"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx": true}),
20588 )
20589 .expect_err("oversized key");
20590 assert!(matches!(invalid, Error::InvalidMemoUpdate(_)));
20591 }
20592
20593 #[test]
20594 fn workflow_memo_replay_distinguishes_signed_zero_identity() {
20595 let negative_zero = AvroValue::Map(BTreeMap::from([(
20596 "reading".to_string(),
20597 AvroValue::Double(-0.0),
20598 )]));
20599 let negative_zero_envelope =
20600 encode_typed_envelope(&negative_zero, DEFAULT_CODEC).expect("negative zero envelope");
20601 let history = vec![history_event(
20602 "MemoUpserted",
20603 json!({
20604 "sequence": 1,
20605 "entries": negative_zero_envelope.clone(),
20606 "merged": negative_zero_envelope,
20607 }),
20608 )];
20609
20610 workflow_context(history.clone())
20611 .upsert_memo(negative_zero)
20612 .expect("matching negative-zero history identity");
20613
20614 let error = workflow_context(history)
20615 .upsert_memo(AvroValue::Map(BTreeMap::from([(
20616 "reading".to_string(),
20617 AvroValue::Double(0.0),
20618 )])))
20619 .expect_err("positive zero must not consume negative-zero memo history");
20620 assert!(matches!(
20621 error,
20622 Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20623 ));
20624 }
20625
20626 #[test]
20627 fn workflow_memo_capability_requires_flag_and_command_advertisement() {
20628 let supported = json!({
20629 "workflow_memo_updates": {"supported": true, "minimum_protocol_version": "1.14"},
20630 "supported_workflow_task_commands": ["complete_workflow", "upsert_memo"]
20631 });
20632 assert!(runtime_supports_workflow_memo_updates(Some(&supported)));
20633 assert!(!runtime_supports_workflow_memo_updates(Some(&json!({
20634 "workflow_memo_updates": {"supported": false},
20635 "supported_workflow_task_commands": ["upsert_memo"]
20636 }))));
20637 assert!(commands_use_workflow_memo_updates(&[json!({
20638 "type": "upsert_memo",
20639 "entries": {"stage": "processing"}
20640 })]));
20641 }
20642
20643 #[test]
20644 fn workflow_task_replay_completes_without_rescheduling_recorded_commands() {
20645 let client = Client::new("http://127.0.0.1:8080").expect("client");
20646 let mut worker = Worker::new(client, "rust-workers");
20647 worker.register_workflow("rust.timer", |ctx, _input| async move {
20648 ctx.sleep(Duration::from_secs(5)).await?;
20649 ctx.activity("after-timer", json!([])).await
20650 });
20651
20652 let task = |history_events| WorkflowTask {
20653 task_id: "wft-rust-timer-1".to_string(),
20654 workflow_command_id: None,
20655 workflow_id: Some("wf-rust-timer".to_string()),
20656 run_id: Some("run-rust-timer".to_string()),
20657 workflow_type: "rust.timer".to_string(),
20658 cancel_requested: false,
20659 payload_codec: DEFAULT_CODEC.to_string(),
20660 arguments: Some(
20661 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
20662 ),
20663 history_events,
20664 total_history_events: None,
20665 history_size_bytes: None,
20666 continue_as_new_recommended: None,
20667 history_budget_pressure: None,
20668 next_history_page_token: None,
20669 workflow_task_attempt: 1,
20670 workflow_signal_id: None,
20671 signal_name: None,
20672 signal_arguments: None,
20673 workflow_update_id: None,
20674 update_name: None,
20675 lease_owner: Some("rust-worker".to_string()),
20676 };
20677
20678 let initial = worker
20679 .execute_workflow_task(task(Vec::new()))
20680 .expect("initial timer task");
20681 assert_eq!(
20682 initial,
20683 vec![json!({"type": "start_timer", "delay_seconds": 5})]
20684 );
20685
20686 let activity_result =
20687 encode_value_envelope(&json!("done"), DEFAULT_CODEC).expect("activity result");
20688 let replayed = worker
20689 .execute_workflow_task(task(vec![
20690 history_event(
20691 "TimerScheduled",
20692 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20693 ),
20694 history_event(
20695 "TimerFired",
20696 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20697 ),
20698 history_event(
20699 "ActivityCompleted",
20700 json!({
20701 "sequence": 2,
20702 "activity_type": "after-timer",
20703 "payload_codec": DEFAULT_CODEC,
20704 "result": activity_result,
20705 }),
20706 ),
20707 ]))
20708 .expect("replayed workflow task");
20709 assert_eq!(replayed.len(), 1);
20710 assert_eq!(replayed[0]["type"], "complete_workflow");
20711 assert_eq!(
20712 decode_wire_value(&replayed[0]["result"], DEFAULT_CODEC).expect("result"),
20713 json!("done")
20714 );
20715 }
20716
20717 #[test]
20718 fn workflow_continue_as_new_emits_arguments_type_and_queue_once() {
20719 let client = Client::new("http://127.0.0.1:8080").expect("client");
20720 let mut worker = Worker::new(client, "rust-workers");
20721 worker.register_workflow("rust.continue", |ctx, _input| async move {
20722 ctx.continue_as_new_with_options(
20723 ContinueAsNewOptions::new()
20724 .workflow_type("rust.next")
20725 .task_queue("next-workers"),
20726 json!([2, {"cursor": "next"}]),
20727 )
20728 });
20729
20730 let commands = worker
20731 .execute_workflow_task(workflow_task("rust.continue", Vec::new(), DEFAULT_CODEC))
20732 .expect("continue-as-new command");
20733
20734 assert_eq!(commands.len(), 1);
20735 assert_eq!(commands[0]["type"], "continue_as_new");
20736 assert_eq!(commands[0]["workflow_type"], "rust.next");
20737 assert_eq!(commands[0]["queue"], "next-workers");
20738 assert_eq!(
20739 decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC)
20740 .expect("continue-as-new arguments"),
20741 json!([2, {"cursor": "next"}])
20742 );
20743 }
20744
20745 #[test]
20746 fn continue_as_new_preserves_typed_arguments() {
20747 let client = Client::new("http://127.0.0.1:8080").expect("client");
20748 let mut worker = Worker::new(client, "rust-workers");
20749 worker.register_workflow_avro_value("rust.typed-continue", |ctx, _input| async move {
20750 ctx.continue_as_new(AvroValue::Array(vec![typed_fidelity_probe()]))?;
20751 unreachable!("continue-as-new returns a control-flow error")
20752 });
20753
20754 let commands = worker
20755 .execute_workflow_task(workflow_task(
20756 "rust.typed-continue",
20757 Vec::new(),
20758 DEFAULT_CODEC,
20759 ))
20760 .expect("typed continue-as-new command");
20761
20762 assert_eq!(commands[0]["type"], "continue_as_new");
20763 assert_eq!(
20764 decode_wire_avro_value(&commands[0]["arguments"], DEFAULT_CODEC)
20765 .expect("typed continue arguments"),
20766 AvroValue::Array(vec![typed_fidelity_probe()])
20767 );
20768 }
20769
20770 #[test]
20771 fn recorded_continue_as_new_is_consumed_without_duplicate_successor_command() {
20772 let client = Client::new("http://127.0.0.1:8080").expect("client");
20773 let mut worker = Worker::new(client, "rust-workers");
20774 worker.register_workflow("rust.continue", |ctx, _input| async move {
20775 ctx.continue_as_new(json!([2]))
20776 });
20777 let task = workflow_task(
20778 "rust.continue",
20779 vec![history_event(
20780 "WorkflowContinuedAsNew",
20781 json!({"sequence": 1, "continued_to_run_id": "run-next"}),
20782 )],
20783 DEFAULT_CODEC,
20784 );
20785
20786 for _worker_restart_or_redelivery in 0..2 {
20787 let commands = worker
20788 .execute_workflow_task(task.clone())
20789 .expect("recorded transition replays");
20790 assert!(
20791 commands.is_empty(),
20792 "replay must not emit another successor"
20793 );
20794 }
20795 }
20796
20797 #[test]
20798 fn continue_as_new_rejects_invalid_overrides_before_emitting_a_command() {
20799 let ctx = workflow_context(Vec::new());
20800 let error = ctx
20801 .continue_as_new_with_options(ContinueAsNewOptions::new().task_queue(" "), json!([1]))
20802 .expect_err("blank queue must be rejected");
20803
20804 let Error::InvalidContinueAsNewOptions(error) = error else {
20805 panic!("expected typed continue-as-new validation error");
20806 };
20807 assert_eq!(error.field, "task_queue");
20808 assert!(ctx.take_commands().expect("commands").is_empty());
20809 }
20810
20811 #[test]
20812 fn workflow_context_exposes_server_history_budget() {
20813 let client = Client::new("http://127.0.0.1:8080").expect("client");
20814 let mut worker = Worker::new(client, "rust-workers");
20815 worker.register_workflow("rust.history-budget", |ctx, _input| async move {
20816 let budget = ctx.history_budget()?;
20817 Ok(json!({
20818 "events": budget.event_count,
20819 "bytes": budget.size_bytes,
20820 "recommended": budget.continue_as_new_recommended,
20821 "pressure": budget.pressure,
20822 }))
20823 });
20824 let task: WorkflowTask = serde_json::from_value(json!({
20825 "task_id": "task-history-budget",
20826 "workflow_type": "rust.history-budget",
20827 "payload_codec": DEFAULT_CODEC,
20828 "history_events": [],
20829 "total_history_events": 480,
20830 "history_size_bytes": 1_048_576,
20831 "continue_as_new_recommended": true,
20832 "history_budget_pressure": "continue_as_new_recommended",
20833 }))
20834 .expect("published workflow task");
20835
20836 let commands = worker
20837 .execute_workflow_task(task)
20838 .expect("history-budget workflow");
20839 let result = decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("result");
20840 assert_eq!(result["events"], 480);
20841 assert_eq!(result["bytes"], 1_048_576);
20842 assert_eq!(result["recommended"], true);
20843 assert_eq!(result["pressure"], "continue_as_new_recommended");
20844 }
20845
20846 #[test]
20847 fn uncaught_workflow_handler_error_emits_terminal_failure_command() {
20848 let client = Client::new("http://127.0.0.1:8080").expect("client");
20849 let mut worker = Worker::new(client, "rust-workers");
20850 worker.register_workflow("rust.failing", |_ctx, _input| async move {
20851 Err(Error::Codec("rust_conformance_failure".to_string()))
20852 });
20853 let task = WorkflowTask {
20854 task_id: "wft-rust-failing-1".to_string(),
20855 workflow_command_id: None,
20856 workflow_id: Some("wf-rust-failing".to_string()),
20857 run_id: Some("run-rust-failing".to_string()),
20858 workflow_type: "rust.failing".to_string(),
20859 cancel_requested: false,
20860 payload_codec: DEFAULT_CODEC.to_string(),
20861 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
20862 history_events: Vec::new(),
20863 total_history_events: Some(0),
20864 history_size_bytes: None,
20865 continue_as_new_recommended: None,
20866 history_budget_pressure: None,
20867 next_history_page_token: None,
20868 workflow_task_attempt: 1,
20869 workflow_signal_id: None,
20870 signal_name: None,
20871 signal_arguments: None,
20872 workflow_update_id: None,
20873 update_name: None,
20874 lease_owner: Some("rust-worker".to_string()),
20875 };
20876
20877 let commands = worker
20878 .execute_workflow_task(task)
20879 .expect("handler failure becomes a workflow command");
20880
20881 assert_eq!(commands.len(), 1);
20882 assert_eq!(commands[0]["type"], "fail_workflow");
20883 assert_eq!(commands[0]["exception_type"], "RustWorkflowError");
20884 assert_eq!(commands[0]["exception_class"], "durable_workflow::Error");
20885 assert_eq!(commands[0]["non_retryable"], false);
20886 assert_eq!(
20887 commands[0]["message"],
20888 "codec error: rust_conformance_failure"
20889 );
20890 assert_eq!(
20891 commands[0]["exception"]["message"],
20892 "codec error: rust_conformance_failure"
20893 );
20894 }
20895
20896 #[test]
20897 fn ordinary_handler_error_preserves_commands_queued_in_the_same_decision() {
20898 let client = Client::new("http://127.0.0.1:8080").expect("client");
20899 let mut worker = Worker::new(client, "rust-workers");
20900 worker.register_workflow("rust.failing-after-side-effect", |ctx, _input| async move {
20901 let _: String = ctx.side_effect(|| "captured".to_string())?;
20902 Err(Error::WorkerLoop("application failure".to_string()))
20903 });
20904
20905 let commands = worker
20906 .execute_workflow_task(workflow_task(
20907 "rust.failing-after-side-effect",
20908 Vec::new(),
20909 DEFAULT_CODEC,
20910 ))
20911 .expect("ordinary failure remains a workflow decision");
20912
20913 assert_eq!(commands.len(), 2);
20914 assert_eq!(commands[0]["type"], "record_side_effect");
20915 assert_eq!(commands[1]["type"], "fail_workflow");
20916 }
20917
20918 #[test]
20919 fn handler_error_cannot_hide_an_unconsumed_committed_side_effect() {
20920 let client = Client::new("http://127.0.0.1:8080").expect("client");
20921 let mut worker = Worker::new(client, "rust-workers");
20922 worker.register_workflow("rust.removed-side-effect", |_ctx, _input| async move {
20923 Err(Error::WorkerLoop("application failure".to_string()))
20924 });
20925 let result =
20926 encode_value_envelope(&json!("committed"), DEFAULT_CODEC).expect("side-effect result");
20927
20928 let error = worker
20929 .execute_workflow_task(workflow_task(
20930 "rust.removed-side-effect",
20931 vec![history_event(
20932 "SideEffectRecorded",
20933 json!({"sequence": 1, "result": result}),
20934 )],
20935 DEFAULT_CODEC,
20936 ))
20937 .expect_err("removed committed history must not become fail_workflow");
20938
20939 let Error::NonDeterministicReplay(failure) = error else {
20940 panic!("expected typed replay failure");
20941 };
20942 assert_eq!(failure.reason, "recorded_commands_unconsumed");
20943 assert_eq!(failure.sequence, Some(1));
20944 assert_eq!(failure.expected.as_deref(), Some("side effect"));
20945 }
20946
20947 #[test]
20948 fn replay_error_discards_side_effect_queued_before_incompatible_marker_check() {
20949 let client = Client::new("http://127.0.0.1:8080").expect("client");
20950 let mut worker = Worker::new(client, "rust-workers");
20951 worker.register_workflow(
20952 "rust.side-effect-before-marker-error",
20953 |ctx, _input| async move {
20954 assert_eq!(ctx.get_version("restart-safe", 1, 1)?, 1);
20955 let _: String = ctx.side_effect(|| "must-not-commit".to_string())?;
20956 ctx.get_version("restart-safe", 2, 2)?;
20957 Ok(Value::Null)
20958 },
20959 );
20960
20961 let error = worker
20962 .execute_workflow_task(workflow_task(
20963 "rust.side-effect-before-marker-error",
20964 vec![history_event(
20965 "VersionMarkerRecorded",
20966 json!({
20967 "sequence": 1,
20968 "change_id": "restart-safe",
20969 "version": 1,
20970 "min_supported": 1,
20971 "max_supported": 1,
20972 }),
20973 )],
20974 DEFAULT_CODEC,
20975 ))
20976 .expect_err("replay error must return no queued workflow commands");
20977
20978 let Error::NonDeterministicReplay(failure) = error else {
20979 panic!("expected typed replay failure");
20980 };
20981 assert_eq!(failure.reason, "version_marker_incompatible_range");
20982 assert_eq!(failure.sequence, Some(1));
20983 }
20984
20985 #[test]
20986 fn workflow_task_replay_keeps_recorded_unfired_timer_pending_without_rescheduling() {
20987 let client = Client::new("http://127.0.0.1:8080").expect("client");
20988 let mut worker = Worker::new(client, "rust-workers");
20989 worker.register_workflow("rust.timer.pending", |ctx, _input| async move {
20990 ctx.sleep(Duration::from_secs(5)).await?;
20991 Ok(json!({"status": "timer fired"}))
20992 });
20993
20994 let task = WorkflowTask {
20995 task_id: "wft-rust-timer-pending".to_string(),
20996 workflow_command_id: None,
20997 workflow_id: Some("wf-rust-timer".to_string()),
20998 run_id: Some("run-rust-timer".to_string()),
20999 workflow_type: "rust.timer.pending".to_string(),
21000 cancel_requested: false,
21001 payload_codec: DEFAULT_CODEC.to_string(),
21002 arguments: Some(
21003 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
21004 ),
21005 history_events: vec![history_event(
21006 "TimerScheduled",
21007 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
21008 )],
21009 total_history_events: Some(1),
21010 history_size_bytes: None,
21011 continue_as_new_recommended: None,
21012 history_budget_pressure: None,
21013 next_history_page_token: None,
21014 workflow_task_attempt: 1,
21015 workflow_signal_id: None,
21016 signal_name: None,
21017 signal_arguments: None,
21018 workflow_update_id: None,
21019 update_name: None,
21020 lease_owner: Some("rust-worker".to_string()),
21021 };
21022
21023 for _redelivery_or_restart in 0..2 {
21024 let commands = worker
21025 .execute_workflow_task(task.clone())
21026 .expect("recorded timer remains pending");
21027 assert!(
21028 commands.is_empty(),
21029 "recorded timer must not be rescheduled"
21030 );
21031 }
21032 }
21033
21034 #[test]
21035 fn workflow_task_rejects_recorded_command_removed_from_workflow_code() {
21036 let client = Client::new("http://127.0.0.1:8080").expect("client");
21037 let mut worker = Worker::new(client, "rust-workers");
21038 worker.register_workflow("rust.timer.removed", |_ctx, _input| async move {
21039 Ok(json!({"status": "completed"}))
21040 });
21041 let task = WorkflowTask {
21042 task_id: "wft-rust-timer-removed".to_string(),
21043 workflow_command_id: None,
21044 workflow_id: Some("wf-rust-timer".to_string()),
21045 run_id: Some("run-rust-timer".to_string()),
21046 workflow_type: "rust.timer.removed".to_string(),
21047 cancel_requested: false,
21048 payload_codec: DEFAULT_CODEC.to_string(),
21049 arguments: Some(
21050 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
21051 ),
21052 history_events: vec![
21053 history_event(
21054 "TimerScheduled",
21055 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
21056 ),
21057 history_event(
21058 "TimerFired",
21059 json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
21060 ),
21061 ],
21062 total_history_events: Some(2),
21063 history_size_bytes: None,
21064 continue_as_new_recommended: None,
21065 history_budget_pressure: None,
21066 next_history_page_token: None,
21067 workflow_task_attempt: 1,
21068 workflow_signal_id: None,
21069 signal_name: None,
21070 signal_arguments: None,
21071 workflow_update_id: None,
21072 update_name: None,
21073 lease_owner: Some("rust-worker".to_string()),
21074 };
21075
21076 let Error::NonDeterministicReplay(failure) = worker
21077 .execute_workflow_task(task)
21078 .expect_err("removed timer must fail replay")
21079 else {
21080 panic!("expected typed replay failure");
21081 };
21082 assert_eq!(failure.reason, "recorded_commands_unconsumed");
21083 assert_eq!(failure.sequence, Some(1));
21084 }
21085
21086 #[test]
21087 fn workflow_context_emits_explicit_child_workflow_contract() {
21088 let ctx = WorkflowContext {
21089 state: Arc::new(Mutex::new(
21090 WorkflowState::new_with_identity(
21091 Vec::new(),
21092 Some("wf-parent".to_string()),
21093 Some("run-parent".to_string()),
21094 "parent-workers".to_string(),
21095 DEFAULT_CODEC.to_string(),
21096 None,
21097 )
21098 .expect("workflow state"),
21099 )),
21100 };
21101 let options = ChildWorkflowOptions::new("python-workers")
21102 .parent_close_policy(ParentClosePolicy::RequestCancel)
21103 .retry_policy(ChildWorkflowRetryPolicy {
21104 max_attempts: Some(3),
21105 backoff_seconds: vec![1, 5],
21106 non_retryable_error_types: vec!["ValidationError".to_string()],
21107 })
21108 .execution_timeout_seconds(600)
21109 .run_timeout_seconds(120);
21110 let mut call = Box::pin(ctx.start_child_workflow(
21111 "python.fulfil-order",
21112 options,
21113 json!([{"order_id": "order-42"}]),
21114 ));
21115 let mut task_context = TaskContext::from_waker(noop_waker_ref());
21116
21117 assert!(matches!(
21118 call.as_mut().poll(&mut task_context),
21119 Poll::Pending
21120 ));
21121 let commands = ctx.take_commands().expect("commands");
21122 assert_eq!(commands.len(), 1);
21123 let command = &commands[0];
21124 assert_eq!(command["type"], "start_child_workflow");
21125 assert_eq!(command["workflow_type"], "python.fulfil-order");
21126 assert_eq!(command["queue"], "python-workers");
21127 assert_eq!(command["parent_close_policy"], "request_cancel");
21128 assert_eq!(command["retry_policy"]["max_attempts"], 3);
21129 assert_eq!(command["execution_timeout_seconds"], 600);
21130 assert_eq!(command["run_timeout_seconds"], 120);
21131 assert_eq!(
21132 decode_wire_value(&command["arguments"], DEFAULT_CODEC).expect("child args"),
21133 json!([{"order_id": "order-42"}])
21134 );
21135 }
21136
21137 fn child_parent_worker() -> Worker {
21138 let client = Client::new("http://127.0.0.1:8080").expect("client");
21139 let mut worker = Worker::new(client, "rust-parent-workers");
21140 worker.register_workflow("rust.parent", |ctx, _input| async move {
21141 let child = ctx
21142 .start_child_workflow(
21143 "python.child",
21144 ChildWorkflowOptions::new("python-child-workers")
21145 .parent_close_policy(ParentClosePolicy::Terminate),
21146 json!([{"codec_probe": [1, true, "rust"]}]),
21147 )
21148 .await?;
21149 Ok(json!({
21150 "parent_workflow_id": child.parent.workflow_id,
21151 "parent_run_id": child.parent.run_id,
21152 "child_workflow_id": child.child.workflow_id,
21153 "child_run_id": child.child.run_id,
21154 "child_workflow_type": child.child_workflow_type,
21155 "result": child.result,
21156 }))
21157 });
21158 worker
21159 }
21160
21161 fn child_parent_task(event_type: &str, payload: Value) -> WorkflowTask {
21162 WorkflowTask {
21163 task_id: "wft-child-parent".to_string(),
21164 workflow_command_id: None,
21165 workflow_id: Some("wf-parent".to_string()),
21166 run_id: Some("run-parent".to_string()),
21167 workflow_type: "rust.parent".to_string(),
21168 cancel_requested: false,
21169 payload_codec: DEFAULT_CODEC.to_string(),
21170 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21171 history_events: vec![
21172 HistoryEvent {
21173 event_type: "ChildWorkflowScheduled".to_string(),
21174 payload: json!({
21175 "sequence": 1,
21176 "child_call_id": "call-child",
21177 "child_workflow_instance_id": "wf-child",
21178 "child_workflow_run_id": "run-child",
21179 "child_workflow_type": "python.child",
21180 }),
21181 raw: HashMap::new(),
21182 },
21183 HistoryEvent {
21184 event_type: event_type.to_string(),
21185 payload,
21186 raw: HashMap::new(),
21187 },
21188 ],
21189 total_history_events: Some(2),
21190 history_size_bytes: None,
21191 continue_as_new_recommended: None,
21192 history_budget_pressure: None,
21193 next_history_page_token: None,
21194 workflow_task_attempt: 1,
21195 workflow_signal_id: None,
21196 signal_name: None,
21197 signal_arguments: None,
21198 workflow_update_id: None,
21199 update_name: None,
21200 lease_owner: Some("rust-worker".to_string()),
21201 }
21202 }
21203
21204 #[test]
21205 fn committed_child_result_replays_without_starting_a_duplicate() {
21206 let worker = child_parent_worker();
21207 let task = child_parent_task(
21208 "ChildRunCompleted",
21209 json!({
21210 "sequence": 1,
21211 "child_call_id": "call-child",
21212 "child_workflow_instance_id": "wf-child",
21213 "child_workflow_run_id": "run-child",
21214 "child_workflow_type": "python.child",
21215 "payload_codec": DEFAULT_CODEC,
21216 "result": fixture_envelope(json!({"from":"python","ok":true})),
21217 }),
21218 );
21219
21220 for _restart in 0..2 {
21221 let commands = worker
21222 .execute_workflow_task(task.clone())
21223 .expect("replayed parent task");
21224 assert_eq!(commands.len(), 1);
21225 assert_eq!(commands[0]["type"], "complete_workflow");
21226 assert!(!commands
21227 .iter()
21228 .any(|command| command["type"] == "start_child_workflow"));
21229 let output =
21230 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
21231 assert_eq!(output["parent_workflow_id"], "wf-parent");
21232 assert_eq!(output["parent_run_id"], "run-parent");
21233 assert_eq!(output["child_workflow_id"], "wf-child");
21234 assert_eq!(output["child_run_id"], "run-child");
21235 assert_eq!(output["result"], json!({"from": "python", "ok": true}));
21236 }
21237 }
21238
21239 #[test]
21240 fn typed_child_arguments_and_results_survive_replay() {
21241 let client = Client::new("http://127.0.0.1:8080").expect("client");
21242 let mut worker = Worker::new(client, "rust-parent-workers");
21243 worker.register_workflow_avro_value("rust.typed-parent", |ctx, _input| async move {
21244 let child = ctx
21245 .start_child_workflow_avro_value(
21246 "python.typed-child",
21247 ChildWorkflowOptions::new("python-workers"),
21248 AvroValue::Array(vec![typed_fidelity_probe()]),
21249 )
21250 .await?;
21251 Ok(child.result)
21252 });
21253
21254 let initial = worker
21255 .execute_workflow_task(workflow_task(
21256 "rust.typed-parent",
21257 Vec::new(),
21258 DEFAULT_CODEC,
21259 ))
21260 .expect("typed child start");
21261 assert_eq!(initial[0]["type"], "start_child_workflow");
21262 assert_eq!(
21263 decode_wire_avro_value(&initial[0]["arguments"], DEFAULT_CODEC)
21264 .expect("typed child arguments"),
21265 AvroValue::Array(vec![typed_fidelity_probe()])
21266 );
21267
21268 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
21269 .expect("typed child result");
21270 let task = workflow_task(
21271 "rust.typed-parent",
21272 vec![
21273 history_event(
21274 "ChildWorkflowScheduled",
21275 json!({
21276 "sequence": 1,
21277 "child_call_id": "call-typed",
21278 "child_workflow_instance_id": "wf-child",
21279 "child_workflow_run_id": "run-child",
21280 "child_workflow_type": "python.typed-child",
21281 }),
21282 ),
21283 history_event(
21284 "ChildRunCompleted",
21285 json!({
21286 "sequence": 1,
21287 "child_call_id": "call-typed",
21288 "child_workflow_instance_id": "wf-child",
21289 "child_workflow_run_id": "run-child",
21290 "child_workflow_type": "python.typed-child",
21291 "payload_codec": DEFAULT_CODEC,
21292 "result": result,
21293 }),
21294 ),
21295 ],
21296 DEFAULT_CODEC,
21297 );
21298
21299 let commands = worker
21300 .execute_workflow_task(task)
21301 .expect("typed child replay");
21302 assert_eq!(commands[0]["type"], "complete_workflow");
21303 assert_eq!(
21304 decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
21305 .expect("typed parent result"),
21306 typed_fidelity_probe()
21307 );
21308 }
21309
21310 #[test]
21311 fn pending_child_replays_after_restart_without_starting_a_duplicate() {
21312 let worker = child_parent_worker();
21313 let mut task = child_parent_task("unused", Value::Null);
21314 task.history_events.truncate(1);
21315 task.total_history_events = Some(1);
21316
21317 for _redelivery_or_restart in 0..2 {
21318 let commands = worker
21319 .execute_workflow_task(task.clone())
21320 .expect("recorded child remains pending");
21321 assert!(
21322 commands.is_empty(),
21323 "recorded pending child must not be started again"
21324 );
21325 }
21326 }
21327
21328 #[test]
21329 fn child_cancellation_becomes_stable_parent_failure_command() {
21330 let worker = child_parent_worker();
21331 let task = child_parent_task(
21332 "ChildRunCancelled",
21333 json!({
21334 "sequence": 1,
21335 "child_workflow_instance_id": "wf-child",
21336 "child_workflow_run_id": "run-child",
21337 "child_workflow_type": "python.child",
21338 "failure_id": "failure-child",
21339 "failure_category": "cancelled",
21340 "message": "cancelled by parent-close policy",
21341 }),
21342 );
21343
21344 let commands = worker
21345 .execute_workflow_task(task)
21346 .expect("parent settlement");
21347 assert_eq!(commands.len(), 1);
21348 assert_eq!(commands[0]["type"], "fail_workflow");
21349 assert_eq!(commands[0]["exception_type"], "ChildWorkflowCancelled");
21350 assert_eq!(
21351 commands[0]["exception"]["properties"]["reason"],
21352 "cancelled"
21353 );
21354 assert_eq!(
21355 commands[0]["exception"]["properties"]["child_workflow_run_id"],
21356 "run-child"
21357 );
21358 }
21359
21360 #[test]
21361 fn workflow_can_handle_typed_child_failure() {
21362 let client = Client::new("http://127.0.0.1:8080").expect("client");
21363 let mut worker = Worker::new(client, "rust-parent-workers");
21364 worker.register_workflow("rust.handled-parent", |ctx, _input| async move {
21365 match ctx
21366 .start_child_workflow(
21367 "python.child",
21368 ChildWorkflowOptions::new("python-child-workers"),
21369 json!([]),
21370 )
21371 .await
21372 {
21373 Err(Error::ChildWorkflowFailed(failure)) => Ok(json!({
21374 "reason": failure.reason,
21375 "failure_id": failure.failure_id,
21376 "exception_class": failure.exception_class,
21377 "child_run_id": failure.child_workflow_run_id,
21378 })),
21379 Err(error) => Err(error),
21380 Ok(_) => Err(Error::WorkerLoop(
21381 "child unexpectedly succeeded".to_string(),
21382 )),
21383 }
21384 });
21385 let mut task = child_parent_task(
21386 "ChildRunFailed",
21387 json!({
21388 "sequence": 1,
21389 "child_workflow_instance_id": "wf-child",
21390 "child_workflow_run_id": "run-child",
21391 "child_workflow_type": "python.child",
21392 "failure_id": "failure-child",
21393 "failure_category": "child_workflow",
21394 "message": "payment rejected",
21395 "exception": {
21396 "type": "PaymentRejected",
21397 "class": "payments.PaymentRejected",
21398 "message": "payment rejected"
21399 }
21400 }),
21401 );
21402 task.workflow_type = "rust.handled-parent".to_string();
21403
21404 let commands = worker.execute_workflow_task(task).expect("handled failure");
21405 assert_eq!(commands[0]["type"], "complete_workflow");
21406 let output =
21407 decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
21408 assert_eq!(output["reason"], "child_workflow");
21409 assert_eq!(output["failure_id"], "failure-child");
21410 assert_eq!(output["exception_class"], "payments.PaymentRejected");
21411 assert_eq!(output["child_run_id"], "run-child");
21412 }
21413
21414 #[test]
21415 fn rust_hello_world_uses_signal_arguments_from_resume_payload() {
21416 let client = Client::new("http://127.0.0.1:8080").expect("client");
21417 let mut worker = Worker::new(client, "rust-workers");
21418
21419 worker.register_workflow("rust.hello_workflow", |ctx, _input| async move {
21420 let signal = ctx.wait_signal("start").await?;
21421 let name = signal
21422 .first()
21423 .and_then(|value| value.as_str())
21424 .unwrap_or("world");
21425 let greeting = ctx.activity("rust.hello_activity", json!([name])).await?;
21426 Ok(json!({
21427 "greeting": greeting,
21428 "language": "rust"
21429 }))
21430 });
21431
21432 let signal_arguments =
21433 encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC).expect("signal arguments");
21434 let task = WorkflowTask {
21435 task_id: "wft-rust-signal-1".to_string(),
21436 workflow_command_id: None,
21437 workflow_id: Some("wf-rust-hello".to_string()),
21438 run_id: Some("run-rust-hello".to_string()),
21439 workflow_type: "rust.hello_workflow".to_string(),
21440 cancel_requested: false,
21441 payload_codec: DEFAULT_CODEC.to_string(),
21442 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21443 history_events: vec![HistoryEvent {
21444 event_type: "SignalReceived".to_string(),
21445 payload: json!({
21446 "signal_id": "sig-rust-1",
21447 "signal_name": "start"
21448 }),
21449 raw: HashMap::new(),
21450 }],
21451 total_history_events: Some(1),
21452 history_size_bytes: None,
21453 continue_as_new_recommended: None,
21454 history_budget_pressure: None,
21455 next_history_page_token: None,
21456 workflow_task_attempt: 1,
21457 workflow_signal_id: Some("sig-rust-1".to_string()),
21458 signal_name: Some("start".to_string()),
21459 signal_arguments: Some(signal_arguments),
21460 workflow_update_id: None,
21461 update_name: None,
21462 lease_owner: Some("rust-worker".to_string()),
21463 };
21464
21465 let commands = worker.execute_workflow_task(task).expect("workflow task");
21466
21467 assert_eq!(commands.len(), 1);
21468 assert_eq!(commands[0]["type"], "schedule_activity");
21469 assert_eq!(commands[0]["activity_type"], "rust.hello_activity");
21470 assert_eq!(
21471 decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC).expect("activity args"),
21472 json!(["Rust"])
21473 );
21474 }
21475
21476 #[test]
21477 fn workflow_task_appends_paginated_history_events() {
21478 let mut task = WorkflowTask {
21479 task_id: "wft-rust-pages-1".to_string(),
21480 workflow_command_id: None,
21481 workflow_id: Some("wf-rust-pages".to_string()),
21482 run_id: Some("run-rust-pages".to_string()),
21483 workflow_type: "rust.hello_workflow".to_string(),
21484 cancel_requested: false,
21485 payload_codec: DEFAULT_CODEC.to_string(),
21486 arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21487 history_events: vec![HistoryEvent {
21488 event_type: "WorkflowStarted".to_string(),
21489 payload: json!({}),
21490 raw: HashMap::new(),
21491 }],
21492 total_history_events: Some(3),
21493 history_size_bytes: None,
21494 continue_as_new_recommended: None,
21495 history_budget_pressure: None,
21496 next_history_page_token: Some("MQ==".to_string()),
21497 workflow_task_attempt: 1,
21498 workflow_signal_id: None,
21499 signal_name: None,
21500 signal_arguments: None,
21501 workflow_update_id: None,
21502 update_name: None,
21503 lease_owner: Some("rust-worker".to_string()),
21504 };
21505
21506 task.append_history_page(WorkflowTaskHistoryPage {
21507 history_events: vec![
21508 HistoryEvent {
21509 event_type: "SignalReceived".to_string(),
21510 payload: json!({
21511 "signal_id": "sig-rust-1",
21512 "signal_name": "start",
21513 "arguments": encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC)
21514 .expect("signal arguments")
21515 }),
21516 raw: HashMap::new(),
21517 },
21518 HistoryEvent {
21519 event_type: "MarkerRecorded".to_string(),
21520 payload: json!({"sequence": 3}),
21521 raw: HashMap::new(),
21522 },
21523 ],
21524 total_history_events: Some(3),
21525 next_history_page_token: None,
21526 });
21527
21528 assert_eq!(task.history_events.len(), 3);
21529 assert_eq!(task.total_history_events, Some(3));
21530 assert_eq!(task.next_history_page_token, None);
21531
21532 let signal = task
21533 .history_events
21534 .iter()
21535 .find(|event| event.event_type == "SignalReceived")
21536 .expect("signal event");
21537 assert_eq!(
21538 decode_signal_event_arguments(signal, DEFAULT_CODEC).expect("signal arguments"),
21539 vec![AvroValue::String("Rust".to_string())]
21540 );
21541 }
21542
21543 #[tokio::test]
21544 async fn query_handler_reads_ordered_cross_codec_signals_without_commands() {
21545 let client = Client::new("http://127.0.0.1:8080").expect("client");
21546 let mut worker = Worker::new(client, "rust-workers");
21547 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21548 worker.register_query("counter", "current", |ctx, _args| async move {
21549 let mut count = 0_i64;
21550 for signal in ctx.signal_events() {
21551 let value = signal
21552 .arguments
21553 .first()
21554 .and_then(Value::as_i64)
21555 .unwrap_or_default();
21556 match signal.name.as_str() {
21557 "increment" => count += value,
21558 "set" => count = value,
21559 _ => {}
21560 }
21561 }
21562 Ok(json!(count))
21563 });
21564
21565 let task = QueryTask {
21566 query_task_id: "query-rust-counter".to_string(),
21567 query_task_attempt: 1,
21568 lease_owner: Some("rust-worker".to_string()),
21569 workflow_id: Some("counter-1".to_string()),
21570 run_id: Some("run-counter-1".to_string()),
21571 workflow_type: "counter".to_string(),
21572 query_name: "current".to_string(),
21573 payload_codec: DEFAULT_CODEC.to_string(),
21574 workflow_arguments: Some(
21575 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
21576 ),
21577 query_arguments: Some(
21578 encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("query arguments"),
21579 ),
21580 history_events: vec![
21581 HistoryEvent {
21582 event_type: "SignalReceived".to_string(),
21583 payload: json!({
21584 "signal_id": "php-signal-1",
21585 "signal_name": "increment",
21586 "workflow_sequence": 1,
21587 "payload_codec": DEFAULT_CODEC,
21588 "arguments": encode_value_envelope(&json!([3]), DEFAULT_CODEC).expect("php avro signal")
21589 }),
21590 raw: HashMap::new(),
21591 },
21592 HistoryEvent {
21593 event_type: "SignalReceived".to_string(),
21594 payload: json!({
21595 "signal_id": "python-signal-2",
21596 "signal_name": "increment",
21597 "workflow_sequence": 2,
21598 "payload_codec": DEFAULT_CODEC,
21599 "arguments": encode_value_envelope(&json!([5]), DEFAULT_CODEC).expect("python avro signal")
21600 }),
21601 raw: HashMap::new(),
21602 },
21603 HistoryEvent {
21604 event_type: "SignalReceived".to_string(),
21605 payload: json!({
21606 "signal_id": "rust-signal-3",
21607 "signal_name": "set",
21608 "workflow_sequence": 3,
21609 "payload_codec": DEFAULT_CODEC,
21610 "arguments": encode_value_envelope(&json!([0]), DEFAULT_CODEC).expect("rust avro signal")
21611 }),
21612 raw: HashMap::new(),
21613 },
21614 ],
21615 history_export: None,
21616 run_status: Some("completed".to_string()),
21617 };
21618
21619 let result = worker.execute_query_task(task).await.expect("query result");
21620 assert_eq!(result.into_json().expect("query projection"), json!(0));
21621 }
21622
21623 #[tokio::test]
21624 async fn replayed_queries_read_running_completed_and_cold_restarted_instance_state() {
21625 let worker = replay_counter_worker();
21626 let running_history = json!([
21627 {
21628 "type": "ActivityCompleted",
21629 "payload": {
21630 "sequence": 1,
21631 "activity_type": "load-counter",
21632 "payload_codec": DEFAULT_CODEC,
21633 "result": fixture_envelope(json!("loaded"))
21634 }
21635 },
21636 {
21637 "type": "SignalWaitOpened",
21638 "payload": {
21639 "sequence": 3,
21640 "signal_name": "increment"
21641 }
21642 },
21643 {
21644 "type": "SignalReceived",
21645 "payload": {
21646 "signal_id": "signal-3",
21647 "signal_name": "increment",
21648 "workflow_sequence": 2,
21649 "payload_codec": DEFAULT_CODEC,
21650 "arguments": fixture_envelope(json!([3]))
21651 }
21652 },
21653 {
21654 "type": "SignalApplied",
21655 "payload": {
21656 "sequence": 3,
21657 "signal_id": "signal-3",
21658 "signal_name": "increment",
21659 "payload_codec": DEFAULT_CODEC,
21660 "value": fixture_envelope(json!([3]))
21661 }
21662 }
21663 ]);
21664
21665 let running = worker
21666 .execute_query_task(replay_counter_query(
21667 "current",
21668 running_history.clone(),
21669 "running",
21670 ))
21671 .await
21672 .expect("running replay query");
21673 assert_eq!(
21674 running.clone().into_json().expect("query projection"),
21675 json!({"loaded": "loaded", "count": 3, "finished": false})
21676 );
21677
21678 let detached = worker
21679 .execute_query_task(replay_counter_query(
21680 "detached-mutation",
21681 running_history.clone(),
21682 "running",
21683 ))
21684 .await
21685 .expect("query mutates only its detached state clone");
21686 assert_eq!(detached.into_json().expect("query projection"), json!(999));
21687 let failed = worker
21688 .execute_query_task(replay_counter_query(
21689 "failed-mutation",
21690 running_history.clone(),
21691 "running",
21692 ))
21693 .await
21694 .expect_err("failed query");
21695 assert_eq!(failed.reason, "query_rejected");
21696 let unchanged = worker
21697 .execute_query_task(replay_counter_query("current", running_history, "running"))
21698 .await
21699 .expect("later query reconstructs unchanged state");
21700 assert_eq!(unchanged, running);
21701
21702 let restarted_worker = replay_counter_worker();
21703 let empty_arguments = fixture_envelope(json!([]));
21704 let loaded_result = fixture_envelope(json!("loaded"));
21705 let signal_three = fixture_blob(json!([3]));
21706 let signal_five = fixture_blob(json!([5]));
21707 let restarted_task: QueryTask = serde_json::from_value(json!({
21708 "query_task_id": "query-after-restart",
21709 "workflow_id": "counter-1",
21710 "run_id": "run-counter-1",
21711 "workflow_type": "replay-counter",
21712 "query_name": "current",
21713 "payload_codec": DEFAULT_CODEC,
21714 "workflow_arguments": empty_arguments.clone(),
21715 "query_arguments": empty_arguments,
21716 "history_events": [],
21717 "history_export": {
21718 "payloads": {"codec": DEFAULT_CODEC},
21719 "history_events": [
21720 {
21721 "type": "ActivityCompleted",
21722 "payload": {
21723 "sequence": 1,
21724 "activity_type": "load-counter",
21725 "payload_codec": DEFAULT_CODEC,
21726 "result": null
21727 }
21728 },
21729 {
21730 "type": "SignalWaitOpened",
21731 "payload": {
21732 "sequence": 3,
21733 "signal_name": "increment"
21734 }
21735 },
21736 {
21737 "type": "SignalReceived",
21738 "payload": {
21739 "signal_id": "signal-3",
21740 "signal_name": "increment",
21741 "workflow_sequence": 2
21742 }
21743 },
21744 {
21745 "type": "SignalApplied",
21746 "payload": {
21747 "sequence": 3,
21748 "signal_id": "signal-3",
21749 "signal_name": "increment"
21750 }
21751 },
21752 {
21753 "type": "SignalWaitOpened",
21754 "payload": {
21755 "sequence": 5,
21756 "signal_name": "increment"
21757 }
21758 },
21759 {
21760 "type": "SignalReceived",
21761 "payload": {
21762 "signal_id": "signal-5",
21763 "signal_name": "increment",
21764 "workflow_sequence": 4
21765 }
21766 },
21767 {
21768 "type": "SignalApplied",
21769 "payload": {
21770 "sequence": 5,
21771 "signal_id": "signal-5",
21772 "signal_name": "increment"
21773 }
21774 }
21775 ],
21776 "activities": [{
21777 "sequence": 1,
21778 "activity_type": "load-counter",
21779 "payload_codec": DEFAULT_CODEC,
21780 "result": loaded_result
21781 }],
21782 "signals": [
21783 {
21784 "id": "signal-3",
21785 "name": "increment",
21786 "workflow_sequence": 2,
21787 "payload_codec": DEFAULT_CODEC,
21788 "arguments": signal_three
21789 },
21790 {
21791 "id": "signal-5",
21792 "name": "increment",
21793 "workflow_sequence": 4,
21794 "payload_codec": DEFAULT_CODEC,
21795 "arguments": signal_five
21796 }
21797 ]
21798 },
21799 "run_status": "completed"
21800 }))
21801 .expect("cold replay query task");
21802 let completed = restarted_worker
21803 .execute_query_task(restarted_task)
21804 .await
21805 .expect("completed cold replay query");
21806 assert_eq!(
21807 completed.into_json().expect("query projection"),
21808 json!({"loaded": "loaded", "count": 8, "finished": true})
21809 );
21810 }
21811
21812 #[tokio::test]
21813 async fn replayed_query_replay_failures_are_machine_readable() {
21814 let worker = replay_counter_worker();
21815 let task = replay_counter_query(
21816 "current",
21817 json!([{
21818 "type": "ActivityCompleted",
21819 "payload": {
21820 "sequence": 1,
21821 "payload_codec": DEFAULT_CODEC,
21822 "result": {"codec": DEFAULT_CODEC, "blob": "{"}
21823 }
21824 }]),
21825 "running",
21826 );
21827 let failure = worker
21828 .execute_query_task(task)
21829 .await
21830 .expect_err("invalid replay history payload");
21831 assert_eq!(failure.reason, "query_payload_decode_failed");
21832 assert_eq!(failure.failure_type, "QueryPayloadDecodeFailed");
21833 assert!(failure.message.contains("invalid_payload_framing"));
21834 }
21835
21836 #[tokio::test]
21837 async fn query_task_restores_compact_history_from_export() {
21838 let client = Client::new("http://127.0.0.1:8080").expect("client");
21839 let mut worker = Worker::new(client, "rust-workers");
21840 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21841 worker.register_query("counter", "current", |ctx, _args| async move {
21842 Ok(json!(ctx.signals("increment")[0][0]))
21843 });
21844 let empty_arguments = fixture_envelope(json!([]));
21845 let exported_signal = fixture_blob(json!([9]));
21846 let task: QueryTask = serde_json::from_value(json!({
21847 "query_task_id": "query-export",
21848 "workflow_type": "counter",
21849 "query_name": "current",
21850 "payload_codec": DEFAULT_CODEC,
21851 "workflow_arguments": empty_arguments.clone(),
21852 "query_arguments": empty_arguments,
21853 "history_events": [],
21854 "history_export": {
21855 "payloads": {"codec": DEFAULT_CODEC},
21856 "history_events": [{
21857 "type": "SignalReceived",
21858 "payload": {"signal_id": "signal-export", "signal_name": "increment"}
21859 }],
21860 "signals": [{
21861 "id": "signal-export",
21862 "name": "increment",
21863 "status": "applied",
21864 "workflow_sequence": 1,
21865 "payload_codec": DEFAULT_CODEC,
21866 "arguments": exported_signal
21867 }]
21868 }
21869 }))
21870 .expect("query task");
21871
21872 let result = worker.execute_query_task(task).await.expect("query result");
21873 assert_eq!(result.into_json().expect("query projection"), json!(9));
21874 }
21875
21876 #[tokio::test]
21877 async fn query_task_failures_have_stable_reasons() {
21878 let client = Client::new("http://127.0.0.1:8080").expect("client");
21879 let mut worker = Worker::new(client, "rust-workers");
21880 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21881 worker.register_query(
21882 "counter",
21883 "current",
21884 |_ctx, _args| async move { Ok(json!(0)) },
21885 );
21886
21887 let base_task = QueryTask {
21888 query_task_id: "query-errors".to_string(),
21889 query_task_attempt: 1,
21890 lease_owner: None,
21891 workflow_id: Some("counter-errors".to_string()),
21892 run_id: Some("run-errors".to_string()),
21893 workflow_type: "counter".to_string(),
21894 query_name: "missing".to_string(),
21895 payload_codec: DEFAULT_CODEC.to_string(),
21896 workflow_arguments: Some(fixture_envelope(json!([]))),
21897 query_arguments: Some(fixture_envelope(json!([]))),
21898 history_events: Vec::new(),
21899 history_export: None,
21900 run_status: Some("running".to_string()),
21901 };
21902
21903 let unknown = worker
21904 .execute_query_task(base_task.clone())
21905 .await
21906 .expect_err("unknown query");
21907 assert_eq!(unknown.reason, "rejected_unknown_query");
21908
21909 let mut malformed = base_task;
21910 malformed.query_name = "current".to_string();
21911 malformed.query_arguments = Some(json!({"codec": DEFAULT_CODEC, "blob": "{"}));
21912 let malformed = worker
21913 .execute_query_task(malformed)
21914 .await
21915 .expect_err("malformed payload");
21916 assert_eq!(malformed.reason, "query_payload_decode_failed");
21917
21918 let client = Client::new("http://127.0.0.1:8080").expect("client");
21919 let mut unavailable_worker = Worker::new(client, "rust-workers");
21920 unavailable_worker
21921 .register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21922 let empty_arguments = fixture_envelope(json!([]));
21923 let unavailable_task: QueryTask = serde_json::from_value(json!({
21924 "query_task_id": "query-unavailable",
21925 "workflow_type": "counter",
21926 "query_name": "current",
21927 "payload_codec": DEFAULT_CODEC,
21928 "workflow_arguments": empty_arguments.clone(),
21929 "query_arguments": empty_arguments
21930 }))
21931 .expect("query task");
21932 let unavailable = unavailable_worker
21933 .execute_query_task(unavailable_task)
21934 .await
21935 .expect_err("query handler unavailable");
21936 assert_eq!(unavailable.reason, "query_handler_unavailable");
21937 }
21938
21939 #[tokio::test]
21940 async fn client_query_decodes_result_and_typed_failure() {
21941 let server = MockWorkerServer::start();
21942 let client = Client::builder(server.base_url())
21943 .timeout(Duration::from_secs(2))
21944 .build()
21945 .expect("client");
21946
21947 let result = client
21948 .query_workflow("counter-1", "current", json!([]))
21949 .await
21950 .expect("query result");
21951 assert_eq!(result, json!({"count": 8}));
21952
21953 let error = client
21954 .query_workflow("counter-1", "missing", json!([]))
21955 .await
21956 .expect_err("unknown query");
21957 let Error::QueryFailed(failure) = error else {
21958 panic!("expected typed query failure");
21959 };
21960 assert_eq!(failure.status, 404);
21961 assert_eq!(failure.reason, "rejected_unknown_query");
21962 }
21963
21964 #[tokio::test]
21965 async fn public_client_surfaces_send_and_receive_lossless_avro_values() {
21966 let server = MockWorkerServer::start();
21967 let client = Client::builder(server.base_url())
21968 .timeout(Duration::from_secs(2))
21969 .build()
21970 .expect("client");
21971 let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
21972
21973 client
21974 .start_workflow(
21975 "typed.echo",
21976 "rust-workers",
21977 "typed-start",
21978 arguments.clone(),
21979 )
21980 .await
21981 .expect("typed workflow start");
21982 assert_eq!(
21983 decode_wire_avro_value(
21984 &server.request_body("/api/workflows")["input"],
21985 DEFAULT_CODEC,
21986 )
21987 .expect("typed start input"),
21988 arguments
21989 );
21990
21991 client
21992 .signal_workflow("typed-1", "changed", arguments.clone())
21993 .await
21994 .expect("typed signal");
21995 assert_eq!(
21996 decode_wire_avro_value(
21997 &server.request_body("/api/workflows/typed-1/signal/changed")["input"],
21998 DEFAULT_CODEC,
21999 )
22000 .expect("typed signal input"),
22001 arguments
22002 );
22003
22004 assert_eq!(
22005 client
22006 .query_workflow_avro_value("typed-1", "inspect", arguments.clone())
22007 .await
22008 .expect("typed query"),
22009 typed_fidelity_probe()
22010 );
22011 assert_eq!(
22012 decode_wire_avro_value(
22013 &server.request_body("/api/workflows/typed-1/query/inspect")["input"],
22014 DEFAULT_CODEC,
22015 )
22016 .expect("typed query input"),
22017 arguments
22018 );
22019
22020 assert_eq!(
22021 client
22022 .update_workflow_avro_value(
22023 "typed-1",
22024 "replace",
22025 arguments.clone(),
22026 Some("typed-request"),
22027 )
22028 .await
22029 .expect("typed update"),
22030 typed_fidelity_probe()
22031 );
22032 let update = server.request_body("/api/workflows/typed-1/update/replace");
22033 assert_eq!(update["request_id"], "typed-request");
22034 assert_eq!(
22035 decode_wire_avro_value(&update["input"], DEFAULT_CODEC).expect("typed update input"),
22036 arguments
22037 );
22038
22039 let handle = WorkflowHandle {
22040 client: client.clone(),
22041 workflow_id: "typed-1".to_string(),
22042 run_id: Some("run-typed-1".to_string()),
22043 workflow_type: "typed.echo".to_string(),
22044 };
22045 assert_eq!(
22046 handle
22047 .result_avro_value(WorkflowResultOptions::default())
22048 .await
22049 .expect("typed workflow result"),
22050 typed_fidelity_probe()
22051 );
22052
22053 client
22054 .complete_activity_task(
22055 "activity-typed",
22056 "attempt-typed",
22057 "rust-worker",
22058 typed_fidelity_probe(),
22059 DEFAULT_CODEC,
22060 )
22061 .await
22062 .expect("typed activity completion");
22063 assert_eq!(
22064 decode_wire_avro_value(
22065 &server.request_body("/api/worker/activity-tasks/activity-typed/complete")
22066 ["result"],
22067 DEFAULT_CODEC,
22068 )
22069 .expect("typed activity result"),
22070 typed_fidelity_probe()
22071 );
22072 client
22073 .fail_activity_task(
22074 "activity-typed",
22075 "attempt-typed",
22076 "rust-worker",
22077 "typed failure",
22078 true,
22079 )
22080 .await
22081 .expect("activity failure");
22082 }
22083
22084 #[tokio::test]
22085 async fn lifecycle_commands_support_instance_and_selected_run_targets() {
22086 let server = MockWorkerServer::start();
22087 let client = Client::builder(server.base_url())
22088 .timeout(Duration::from_secs(2))
22089 .build()
22090 .expect("client");
22091
22092 let options = WorkflowCommandOptions::new()
22093 .reason("cleanup requested")
22094 .request_id("cancel-17");
22095 let cancelled = client
22096 .cancel_workflow("wf-lifecycle", options)
22097 .await
22098 .expect("instance cancellation");
22099 assert_eq!(cancelled.command, WorkflowCommandKind::Cancel);
22100 assert_eq!(cancelled.run_id.as_deref(), Some("run-current"));
22101 assert_eq!(cancelled.outcome.as_deref(), Some("cancelled"));
22102 assert_eq!(
22103 server.request_body("/api/workflows/wf-lifecycle/cancel"),
22104 json!({"reason":"cleanup requested","request_id":"cancel-17"})
22105 );
22106
22107 let terminated = client
22108 .terminate_workflow(
22109 "wf-lifecycle",
22110 WorkflowCommandOptions::new().reason("forced stop"),
22111 )
22112 .await
22113 .expect("instance termination");
22114 assert_eq!(terminated.command, WorkflowCommandKind::Terminate);
22115 assert_eq!(terminated.outcome.as_deref(), Some("terminated"));
22116
22117 client
22118 .cancel_workflow_run(
22119 "wf-lifecycle",
22120 "run-current",
22121 WorkflowCommandOptions::default(),
22122 )
22123 .await
22124 .expect("selected run cancellation");
22125 client
22126 .terminate_workflow_run(
22127 "wf-lifecycle",
22128 "run-current",
22129 WorkflowCommandOptions::default(),
22130 )
22131 .await
22132 .expect("selected run termination");
22133
22134 for (command, error) in [
22135 (
22136 WorkflowCommandKind::Cancel,
22137 client
22138 .cancel_workflow_run(
22139 "wf-lifecycle",
22140 "run-stale",
22141 WorkflowCommandOptions::default(),
22142 )
22143 .await
22144 .expect_err("stale cancellation must be rejected"),
22145 ),
22146 (
22147 WorkflowCommandKind::Terminate,
22148 client
22149 .terminate_workflow_run(
22150 "wf-lifecycle",
22151 "run-stale",
22152 WorkflowCommandOptions::default(),
22153 )
22154 .await
22155 .expect_err("stale termination must be rejected"),
22156 ),
22157 ] {
22158 let Error::WorkflowCommandRejected(rejection) = error else {
22159 panic!("expected typed command rejection");
22160 };
22161 assert_eq!(rejection.command, command);
22162 assert_eq!(rejection.status, 409);
22163 assert_eq!(rejection.reason, "historical_run_command_rejected");
22164 assert_eq!(rejection.run_id.as_deref(), Some("run-stale"));
22165 assert_eq!(rejection.target_scope.as_deref(), Some("run"));
22166 }
22167 }
22168
22169 #[tokio::test]
22170 async fn workflow_start_options_send_server_enforced_deadlines() {
22171 let server = MockWorkerServer::start();
22172 let client = Client::builder(server.base_url())
22173 .timeout(Duration::from_secs(2))
22174 .build()
22175 .expect("client");
22176
22177 let handle = client
22178 .start_workflow_with_options(
22179 "rust.timeout",
22180 "rust-timeouts",
22181 "wf-start-options",
22182 WorkflowStartOptions::new()
22183 .execution_timeout_seconds(30)
22184 .run_timeout_seconds(1),
22185 json!([]),
22186 )
22187 .await
22188 .expect("workflow start");
22189
22190 assert_eq!(handle.run_id.as_deref(), Some("run-start-options"));
22191 let body = server.request_body("/api/workflows");
22192 assert_eq!(body["execution_timeout_seconds"], 30);
22193 assert_eq!(body["run_timeout_seconds"], 1);
22194
22195 let invalid = client
22196 .start_workflow_with_options(
22197 "rust.timeout",
22198 "rust-timeouts",
22199 "wf-invalid-options",
22200 WorkflowStartOptions::new()
22201 .execution_timeout_seconds(1)
22202 .run_timeout_seconds(2),
22203 json!([]),
22204 )
22205 .await
22206 .expect_err("invalid deadline ordering");
22207 assert!(invalid
22208 .to_string()
22209 .contains("run_timeout_seconds cannot exceed execution_timeout_seconds"));
22210 }
22211
22212 #[tokio::test]
22213 async fn workflow_result_returns_each_typed_terminal_outcome() {
22214 let server = MockWorkerServer::start();
22215 let client = Client::builder(server.base_url())
22216 .timeout(Duration::from_secs(2))
22217 .build()
22218 .expect("client");
22219 let options = WorkflowResultOptions {
22220 poll_interval: Duration::ZERO,
22221 timeout: Duration::from_secs(1),
22222 };
22223
22224 let failed = WorkflowHandle {
22225 client: client.clone(),
22226 workflow_id: "wf-failed".to_string(),
22227 run_id: Some("run-failed".to_string()),
22228 workflow_type: "failure".to_string(),
22229 }
22230 .result(options)
22231 .await
22232 .expect_err("failed outcome");
22233 let Error::WorkflowFailed(failure) = failed else {
22234 panic!("expected WorkflowFailed");
22235 };
22236 assert_eq!(failure.workflow_id, "wf-failed");
22237 assert_eq!(failure.run_id.as_deref(), Some("run-failed"));
22238 assert_eq!(failure.failure_id.as_deref(), Some("failure-17"));
22239 assert_eq!(failure.failure_category.as_deref(), Some("application"));
22240 assert_eq!(failure.exception_type.as_deref(), Some("PaymentError"));
22241 assert_eq!(
22242 failure.exception_class.as_deref(),
22243 Some("billing::PaymentError")
22244 );
22245 assert_eq!(failure.non_retryable, Some(true));
22246
22247 for (workflow_id, expected_kind, expected_reason) in [
22248 (
22249 "wf-cancelled",
22250 WorkflowTerminalKind::Cancelled,
22251 "cleanup requested",
22252 ),
22253 (
22254 "wf-terminated",
22255 WorkflowTerminalKind::Terminated,
22256 "forced stop",
22257 ),
22258 (
22259 "wf-timed-out",
22260 WorkflowTerminalKind::TimedOut,
22261 "run_timeout",
22262 ),
22263 ] {
22264 let error = WorkflowHandle {
22265 client: client.clone(),
22266 workflow_id: workflow_id.to_string(),
22267 run_id: None,
22268 workflow_type: "terminal".to_string(),
22269 }
22270 .result(options)
22271 .await
22272 .expect_err("typed terminal outcome");
22273 let outcome = match error {
22274 Error::WorkflowCancelled(outcome) => outcome,
22275 Error::WorkflowTerminated(outcome) => outcome,
22276 Error::WorkflowTimedOut(outcome) => outcome,
22277 other => panic!("unexpected terminal error: {other}"),
22278 };
22279 assert_eq!(outcome.kind, expected_kind);
22280 assert_eq!(outcome.workflow_id, workflow_id);
22281 assert_eq!(outcome.reason, expected_reason);
22282 }
22283
22284 let wait_timeout = WorkflowHandle {
22285 client,
22286 workflow_id: "wf-waiting".to_string(),
22287 run_id: Some("run-waiting".to_string()),
22288 workflow_type: "waiting".to_string(),
22289 }
22290 .result(WorkflowResultOptions {
22291 poll_interval: Duration::ZERO,
22292 timeout: Duration::ZERO,
22293 })
22294 .await
22295 .expect_err("client wait timeout");
22296 let Error::WorkflowTimedOut(timeout) = wait_timeout else {
22297 panic!("expected typed client timeout");
22298 };
22299 assert_eq!(timeout.reason, "result_wait_timeout");
22300 assert_eq!(timeout.failure_category.as_deref(), Some("client_timeout"));
22301 assert_eq!(timeout.run_id.as_deref(), Some("run-waiting"));
22302 }
22303
22304 #[tokio::test]
22305 async fn workflow_result_follows_chain_and_selected_result_preserves_history() {
22306 let server = MockWorkerServer::start();
22307 let client = Client::builder(server.base_url())
22308 .timeout(Duration::from_secs(2))
22309 .build()
22310 .expect("client");
22311
22312 let handle = WorkflowHandle {
22313 client,
22314 workflow_id: "wf-selected".to_string(),
22315 run_id: Some("run-selected".to_string()),
22316 workflow_type: "selected".to_string(),
22317 };
22318 let options = WorkflowResultOptions {
22319 poll_interval: Duration::ZERO,
22320 timeout: Duration::from_secs(1),
22321 };
22322
22323 let current = handle
22324 .result(options)
22325 .await
22326 .expect("instance result follows the current run");
22327 assert_eq!(current, json!("current run output"));
22328
22329 let error = handle
22330 .result_selected_run(options)
22331 .await
22332 .expect_err("the selected run is cancelled even though the current run completed");
22333
22334 let Error::WorkflowCancelled(outcome) = error else {
22335 panic!("expected selected run cancellation");
22336 };
22337 assert_eq!(outcome.run_id.as_deref(), Some("run-selected"));
22338 assert_eq!(outcome.reason, "selected run cancelled");
22339 assert_eq!(
22340 server.request_count("/api/workflows/wf-selected/runs/run-selected"),
22341 1
22342 );
22343 assert_eq!(server.request_count("/api/workflows/wf-selected"), 1);
22344 }
22345
22346 #[tokio::test]
22347 async fn poll_responses_decode_http_conflict_drain_as_a_stable_stop() {
22348 let server = MockWorkerServer::draining_polls();
22349 let client = Client::builder(server.base_url())
22350 .timeout(Duration::from_secs(2))
22351 .build()
22352 .expect("client");
22353
22354 let workflow = client
22355 .poll_workflow_task_response("draining-worker", "rust-workers", Duration::ZERO)
22356 .await
22357 .expect("workflow drain response");
22358 let activity = client
22359 .poll_activity_task_response("draining-worker", "rust-workers", Duration::ZERO)
22360 .await
22361 .expect("activity drain response");
22362 let query = client
22363 .poll_query_task_response("draining-worker", "rust-workers", Duration::ZERO)
22364 .await
22365 .expect("query drain response");
22366
22367 for outcome in [workflow.outcome(), activity.outcome(), query.outcome()] {
22368 assert_eq!(
22369 outcome,
22370 WorkerPollOutcome::Stop {
22371 poll_status: Some("draining".to_string()),
22372 reason: Some("worker_draining".to_string()),
22373 }
22374 );
22375 }
22376
22377 assert!(client
22378 .poll_workflow_task("draining-worker", "rust-workers", Duration::ZERO)
22379 .await
22380 .expect("compatibility poll")
22381 .is_none());
22382 }
22383
22384 #[tokio::test]
22385 async fn managed_worker_honors_drain_stop_for_every_task_family() {
22386 let server = MockWorkerServer::draining_polls();
22387 let client = Client::builder(server.base_url())
22388 .timeout(Duration::from_secs(2))
22389 .build()
22390 .expect("client");
22391
22392 let mut workflow_worker = Worker::new(client.clone(), "rust-workers")
22393 .worker_id("draining-workflow-worker")
22394 .poll_timeout(Duration::ZERO);
22395 workflow_worker.register_workflow("counter", |_ctx, _args| async { Ok(Value::Null) });
22396 workflow_worker
22397 .run()
22398 .await
22399 .expect("workflow drain is a clean stop");
22400
22401 let mut activity_worker = Worker::new(client.clone(), "rust-workers")
22402 .worker_id("draining-activity-worker")
22403 .poll_timeout(Duration::ZERO);
22404 activity_worker.register_activity("write", |_ctx, _args| async { Ok(Value::Null) });
22405 activity_worker
22406 .run()
22407 .await
22408 .expect("activity drain is a clean stop");
22409
22410 let mut query_worker = Worker::new(client, "rust-workers")
22411 .worker_id("draining-query-worker")
22412 .poll_timeout(Duration::ZERO);
22413 query_worker.register_query("counter", "current", |_ctx, _args| async {
22414 Ok(Value::Null)
22415 });
22416 query_worker
22417 .run()
22418 .await
22419 .expect("query drain is a clean stop");
22420 }
22421
22422 #[tokio::test]
22423 async fn activity_cancellation_and_late_completion_remain_machine_readable() {
22424 let server = MockWorkerServer::start();
22425 let client = Client::builder(server.base_url())
22426 .timeout(Duration::from_secs(2))
22427 .build()
22428 .expect("client");
22429
22430 let heartbeat = client
22431 .heartbeat_activity_task(
22432 "activity-cancel",
22433 "attempt-cancel",
22434 "rust-worker",
22435 typed_fidelity_probe(),
22436 )
22437 .await
22438 .expect("cancellation heartbeat");
22439 assert!(heartbeat.cancel_requested);
22440 assert!(heartbeat.should_stop());
22441 assert_eq!(heartbeat.reason.as_deref(), Some("run_cancelled"));
22442 assert_eq!(heartbeat.run_closed_reason.as_deref(), Some("cancelled"));
22443 let heartbeat_body =
22444 server.request_body("/api/worker/activity-tasks/activity-cancel/heartbeat");
22445 assert_eq!(heartbeat_body["details"]["codec"], DEFAULT_CODEC);
22446 assert_eq!(
22447 decode_wire_avro_value(&heartbeat_body["details"], DEFAULT_CODEC)
22448 .expect("typed heartbeat details"),
22449 typed_fidelity_probe()
22450 );
22451
22452 let error = client
22453 .complete_activity_task(
22454 "activity-cancel",
22455 "attempt-cancel",
22456 "rust-worker",
22457 json!({"late":true}),
22458 DEFAULT_CODEC,
22459 )
22460 .await
22461 .expect_err("late completion must be refused");
22462 assert!(activity_task_rejection_is_final(&error));
22463 let Error::ActivityTaskRejected(rejection) = error else {
22464 panic!("expected typed activity rejection");
22465 };
22466 assert_eq!(rejection.status, 409);
22467 assert_eq!(rejection.reason, "run_cancelled");
22468 assert!(rejection.cancel_requested);
22469 assert_eq!(rejection.can_continue, Some(false));
22470 }
22471
22472 #[tokio::test]
22473 async fn managed_worker_survives_late_completion_and_restart_during_cancellation() {
22474 let server = MockWorkerServer::cancelled_activity();
22475 let client = Client::builder(server.base_url())
22476 .timeout(Duration::from_secs(2))
22477 .build()
22478 .expect("client");
22479 let cancellation_observed = Arc::new(AtomicBool::new(false));
22480 let observed = Arc::clone(&cancellation_observed);
22481 let mut worker = Worker::new(client.clone(), "rust-workers")
22482 .worker_id("rust-cancel-worker")
22483 .poll_timeout(Duration::from_millis(10));
22484 worker.register_activity("cancel-aware", move |ctx, _args| {
22485 let observed = Arc::clone(&observed);
22486 async move {
22487 let heartbeat = ctx.heartbeat(json!({"stage":"running"})).await?;
22488 observed.store(heartbeat.should_stop(), Ordering::SeqCst);
22489 Ok(json!({"late":"completion"}))
22490 }
22491 });
22492
22493 assert_eq!(
22494 worker.run_once().await.expect("cancelled attempt handled"),
22495 1
22496 );
22497 assert!(cancellation_observed.load(Ordering::SeqCst));
22498 assert_eq!(
22499 server.request_count("/api/worker/activity-tasks/activity-cancel/complete"),
22500 1
22501 );
22502
22503 let mut restarted = Worker::new(client, "rust-workers")
22504 .worker_id("rust-cancel-worker-restarted")
22505 .poll_timeout(Duration::from_millis(10));
22506 restarted.register_activity("cancel-aware", |_ctx, _args| async move { Ok(Value::Null) });
22507 assert_eq!(
22508 restarted
22509 .run_once()
22510 .await
22511 .expect("replacement worker continues polling"),
22512 0
22513 );
22514 }
22515
22516 #[tokio::test]
22517 async fn managed_worker_absorbs_selected_run_terminal_timeout_completion_race() {
22518 let response = r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"outcome":"completed","recorded":false,"run_id":"run-selected-timeout","run_status":"failed","created_task_ids":[],"reason":"run_timed_out"}"#;
22519 let server = MockWorkerServer::workflow_completion("409 Conflict", response);
22520 let client = Client::builder(server.base_url())
22521 .timeout(Duration::from_secs(2))
22522 .build()
22523 .expect("client");
22524
22525 let direct_error = client
22526 .complete_workflow_task(
22527 "workflow-timeout-task",
22528 "timeout-worker",
22529 3,
22530 vec![json!({
22531 "type": "complete_workflow",
22532 "result": fixture_envelope(Value::Null)
22533 })],
22534 )
22535 .await
22536 .expect_err("the low-level client preserves the completion rejection");
22537 let Error::Http { status, body } = direct_error else {
22538 panic!("expected the original HTTP completion rejection");
22539 };
22540 assert_eq!(status, reqwest::StatusCode::CONFLICT);
22541 assert_eq!(
22542 serde_json::from_str::<Value>(&body).expect("response body")["reason"],
22543 "run_timed_out"
22544 );
22545
22546 let mut worker = Worker::new(client, "rust-workers")
22547 .worker_id("timeout-worker")
22548 .poll_timeout(Duration::from_millis(10));
22549 worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
22550 Ok(json!({"late": "result"}))
22551 });
22552
22553 assert_eq!(
22554 worker
22555 .run_once()
22556 .await
22557 .expect("authoritative selected-run timeout settles the tick"),
22558 1
22559 );
22560 assert_eq!(
22561 server.request_count("/api/worker/workflow-tasks/workflow-timeout-task/complete"),
22562 2,
22563 "both the direct client proof and managed worker must see the rejection"
22564 );
22565 }
22566
22567 #[tokio::test]
22568 async fn managed_worker_does_not_swallow_nearby_completion_errors() {
22569 for (name, status, response) in [
22570 ("bare conflict", "409 Conflict", r#"{"message":"conflict"}"#),
22571 (
22572 "command was recorded",
22573 "409 Conflict",
22574 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":true,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22575 ),
22576 (
22577 "lease conflict",
22578 "409 Conflict",
22579 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"lease_expired"}"#,
22580 ),
22581 (
22582 "nonterminal run",
22583 "409 Conflict",
22584 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"waiting","reason":"run_timed_out"}"#,
22585 ),
22586 (
22587 "different selected run",
22588 "409 Conflict",
22589 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-reused-workflow-current","run_status":"failed","reason":"run_timed_out"}"#,
22590 ),
22591 (
22592 "different task attempt",
22593 "409 Conflict",
22594 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":4,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22595 ),
22596 (
22597 "authentication failure",
22598 "401 Unauthorized",
22599 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22600 ),
22601 (
22602 "authorization failure",
22603 "403 Forbidden",
22604 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22605 ),
22606 (
22607 "protocol failure",
22608 "400 Bad Request",
22609 r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.2","requested_version":"1.3"}"#,
22610 ),
22611 (
22612 "malformed command",
22613 "422 Unprocessable Entity",
22614 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22615 ),
22616 (
22617 "transient server failure",
22618 "503 Service Unavailable",
22619 r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22620 ),
22621 ] {
22622 let server = MockWorkerServer::workflow_completion(status, response);
22623 let client = Client::builder(server.base_url())
22624 .timeout(Duration::from_secs(2))
22625 .build()
22626 .expect("client");
22627 let mut worker = Worker::new(client, "rust-workers")
22628 .worker_id("timeout-worker")
22629 .poll_timeout(Duration::from_millis(10));
22630 worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
22631 Ok(json!({"late": "result"}))
22632 });
22633
22634 let error = worker
22635 .run_once()
22636 .await
22637 .expect_err(&format!("{name} must remain an error"));
22638 assert!(
22639 matches!(error, Error::Http { .. } | Error::Protocol(_)),
22640 "{name} returned an unexpected error variant: {error}"
22641 );
22642 }
22643 }
22644
22645 #[tokio::test]
22646 async fn worker_deregistration_uses_worker_plane_method_path_headers_and_result() {
22647 let server = MockWorkerServer::start();
22648 let client = Client::builder(server.base_url())
22649 .worker_token(Some("worker-secret".to_string()))
22650 .namespace("orders")
22651 .timeout(Duration::from_secs(2))
22652 .build()
22653 .expect("client");
22654 let path = "/api/worker/registrations/worker%2F%CE%B1%20space";
22655
22656 let result = client
22657 .deregister_worker_registration("worker/α space")
22658 .await
22659 .expect("deregister worker registration");
22660
22661 assert_eq!(server.method_for(path).as_deref(), Some("DELETE"));
22662 assert_eq!(
22663 server.worker_protocol_for(path).as_deref(),
22664 Some(WORKER_PROTOCOL_VERSION)
22665 );
22666 assert_eq!(server.control_protocol_for(path), None);
22667 assert_eq!(server.namespace_for(path).as_deref(), Some("orders"));
22668 assert_eq!(
22669 server.authorization_for(path).as_deref(),
22670 Some("Bearer worker-secret")
22671 );
22672 assert_eq!(
22673 result,
22674 WorkerDeregistrationEnvelope {
22675 worker_id: "deregistered-worker".to_string(),
22676 outcome: "deregistered".to_string(),
22677 recovered_workflow_task_count: 2,
22678 }
22679 );
22680 }
22681
22682 #[tokio::test]
22683 async fn low_level_registration_rejects_update_validators_before_transport() {
22684 let server = MockWorkerServer::start();
22685 let client = Client::builder(server.base_url())
22686 .timeout(Duration::from_secs(2))
22687 .build()
22688 .expect("client");
22689
22690 for update_validators in [json!(["approve"]), json!("approve")] {
22691 let error = client
22692 .register_worker_with_command_contracts(
22693 "validator-claiming-worker",
22694 "rust-workers",
22695 vec!["orders".to_string()],
22696 vec![],
22697 1,
22698 1,
22699 vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
22700 json!({
22701 "orders": {
22702 "queries": ["current"],
22703 "updates": ["approve"],
22704 "update_validators": update_validators,
22705 },
22706 }),
22707 )
22708 .await
22709 .expect_err("unsupported validator claims must fail before registration");
22710
22711 let Error::UnsupportedUpdateValidators { workflow_type } = error else {
22712 panic!("expected typed unsupported-validator failure");
22713 };
22714 assert_eq!(workflow_type, "orders");
22715 }
22716 assert_eq!(server.request_count("/api/worker/register"), 0);
22717 }
22718
22719 #[tokio::test]
22720 async fn low_level_registration_preserves_query_and_update_contracts() {
22721 let server = MockWorkerServer::start();
22722 let client = Client::builder(server.base_url())
22723 .timeout(Duration::from_secs(2))
22724 .build()
22725 .expect("client");
22726 let contracts = json!({
22727 "orders": {
22728 "queries": ["current"],
22729 "updates": ["approve"],
22730 "update_validators": [],
22731 },
22732 "payments": {
22733 "queries": ["status"],
22734 "updates": ["capture"],
22735 },
22736 });
22737
22738 client
22739 .register_worker_with_command_contracts(
22740 "command-worker",
22741 "rust-workers",
22742 vec!["orders".to_string(), "payments".to_string()],
22743 vec![],
22744 1,
22745 1,
22746 vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
22747 contracts.clone(),
22748 )
22749 .await
22750 .expect("query and update contracts must remain supported");
22751
22752 assert_eq!(
22753 server.request_body("/api/worker/register")["workflow_command_contracts"],
22754 contracts
22755 );
22756 }
22757
22758 #[tokio::test]
22759 async fn role_scoped_tokens_are_never_used_for_the_opposite_plane() {
22760 let server = MockWorkerServer::start();
22761 let control_only = Client::builder(server.base_url())
22762 .control_token(Some("control-secret".to_string()))
22763 .build()
22764 .expect("control client");
22765
22766 let error = control_only
22767 .register_worker("worker", "queue", vec![], vec![], 1, 1)
22768 .await
22769 .expect_err("control token must not authorize a worker request");
22770 assert!(matches!(
22771 error,
22772 Error::MissingRoleCredentials { role: "worker", .. }
22773 ));
22774 assert_eq!(server.request_count("/api/worker/register"), 0);
22775
22776 let worker_only = Client::builder(server.base_url())
22777 .worker_token(Some("worker-secret".to_string()))
22778 .build()
22779 .expect("worker client");
22780 let error = worker_only
22781 .health()
22782 .await
22783 .expect_err("worker token must not authorize a control request");
22784 assert!(matches!(
22785 error,
22786 Error::MissingRoleCredentials {
22787 role: "control",
22788 ..
22789 }
22790 ));
22791 assert_eq!(server.request_count("/api/health"), 0);
22792 }
22793
22794 #[tokio::test]
22795 async fn shared_token_supports_worker_and_control_planes() {
22796 let server = MockWorkerServer::start();
22797 let client = Client::builder(server.base_url())
22798 .token(Some("shared-secret".to_string()))
22799 .build()
22800 .expect("client");
22801
22802 client.health().await.expect("control request");
22803 client
22804 .register_worker("worker", "queue", vec![], vec![], 1, 1)
22805 .await
22806 .expect("worker request");
22807
22808 assert_eq!(
22809 server.authorization_for("/api/health").as_deref(),
22810 Some("Bearer shared-secret")
22811 );
22812 assert_eq!(
22813 server.control_protocol_for("/api/health").as_deref(),
22814 Some(CONTROL_PLANE_VERSION)
22815 );
22816 assert_eq!(
22817 server.authorization_for("/api/worker/register").as_deref(),
22818 Some("Bearer shared-secret")
22819 );
22820 assert_eq!(
22821 server
22822 .worker_protocol_for("/api/worker/register")
22823 .as_deref(),
22824 Some(WORKER_PROTOCOL_VERSION)
22825 );
22826 }
22827
22828 #[tokio::test]
22829 async fn baseline_worker_endpoints_send_the_baseline_protocol() {
22830 let server = MockWorkerServer::start();
22831 let client = Client::builder(server.base_url())
22832 .timeout(Duration::from_secs(2))
22833 .build()
22834 .expect("client");
22835
22836 client
22837 .register_worker("capture-worker", "capture", vec![], vec![], 1, 1)
22838 .await
22839 .expect("register");
22840 client
22841 .heartbeat_worker("capture-worker", 1, 1)
22842 .await
22843 .expect("heartbeat");
22844 client
22845 .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
22846 .await
22847 .expect("workflow poll");
22848 client
22849 .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
22850 .await
22851 .expect("activity poll");
22852
22853 for path in [
22854 "/api/worker/register",
22855 "/api/worker/heartbeat",
22856 "/api/worker/workflow-tasks/poll",
22857 "/api/worker/activity-tasks/poll",
22858 ] {
22859 assert_eq!(
22860 server.worker_protocol_for(path).as_deref(),
22861 Some(WORKER_PROTOCOL_VERSION),
22862 "unexpected protocol for {path}"
22863 );
22864 }
22865
22866 assert_eq!(
22867 server.request_body("/api/worker/workflow-tasks/poll")["timeout_seconds"],
22868 1
22869 );
22870 assert_eq!(
22871 server.request_body("/api/worker/activity-tasks/poll")["timeout_seconds"],
22872 1
22873 );
22874 assert!(
22875 server.request_body("/api/worker/workflow-tasks/poll")["poll_request_id"]
22876 .as_str()
22877 .is_some_and(|id| id.starts_with("rust-workflow-poll-"))
22878 );
22879 assert!(
22880 server.request_body("/api/worker/activity-tasks/poll")["poll_request_id"]
22881 .as_str()
22882 .is_some_and(|id| id.starts_with("rust-activity-poll-"))
22883 );
22884 }
22885
22886 #[tokio::test]
22887 async fn query_task_endpoints_send_the_query_feature_protocol() {
22888 let server = MockWorkerServer::start();
22889 let client = Client::builder(server.base_url())
22890 .timeout(Duration::from_secs(2))
22891 .build()
22892 .expect("client");
22893
22894 client
22895 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
22896 .await
22897 .expect("query poll");
22898 client
22899 .complete_query_task(
22900 "query-capture",
22901 "capture-worker",
22902 1,
22903 json!(8),
22904 DEFAULT_CODEC,
22905 )
22906 .await
22907 .expect("query complete");
22908 client
22909 .fail_query_task(
22910 "query-capture",
22911 "capture-worker",
22912 1,
22913 "failed",
22914 "query_rejected",
22915 "QueryFailed",
22916 )
22917 .await
22918 .expect("query fail");
22919
22920 for path in [
22921 "/api/worker/query-tasks/poll",
22922 "/api/worker/query-tasks/query-capture/complete",
22923 "/api/worker/query-tasks/query-capture/fail",
22924 ] {
22925 assert_eq!(
22926 server.worker_protocol_for(path).as_deref(),
22927 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
22928 "unexpected protocol for {path}"
22929 );
22930 }
22931
22932 assert_eq!(
22933 server.request_body("/api/worker/query-tasks/poll")["timeout_seconds"],
22934 1
22935 );
22936 assert!(
22937 server.request_body("/api/worker/query-tasks/poll")["poll_request_id"]
22938 .as_str()
22939 .is_some_and(|id| id.starts_with("rust-query-poll-"))
22940 );
22941 }
22942
22943 #[tokio::test]
22944 async fn disconnected_client_polls_retry_once_with_the_same_request_id() {
22945 let server = MockWorkerServer::transient_worker_failures();
22946 let client = Client::builder(server.base_url())
22947 .timeout(Duration::from_secs(2))
22948 .build()
22949 .expect("client");
22950
22951 client
22952 .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
22953 .await
22954 .expect("workflow poll retry");
22955 client
22956 .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
22957 .await
22958 .expect("activity poll retry");
22959 client
22960 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
22961 .await
22962 .expect("query poll retry");
22963
22964 for path in [
22965 "/api/worker/workflow-tasks/poll",
22966 "/api/worker/activity-tasks/poll",
22967 "/api/worker/query-tasks/poll",
22968 ] {
22969 let bodies = server.request_bodies(path);
22970 assert_eq!(bodies.len(), 2, "{path} must be retried once");
22971 assert_eq!(
22972 bodies[0]["poll_request_id"], bodies[1]["poll_request_id"],
22973 "{path} must preserve the request binding across retry"
22974 );
22975 }
22976 }
22977
22978 #[tokio::test]
22979 async fn worker_poll_retries_preserve_request_id_across_consecutive_failures() {
22980 let server = MockWorkerServer::consecutive_poll_failures(2);
22981 let client = Client::builder(server.base_url())
22982 .timeout(Duration::from_secs(2))
22983 .build()
22984 .expect("client");
22985 let mut worker = Worker::new(client, "capture")
22986 .worker_id("capture-worker")
22987 .poll_timeout(Duration::from_millis(10))
22988 .retry_policy(WorkerRetryPolicy {
22989 max_retries: 2,
22990 initial_backoff: Duration::from_millis(1),
22991 max_backoff: Duration::from_millis(1),
22992 });
22993 worker.register_workflow(
22994 "capture.workflow",
22995 |_ctx, _input| async move { Ok(Value::Null) },
22996 );
22997 worker.register_activity(
22998 "capture.activity",
22999 |_ctx, _input| async move { Ok(Value::Null) },
23000 );
23001 worker.register_query("capture.workflow", "current", |_ctx, _args| async move {
23002 Ok(Value::Null)
23003 });
23004
23005 assert_eq!(worker.run_once().await.expect("poll retries"), 0);
23006
23007 for path in [
23008 "/api/worker/workflow-tasks/poll",
23009 "/api/worker/activity-tasks/poll",
23010 "/api/worker/query-tasks/poll",
23011 ] {
23012 let bodies = server.request_bodies(path);
23013 assert_eq!(bodies.len(), 3, "{path} must use exactly two retries");
23014 assert!(
23015 bodies
23016 .iter()
23017 .all(|body| body["poll_request_id"] == bodies[0]["poll_request_id"]),
23018 "{path} must preserve one request binding across every retry"
23019 );
23020 }
23021 }
23022
23023 fn storage_refusal(poll_id: Option<&str>, unavailable: bool, mid_poll: bool) -> Value {
23024 let reason = if unavailable {
23025 "storage_admission_unavailable"
23026 } else {
23027 "storage_pressure"
23028 };
23029 let mut body = json!({
23030 "reason": reason,
23031 "storage_state": if unavailable { "fenced" } else { "draining" },
23032 "retryable": true,
23033 "retry_after_seconds": 1,
23034 });
23035 if !mid_poll {
23036 body["request_admitted"] = json!(false);
23037 }
23038 if let Some(id) = poll_id {
23039 body["task"] = Value::Null;
23040 body["poll_status"] = json!(reason);
23041 body["poll_request_id"] = json!(id);
23042 body["retry_same_poll_request_id"] = json!(true);
23043 body["claim_admitted"] = json!(false);
23044 }
23045 body
23046 }
23047
23048 fn storage_worker(server: &MockWorkerServer) -> Worker {
23049 Worker::new(Client::new(server.base_url()).expect("client"), "storage")
23050 .worker_id("storage-worker")
23051 .retry_policy(WorkerRetryPolicy {
23052 max_retries: 1,
23053 initial_backoff: Duration::from_millis(1),
23054 max_backoff: Duration::from_millis(1),
23055 })
23056 }
23057
23058 fn assert_identical_requests(server: &MockWorkerServer, path: &str, count: usize) {
23059 let requests = server.requests.lock().expect("requests");
23060 let bodies: Vec<_> = requests
23061 .iter()
23062 .filter(|request| request.path == path)
23063 .map(|request| &request.body)
23064 .collect();
23065 assert_eq!(bodies.len(), count, "{path}");
23066 assert!(bodies.iter().all(|body| body == &bodies[0]), "{path}");
23067 }
23068
23069 #[test]
23070 fn storage_admission_requires_an_explicit_identity_preserving_contract() {
23071 for unavailable in [false, true] {
23072 for mid_poll in [false, true] {
23073 let body = storage_refusal(Some("same-poll"), unavailable, mid_poll);
23074 let error = Error::Http {
23075 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23076 body: body.to_string(),
23077 };
23078 assert_eq!(
23079 worker_storage_admission_retry_after(&error, Some("same-poll")),
23080 Some(Duration::from_secs(1))
23081 );
23082 assert!(
23083 !worker_operation_is_retryable(&error),
23084 "storage is not a bounded generic retry"
23085 );
23086 for (field, value) in [
23087 ("poll_request_id", json!("wrong-poll")),
23088 ("task", json!({"task_id":"claimed"})),
23089 ("retryable", json!(false)),
23090 ("retry_after_seconds", json!(0)),
23091 ("retry_after_seconds", json!(true)),
23092 ("retry_after_seconds", json!(1.0)),
23093 ("storage_state", json!("normal")),
23094 ("poll_status", json!("empty")),
23095 ("claim_admitted", json!(true)),
23096 ("retry_same_poll_request_id", json!(false)),
23097 ("request_admitted", json!(true)),
23098 ] {
23099 let mut invalid = body.clone();
23100 invalid[field] = value;
23101 let error = Error::Http {
23102 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23103 body: invalid.to_string(),
23104 };
23105 assert!(
23106 worker_storage_admission_retry_after(&error, Some("same-poll")).is_none(),
23107 "{field}"
23108 );
23109 }
23110 }
23111 }
23112 let body = storage_refusal(None, false, false);
23113 let error = Error::Http {
23114 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23115 body: body.to_string(),
23116 };
23117 assert!(worker_storage_admission_retry_after(&error, None).is_some());
23118 assert!(worker_storage_admission_retry_after(&error, Some("")).is_none());
23119 let error = Error::Http {
23120 status: reqwest::StatusCode::FORBIDDEN,
23121 body: body.to_string(),
23122 };
23123 assert!(worker_storage_admission_retry_after(&error, None).is_none());
23124 let body = storage_refusal(None, false, true);
23125 let error = Error::Http {
23126 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23127 body: body.to_string(),
23128 };
23129 assert!(worker_storage_admission_retry_after(&error, None).is_none());
23130 }
23131
23132 #[tokio::test]
23133 async fn storage_poll_recovery_preserves_ambiguous_claim_identity() {
23134 for unavailable in [false, true] {
23135 for mid_poll in [false, true] {
23136 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23137 poll_failures_per_path: 1,
23138 storage_refusals: 7,
23139 storage_path: Some("/poll"),
23140 storage_unavailable: unavailable,
23141 storage_mid_poll: mid_poll,
23142 ..MockWorkerBehavior::default()
23143 });
23144 let mut worker = storage_worker(&server);
23145 worker.register_query("unused", "state", |_, _| async { Ok(Value::Null) });
23146 assert_eq!(worker.run_once().await.expect("storage recovery"), 0);
23147 for path in [
23148 "/api/worker/workflow-tasks/poll",
23149 "/api/worker/activity-tasks/poll",
23150 "/api/worker/query-tasks/poll",
23151 ] {
23152 assert_identical_requests(&server, path, 9);
23153 }
23154 }
23155 }
23156 }
23157
23158 #[tokio::test]
23159 async fn storage_refused_mutations_do_not_reserialize_or_change_client_scope() {
23160 struct CountedBody(Arc<AtomicUsize>);
23161 impl Serialize for CountedBody {
23162 fn serialize<S: Serializer>(
23163 &self,
23164 serializer: S,
23165 ) -> std::result::Result<S::Ok, S::Error> {
23166 let count = self.0.fetch_add(1, Ordering::SeqCst);
23167 json!({"serialization":count,"lease_owner":"worker","attempt":7})
23168 .serialize(serializer)
23169 }
23170 }
23171 for path in [
23172 "/api/worker/register",
23173 "/api/worker/heartbeat",
23174 "/api/worker/workflow-tasks/storage-task/complete",
23175 "/api/worker/workflow-tasks/storage-task/fail",
23176 "/api/worker/activity-tasks/storage-task/complete",
23177 "/api/worker/activity-tasks/storage-task/fail",
23178 "/api/worker/activity-tasks/storage-task/heartbeat",
23179 "/api/worker/query-tasks/storage-task/complete",
23180 "/api/worker/query-tasks/storage-task/fail",
23181 ] {
23182 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23183 storage_refusals: 7,
23184 storage_path: Some(path),
23185 ..MockWorkerBehavior::default()
23186 });
23187 let worker =
23188 storage_worker(&server).with_storage_admission(Arc::new(AtomicBool::new(false)));
23189 let calls = Arc::new(AtomicUsize::new(0));
23190 let _: Value = worker
23191 .client
23192 .request_json(
23193 reqwest::Method::POST,
23194 &path[4..],
23195 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
23196 Some(&CountedBody(Arc::clone(&calls))),
23197 )
23198 .await
23199 .expect("prepared request recovery");
23200 assert_eq!(calls.load(Ordering::SeqCst), 1);
23201 assert_identical_requests(&server, path, 8);
23202 }
23203 for worker_scope in [false, true] {
23204 let path = if worker_scope {
23205 "/api/health"
23206 } else {
23207 "/api/worker/register"
23208 };
23209 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23210 storage_refusals: usize::MAX,
23211 storage_path: Some(path),
23212 ..MockWorkerBehavior::default()
23213 });
23214 let worker = storage_worker(&server);
23215 let client = worker.client.clone();
23216 let worker = worker.with_storage_admission(Arc::new(AtomicBool::new(false)));
23217 let error = if worker_scope {
23218 worker
23219 .client
23220 .health()
23221 .await
23222 .expect_err("control plane is not retried")
23223 } else {
23224 client
23225 .request_json::<Value, Value>(
23226 reqwest::Method::POST,
23227 "/worker/register",
23228 RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
23229 Some(&json!({})),
23230 )
23231 .await
23232 .expect_err("direct client is not retried")
23233 };
23234 assert!(worker_storage_admission_body(&error).is_some());
23235 assert_eq!(server.request_count(path), 1);
23236 }
23237 }
23238
23239 #[tokio::test]
23240 async fn storage_activity_outcome_is_retained_without_reexecuting_handler() {
23241 for fail in [false, true] {
23242 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23243 storage_activity: true,
23244 storage_refusals: 7,
23245 storage_path: Some("/storage-activity/"),
23246 ..MockWorkerBehavior::default()
23247 });
23248 let mut worker = storage_worker(&server);
23249 let calls = Arc::new(AtomicUsize::new(0));
23250 let observed = Arc::clone(&calls);
23251 worker.register_activity("storage.activity", move |ctx, _| {
23252 let calls = Arc::clone(&observed);
23253 async move {
23254 calls.fetch_add(1, Ordering::SeqCst);
23255 ctx.heartbeat(json!({"step":2})).await?;
23256 if fail {
23257 Err(Error::WorkerLoop("intentional handler failure".to_string()))
23258 } else {
23259 Ok(json!({"receipt":true}))
23260 }
23261 }
23262 });
23263 assert_eq!(worker.run_once().await.expect("activity settled"), 1);
23264 assert_eq!(calls.load(Ordering::SeqCst), 1);
23265 assert_identical_requests(
23266 &server,
23267 "/api/worker/activity-tasks/storage-activity/heartbeat",
23268 8,
23269 );
23270 let suffix = if fail { "fail" } else { "complete" };
23271 assert_identical_requests(
23272 &server,
23273 &format!("/api/worker/activity-tasks/storage-activity/{suffix}"),
23274 8,
23275 );
23276 let other = if fail { "complete" } else { "fail" };
23277 assert_eq!(
23278 server.request_count(&format!(
23279 "/api/worker/activity-tasks/storage-activity/{other}"
23280 )),
23281 0
23282 );
23283 }
23284 }
23285
23286 #[tokio::test]
23287 async fn storage_waits_are_interruptible_without_false_activity_failure() {
23288 for path in [
23289 "/api/worker/register",
23290 "/api/worker/heartbeat",
23291 "/api/worker/activity-tasks/poll",
23292 "/api/worker/activity-tasks/storage-activity/heartbeat",
23293 "/api/worker/activity-tasks/storage-activity/complete",
23294 ] {
23295 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23296 storage_activity: true,
23297 storage_refusals: usize::MAX,
23298 storage_path: Some(path),
23299 ..MockWorkerBehavior::default()
23300 });
23301 let mut worker = storage_worker(&server).retry_policy(WorkerRetryPolicy::default());
23302 worker.register_activity("storage.activity", |ctx, _| async move {
23303 ctx.heartbeat(json!({"step":2})).await?;
23304 Ok(json!({"receipt":true}))
23305 });
23306 let shutdown = async {
23307 while server.request_count(path) == 0 {
23308 tokio::time::sleep(Duration::from_millis(1)).await;
23309 }
23310 };
23311 let result = tokio::time::timeout(Duration::from_secs(2), worker.run_until(shutdown))
23312 .await
23313 .expect("shutdown interrupts admission");
23314 assert!(
23315 result.is_err(),
23316 "a refused operation must not appear acknowledged: {path}, {result:?}"
23317 );
23318 assert_eq!(server.request_count(path), 1);
23319 assert_eq!(
23320 server.request_count("/api/worker/activity-tasks/storage-activity/fail"),
23321 0
23322 );
23323 assert_eq!(
23324 server.request_count("/api/worker/registrations/mock-worker"),
23325 usize::from(!path.ends_with("/register"))
23326 );
23327 }
23328 }
23329
23330 #[tokio::test]
23331 async fn storage_query_outcome_is_retained_without_reexecuting_handler() {
23332 for fail in [false, true] {
23333 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23334 storage_query: true,
23335 storage_refusals: 7,
23336 storage_path: Some("/storage-query/"),
23337 ..MockWorkerBehavior::default()
23338 });
23339 let mut worker = storage_worker(&server);
23340 worker.register_workflow("storage.workflow", |_, _| async { Ok(Value::Null) });
23341 let calls = Arc::new(AtomicUsize::new(0));
23342 let observed = Arc::clone(&calls);
23343 worker.register_query("storage.workflow", "state", move |_, _| {
23344 let calls = Arc::clone(&observed);
23345 async move {
23346 calls.fetch_add(1, Ordering::SeqCst);
23347 if fail {
23348 Err(Error::WorkerLoop("intentional query failure".to_string()))
23349 } else {
23350 Ok(json!({"state":"waiting"}))
23351 }
23352 }
23353 });
23354 assert_eq!(worker.run_once().await.expect("query settled"), 1);
23355 assert_eq!(calls.load(Ordering::SeqCst), 1);
23356 let suffix = if fail { "fail" } else { "complete" };
23357 assert_identical_requests(
23358 &server,
23359 &format!("/api/worker/query-tasks/storage-query/{suffix}"),
23360 8,
23361 );
23362 let other = if fail { "complete" } else { "fail" };
23363 assert_eq!(
23364 server.request_count(&format!("/api/worker/query-tasks/storage-query/{other}")),
23365 0
23366 );
23367 }
23368 }
23369
23370 #[tokio::test]
23371 async fn storage_recovery_does_not_override_auth_lease_or_invalid_contract() {
23372 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23373 storage_refusals: 7,
23374 storage_path: Some("/poll"),
23375 unauthorized_polls: true,
23376 ..MockWorkerBehavior::default()
23377 });
23378 let error = storage_worker(&server)
23379 .run_once()
23380 .await
23381 .expect_err("auth remains terminal");
23382 assert!(matches!(
23383 error,
23384 Error::Http {
23385 status: reqwest::StatusCode::UNAUTHORIZED,
23386 ..
23387 }
23388 ));
23389 assert_identical_requests(&server, "/api/worker/workflow-tasks/poll", 8);
23390
23391 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23392 storage_refusals: 7,
23393 storage_path: Some("/activity-cancel/complete"),
23394 ..MockWorkerBehavior::default()
23395 });
23396 let worker =
23397 storage_worker(&server).with_storage_admission(Arc::new(AtomicBool::new(false)));
23398 let error = worker
23399 .client
23400 .complete_activity_task(
23401 "activity-cancel",
23402 "attempt-cancel",
23403 "worker",
23404 json!({}),
23405 DEFAULT_CODEC,
23406 )
23407 .await
23408 .expect_err("cancellation remains terminal");
23409 assert!(activity_task_rejection_is_final(&error));
23410 assert_identical_requests(
23411 &server,
23412 "/api/worker/activity-tasks/activity-cancel/complete",
23413 8,
23414 );
23415
23416 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23417 storage_refusals: usize::MAX,
23418 storage_path: Some("/poll"),
23419 storage_wrong_poll_id: true,
23420 ..MockWorkerBehavior::default()
23421 });
23422 assert!(storage_worker(&server).run_once().await.is_err());
23423 assert_eq!(server.request_count("/api/worker/workflow-tasks/poll"), 1);
23424 }
23425
23426 #[tokio::test]
23427 async fn storage_pollers_stop_when_the_run_future_is_aborted() {
23428 let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23429 storage_refusals: usize::MAX,
23430 storage_path: Some("/poll"),
23431 ..MockWorkerBehavior::default()
23432 });
23433 let mut worker = storage_worker(&server).retry_policy(WorkerRetryPolicy::default());
23434 worker.register_activity("unused", |_, _| async { Ok(Value::Null) });
23435 let run = tokio::spawn(async move { worker.run().await });
23436 tokio::time::timeout(Duration::from_secs(2), async {
23437 while server.request_count("/api/worker/activity-tasks/poll") == 0 {
23438 tokio::time::sleep(Duration::from_millis(1)).await;
23439 }
23440 })
23441 .await
23442 .expect("poll started");
23443 run.abort();
23444 assert!(run.await.expect_err("cancelled run").is_cancelled());
23445 tokio::time::sleep(Duration::from_millis(250)).await;
23446 assert_eq!(server.request_count("/api/worker/activity-tasks/poll"), 1);
23447 }
23448
23449 #[tokio::test]
23450 async fn query_protocol_rejection_from_older_server_is_typed() {
23451 let server = MockWorkerServer::reject_query_protocol();
23452 let client = Client::builder(server.base_url())
23453 .timeout(Duration::from_secs(2))
23454 .build()
23455 .expect("client");
23456
23457 let error = client
23458 .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
23459 .await
23460 .expect_err("server below query protocol floor must reject");
23461 let Error::Protocol(failure) = error else {
23462 panic!("expected typed protocol failure");
23463 };
23464
23465 assert_eq!(failure.status, 400);
23466 assert_eq!(failure.reason, "unsupported_protocol_version");
23467 assert_eq!(failure.supported_version.as_deref(), Some("1.7"));
23468 assert_eq!(
23469 failure.requested_version.as_deref(),
23470 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
23471 );
23472 assert_eq!(
23473 server
23474 .worker_protocol_for("/api/worker/query-tasks/poll")
23475 .as_deref(),
23476 Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
23477 );
23478 }
23479
23480 #[tokio::test]
23481 async fn run_once_without_query_handlers_keeps_pre_query_server_compatibility() {
23482 let server = MockWorkerServer::reject_query_protocol();
23483 let client = Client::builder(server.base_url())
23484 .timeout(Duration::from_secs(2))
23485 .build()
23486 .expect("client");
23487 let mut worker = Worker::new(client, "rust-workers")
23488 .worker_id("baseline-worker")
23489 .poll_timeout(Duration::from_millis(10));
23490
23491 worker.register_workflow("baseline.workflow", |_ctx, _input| async move {
23492 Ok(Value::Null)
23493 });
23494
23495 assert_eq!(worker.run_once().await.expect("baseline run once"), 0);
23496 assert_eq!(
23497 server
23498 .worker_protocol_for("/api/worker/workflow-tasks/poll")
23499 .as_deref(),
23500 Some(WORKER_PROTOCOL_VERSION)
23501 );
23502 assert_eq!(
23503 server.worker_protocol_for("/api/worker/query-tasks/poll"),
23504 None,
23505 "a worker without query handlers must not use the query-task endpoint"
23506 );
23507 }
23508
23509 #[tokio::test]
23510 async fn completion_time_query_rejection_is_typed_without_stopping_worker() {
23511 let server = MockWorkerServer::reject_query_completion();
23512 let client = Client::builder(server.base_url())
23513 .timeout(Duration::from_secs(2))
23514 .build()
23515 .expect("client");
23516
23517 let error = client
23518 .complete_query_task("query-late", "late-worker", 1, json!(8), DEFAULT_CODEC)
23519 .await
23520 .expect_err("expired completion must be rejected");
23521 let Error::QueryFailed(failure) = error else {
23522 panic!("expected typed query failure");
23523 };
23524 assert_eq!(failure.status, 409);
23525 assert_eq!(failure.reason, "query_task_timed_out");
23526
23527 let mut worker = Worker::new(client, "rust-workers")
23528 .worker_id("late-worker")
23529 .poll_timeout(Duration::from_millis(10));
23530 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
23531 worker.register_query(
23532 "counter",
23533 "current",
23534 |_ctx, _args| async move { Ok(json!(8)) },
23535 );
23536
23537 assert_eq!(worker.run_once().await.expect("late task is handled"), 1);
23538 assert_eq!(
23539 worker
23540 .run_once()
23541 .await
23542 .expect("worker continues after late completion"),
23543 0
23544 );
23545 assert_eq!(
23546 server.request_count("/api/worker/query-tasks/query-late/complete"),
23547 2
23548 );
23549 assert_eq!(
23550 server.request_count("/api/worker/query-tasks/query-late/fail"),
23551 0,
23552 "a server completion rejection must not be reported as an encoding failure"
23553 );
23554 }
23555
23556 #[tokio::test]
23557 async fn normal_shutdown_joins_pollers_and_deregisters_once() {
23558 let server = MockWorkerServer::start();
23559 let client = Client::builder(server.base_url())
23560 .timeout(Duration::from_secs(2))
23561 .build()
23562 .expect("client");
23563 let mut worker = Worker::new(client, "rust-workers")
23564 .worker_id("joined-worker")
23565 .poll_timeout(Duration::from_millis(10));
23566 worker.register_workflow(
23567 "joined.workflow",
23568 |_ctx, _input| async move { Ok(Value::Null) },
23569 );
23570 worker.register_activity(
23571 "joined.activity",
23572 |_ctx, _input| async move { Ok(Value::Null) },
23573 );
23574 worker.register_query("joined.workflow", "state", |_ctx, _input| async move {
23575 Ok(Value::Null)
23576 });
23577
23578 worker
23579 .run_until(tokio::time::sleep(Duration::from_millis(20)))
23580 .await
23581 .expect("normal shutdown");
23582
23583 let deregistration_path = "/api/worker/registrations/mock-worker";
23584 assert_eq!(server.request_count(deregistration_path), 1);
23585 for poll_path in [
23586 "/api/worker/workflow-tasks/poll",
23587 "/api/worker/activity-tasks/poll",
23588 "/api/worker/query-tasks/poll",
23589 ] {
23590 assert!(server.request_count(poll_path) > 0, "missing {poll_path}");
23591 }
23592 assert_eq!(
23593 server.captured_paths().last().map(String::as_str),
23594 Some(deregistration_path),
23595 "deregistration must start only after every poller has joined"
23596 );
23597 }
23598
23599 #[tokio::test]
23600 async fn registration_failure_does_not_deregister() {
23601 let server = MockWorkerServer::rejected_registration();
23602 let client = Client::builder(server.base_url())
23603 .timeout(Duration::from_secs(2))
23604 .build()
23605 .expect("client");
23606 let worker = Worker::new(client, "rust-workers").worker_id("never-registered");
23607
23608 let error = worker
23609 .run_until(async {})
23610 .await
23611 .expect_err("registration must fail");
23612 assert!(matches!(
23613 error,
23614 Error::Http {
23615 status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23616 ..
23617 }
23618 ));
23619 assert!(server
23620 .captured_paths()
23621 .iter()
23622 .all(|path| !path.starts_with("/api/worker/registrations/")));
23623 }
23624
23625 #[tokio::test]
23626 async fn protocol_116_server_rejects_occurrence_identity_worker_registration() {
23627 let server = MockWorkerServer::rejected_registration_protocol();
23628 let client = Client::builder(server.base_url())
23629 .timeout(Duration::from_secs(2))
23630 .build()
23631 .expect("client");
23632 let worker = Worker::new(client, "rust-workers").worker_id("protocol-117-worker");
23633
23634 let error = worker
23635 .run_until(async {})
23636 .await
23637 .expect_err("a protocol 1.16 server must reject this worker");
23638 let Error::Protocol(failure) = error else {
23639 panic!("expected typed protocol rejection");
23640 };
23641 assert_eq!(failure.reason, "unsupported_protocol_version");
23642 assert_eq!(failure.supported_version.as_deref(), Some("1.16"));
23643 assert_eq!(failure.requested_version.as_deref(), Some("1.17"));
23644 assert_eq!(
23645 server
23646 .worker_protocol_for("/api/worker/register")
23647 .as_deref(),
23648 Some(WORKER_PROTOCOL_VERSION)
23649 );
23650 }
23651
23652 #[tokio::test]
23653 async fn declined_registration_does_not_deregister() {
23654 let server = MockWorkerServer::declined_registration();
23655 let client = Client::builder(server.base_url())
23656 .timeout(Duration::from_secs(2))
23657 .build()
23658 .expect("client");
23659 let worker = Worker::new(client, "rust-workers").worker_id("declined-worker");
23660
23661 let error = worker
23662 .run_until(async {})
23663 .await
23664 .expect_err("declined registration must fail");
23665 assert!(matches!(error, Error::WorkerLoop(_)));
23666 assert!(error.to_string().contains("was not accepted"));
23667 assert!(server
23668 .captured_paths()
23669 .iter()
23670 .all(|path| !path.starts_with("/api/worker/registrations/")));
23671 }
23672
23673 #[tokio::test]
23674 async fn deregistration_http_failure_is_returned_after_normal_shutdown() {
23675 let server = MockWorkerServer::rejected_deregistration();
23676 let client = Client::builder(server.base_url())
23677 .timeout(Duration::from_secs(2))
23678 .build()
23679 .expect("client");
23680 let worker = Worker::new(client, "rust-workers").worker_id("forbidden-cleanup");
23681
23682 let error = worker
23683 .run_until(async {})
23684 .await
23685 .expect_err("deregistration must fail");
23686 assert!(matches!(
23687 error,
23688 Error::Http {
23689 status: reqwest::StatusCode::FORBIDDEN,
23690 ..
23691 }
23692 ));
23693 assert_eq!(
23694 server.request_count("/api/worker/registrations/mock-worker"),
23695 1
23696 );
23697 }
23698
23699 #[tokio::test]
23700 async fn deregistration_protocol_failure_is_returned_after_normal_shutdown() {
23701 let server = MockWorkerServer::rejected_deregistration_protocol();
23702 let client = Client::builder(server.base_url())
23703 .timeout(Duration::from_secs(2))
23704 .build()
23705 .expect("client");
23706 let worker = Worker::new(client, "rust-workers").worker_id("protocol-cleanup");
23707
23708 let error = worker
23709 .run_until(async {})
23710 .await
23711 .expect_err("protocol rejection must fail shutdown");
23712 let Error::Protocol(failure) = error else {
23713 panic!("expected typed protocol failure");
23714 };
23715 assert_eq!(failure.reason, "unsupported_protocol_version");
23716 assert_eq!(
23717 failure.requested_version.as_deref(),
23718 Some(WORKER_PROTOCOL_VERSION)
23719 );
23720 assert_eq!(
23721 server.request_count("/api/worker/registrations/mock-worker"),
23722 1
23723 );
23724 }
23725
23726 #[tokio::test]
23727 async fn primary_poller_error_retains_deregistration_failure_context() {
23728 let server = MockWorkerServer::unauthorized_polls_and_rejected_deregistration();
23729 let client = Client::builder(server.base_url())
23730 .timeout(Duration::from_secs(2))
23731 .build()
23732 .expect("client");
23733 let mut worker = Worker::new(client, "rust-workers")
23734 .worker_id("combined-failure")
23735 .poll_timeout(Duration::from_millis(10));
23736 worker.register_workflow("combined.workflow", |_ctx, _input| async move {
23737 Ok(Value::Null)
23738 });
23739
23740 let error = worker
23741 .run()
23742 .await
23743 .expect_err("worker and cleanup must fail");
23744 let summary = error.to_string();
23745 assert!(summary.contains("authentication_failed"));
23746 assert!(summary.contains("worker cannot deregister"));
23747 let Error::WorkerShutdown {
23748 primary,
23749 deregistration,
23750 } = error
23751 else {
23752 panic!("expected combined worker shutdown error");
23753 };
23754 assert!(matches!(
23755 *primary,
23756 Error::Http {
23757 status: reqwest::StatusCode::UNAUTHORIZED,
23758 ..
23759 }
23760 ));
23761 assert!(matches!(
23762 *deregistration,
23763 Error::Http {
23764 status: reqwest::StatusCode::FORBIDDEN,
23765 ..
23766 }
23767 ));
23768 assert_eq!(
23769 server.request_count("/api/worker/registrations/mock-worker"),
23770 1
23771 );
23772 }
23773
23774 #[tokio::test]
23775 async fn activity_only_worker_can_shutdown_without_workflow_poller() {
23776 let server = MockWorkerServer::start();
23777 let client = Client::builder(server.base_url())
23778 .timeout(Duration::from_secs(2))
23779 .build()
23780 .expect("client");
23781 let mut worker = Worker::new(client, "rust-workers")
23782 .worker_id("activity-only-worker")
23783 .poll_timeout(Duration::from_millis(10));
23784
23785 worker.register_activity(
23786 "activity.only",
23787 |_ctx, _args| async move { Ok(Value::Null) },
23788 );
23789
23790 worker.run_until(async {}).await.expect("run worker");
23791 }
23792
23793 #[tokio::test]
23794 async fn workflow_only_worker_can_shutdown_without_activity_poller() {
23795 let server = MockWorkerServer::start();
23796 let client = Client::builder(server.base_url())
23797 .timeout(Duration::from_secs(2))
23798 .build()
23799 .expect("client");
23800 let mut worker = Worker::new(client, "rust-workers")
23801 .worker_id("workflow-only-worker")
23802 .poll_timeout(Duration::from_millis(10));
23803
23804 worker.register_workflow(
23805 "workflow.only",
23806 |_ctx, _input| async move { Ok(Value::Null) },
23807 );
23808
23809 worker.run_until(async {}).await.expect("run worker");
23810 }
23811
23812 #[tokio::test]
23813 async fn worker_heartbeat_observer_receives_server_acknowledgements() {
23814 let server = MockWorkerServer::start();
23815 let client = Client::builder(server.base_url())
23816 .timeout(Duration::from_secs(2))
23817 .build()
23818 .expect("client");
23819 let observations = Arc::new(Mutex::new(Vec::new()));
23820 let observed = Arc::clone(&observations);
23821 let mut worker = Worker::new(client, "rust-workers")
23822 .worker_id("observed-heartbeat-worker")
23823 .poll_timeout(Duration::from_millis(10))
23824 .on_worker_heartbeat(move |observation| {
23825 observed
23826 .lock()
23827 .expect("heartbeat observations")
23828 .push(observation.clone());
23829 });
23830
23831 worker.register_workflow("workflow.observed", |_ctx, _input| async move {
23832 Ok(Value::Null)
23833 });
23834 let acknowledged = Arc::clone(&observations);
23835 worker
23836 .run_until(async move {
23837 tokio::time::timeout(Duration::from_secs(2), async move {
23838 loop {
23839 if !acknowledged
23840 .lock()
23841 .expect("heartbeat observations")
23842 .is_empty()
23843 {
23844 break;
23845 }
23846 tokio::time::sleep(Duration::from_millis(1)).await;
23847 }
23848 })
23849 .await
23850 .expect("heartbeat acknowledgement within timeout");
23851 })
23852 .await
23853 .expect("run worker");
23854
23855 let observations = observations.lock().expect("heartbeat observations");
23856 let first = observations.first().expect("heartbeat acknowledgement");
23857 assert_eq!(first.worker_id, "observed-heartbeat-worker");
23858 assert_eq!(first.task_queue, "rust-workers");
23859 assert!(first.acknowledged_at_unix_millis > 0);
23860 assert_eq!(first.acknowledgement, json!({}));
23861 }
23862
23863 #[tokio::test]
23864 async fn delayed_worker_heartbeat_keeps_cadence_and_pollers_live() {
23865 let server = MockWorkerServer::delayed_heartbeat_worker();
23866 let client = Client::builder(server.base_url())
23867 .timeout(Duration::from_secs(3))
23868 .build()
23869 .expect("client");
23870 let observations = Arc::new(Mutex::new(Vec::new()));
23871 let observed = Arc::clone(&observations);
23872 let mut worker = Worker::new(client, "rust-snapshot-workers")
23873 .worker_id("rust-snapshot-worker")
23874 .poll_timeout(Duration::from_millis(10))
23875 .on_worker_heartbeat(move |observation| {
23876 observed
23877 .lock()
23878 .expect("heartbeat observations")
23879 .push(observation.clone());
23880 });
23881
23882 worker.register_workflow("snapshot", |ctx, _input| async move {
23883 ctx.wait_signal("finish").await?;
23884 Ok(json!({"status": "finished"}))
23885 });
23886 worker.register_query("snapshot", "current", |ctx, _args| async move {
23887 Ok(json!(ctx
23888 .signals("increment")
23889 .iter()
23890 .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
23891 .sum::<i64>()))
23892 });
23893 worker.register_activity("cancel-aware", |_ctx, _args| async move {
23894 Ok(json!({"late": "completion"}))
23895 });
23896
23897 worker
23898 .run_until(tokio::time::sleep(Duration::from_millis(3_800)))
23899 .await
23900 .expect("delayed heartbeat must allow a clean worker shutdown");
23901
23902 let observations = observations.lock().expect("heartbeat observations");
23903 assert!(
23904 observations.len() >= 3,
23905 "the immediate heartbeat, delayed acknowledgement, and next cadence heartbeat must complete"
23906 );
23907 assert!(
23908 observations.windows(2).all(|pair| {
23909 pair[1].acknowledged_at_unix_millis
23910 .saturating_sub(pair[0].acknowledged_at_unix_millis)
23911 >= 850
23912 }),
23913 "successful acknowledgements must not catch up faster than the advertised one-second cadence: {observations:?}"
23914 );
23915 drop(observations);
23916
23917 let heartbeat_times = server.request_times("/api/worker/heartbeat");
23918 let delayed_request_at = *heartbeat_times
23919 .get(1)
23920 .expect("intentionally delayed heartbeat request");
23921 let delay_window_start = delayed_request_at + Duration::from_millis(100);
23922 let delay_window_end = delayed_request_at + Duration::from_millis(1_400);
23923 for path in [
23924 "/api/worker/workflow-tasks/poll",
23925 "/api/worker/activity-tasks/poll",
23926 "/api/worker/query-tasks/poll",
23927 ] {
23928 assert!(
23929 server
23930 .request_times(path)
23931 .iter()
23932 .any(|received_at| *received_at >= delay_window_start
23933 && *received_at <= delay_window_end),
23934 "{path} must keep polling while a heartbeat acknowledgement is delayed"
23935 );
23936 }
23937 assert!(
23938 server.request_count("/api/worker/workflow-tasks/snapshot-wait-3/fail") >= 1,
23939 "workflow work must be settled"
23940 );
23941 assert!(
23942 server.request_count("/api/worker/activity-tasks/activity-cancel/complete") >= 1,
23943 "activity work must be settled"
23944 );
23945 assert!(
23946 server.request_count("/api/worker/query-tasks/snapshot-current/complete") >= 1,
23947 "query work must be settled"
23948 );
23949 }
23950
23951 #[tokio::test]
23952 async fn retried_worker_heartbeat_restarts_the_advertised_cadence() {
23953 let server = MockWorkerServer::heartbeat_retry_worker();
23954 let client = Client::builder(server.base_url())
23955 .timeout(Duration::from_secs(2))
23956 .build()
23957 .expect("client");
23958 let observations = Arc::new(Mutex::new(Vec::new()));
23959 let observed = Arc::clone(&observations);
23960 let worker = Worker::new(client, "rust-workers")
23961 .worker_id("heartbeat-retry-worker")
23962 .retry_policy(WorkerRetryPolicy {
23963 max_retries: 1,
23964 initial_backoff: Duration::from_millis(300),
23965 max_backoff: Duration::from_millis(300),
23966 })
23967 .on_worker_heartbeat(move |observation| {
23968 observed
23969 .lock()
23970 .expect("heartbeat observations")
23971 .push(observation.clone());
23972 });
23973
23974 worker
23975 .run_until(tokio::time::sleep(Duration::from_millis(2_700)))
23976 .await
23977 .expect("retryable heartbeat failure must remain bounded and recover");
23978
23979 let observations = observations.lock().expect("heartbeat observations");
23980 assert!(observations.len() >= 3, "heartbeat retry must recover");
23981 assert!(
23982 observations.windows(2).all(|pair| {
23983 pair[1]
23984 .acknowledged_at_unix_millis
23985 .saturating_sub(pair[0].acknowledged_at_unix_millis)
23986 >= 850
23987 }),
23988 "a successful retry must start a fresh advertised cadence: {observations:?}"
23989 );
23990 assert_eq!(
23991 server.request_count("/api/worker/heartbeat"),
23992 observations.len() + 1,
23993 "one retryable failure must add exactly one bounded request"
23994 );
23995 }
23996
23997 #[tokio::test]
23998 async fn query_enabled_worker_ignores_unmatched_signals_then_completes_once() {
23999 let server = MockWorkerServer::waiting_query_worker();
24000 let client = Client::builder(server.base_url())
24001 .timeout(Duration::from_secs(2))
24002 .build()
24003 .expect("client");
24004 let observations = Arc::new(Mutex::new(Vec::new()));
24005 let observed = Arc::clone(&observations);
24006 let mut worker = Worker::new(client, "rust-snapshot-workers")
24007 .worker_id("rust-snapshot-worker")
24008 .poll_timeout(Duration::from_millis(10))
24009 .on_worker_heartbeat(move |observation| {
24010 observed
24011 .lock()
24012 .expect("heartbeat observations")
24013 .push(observation.clone());
24014 });
24015
24016 worker.register_workflow("snapshot", |ctx, _input| async move {
24017 ctx.wait_signal("finish").await?;
24018 Ok(json!({"status": "finished"}))
24019 });
24020 worker.register_query("snapshot", "current", |ctx, _args| async move {
24021 let current = ctx
24022 .signals("increment")
24023 .iter()
24024 .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
24025 .sum::<i64>();
24026 Ok(json!(current))
24027 });
24028 worker.register_update("snapshot", "replace", |_ctx, args| async move { Ok(args) });
24029
24030 worker
24031 .run_until(tokio::time::sleep(Duration::from_millis(3_200)))
24032 .await
24033 .expect("pending workflow and query poller must remain live until shutdown");
24034
24035 assert!(
24036 observations.lock().expect("heartbeat observations").len() >= 4,
24037 "the immediate heartbeat and at least three advertised one-second intervals must be acknowledged"
24038 );
24039 assert!(
24040 server.request_count("/api/worker/workflow-tasks/poll") >= 3,
24041 "workflow polling must continue after empty replay acknowledgements"
24042 );
24043 assert!(
24044 server.request_count("/api/worker/query-tasks/poll") >= 2,
24045 "query polling must continue after serving the current query"
24046 );
24047 assert_eq!(
24048 server.request_body("/api/worker/register")["capabilities"],
24049 json!([
24050 CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY,
24051 DURABLE_SELECTION_CAPABILITY,
24052 MEMO_UPSERTS_CAPABILITY,
24053 TYPED_SEARCH_ATTRIBUTES_CAPABILITY,
24054 QUERY_TASKS_CAPABILITY,
24055 WORKFLOW_UPDATES_CAPABILITY,
24056 MESSAGE_STREAMS_CAPABILITY
24057 ])
24058 );
24059 assert_eq!(
24060 server.request_body("/api/worker/register")["workflow_command_contracts"]["snapshot"],
24061 json!({
24062 "queries": ["current"],
24063 "query_contracts": [],
24064 "signals": [],
24065 "signal_contracts": [],
24066 "updates": ["replace"],
24067 "update_contracts": [],
24068 "update_validators": [],
24069 })
24070 );
24071
24072 let opened = server.request_body("/api/worker/workflow-tasks/snapshot-open/complete");
24073 assert_eq!(
24074 opened["commands"],
24075 json!([{
24076 "type": "open_signal_wait",
24077 "signal_name": "finish",
24078 }])
24079 );
24080
24081 for task_id in ["snapshot-wait-3", "snapshot-wait-5"] {
24082 let fail_path = format!("/api/worker/workflow-tasks/{task_id}/fail");
24083 let completion_path = format!("/api/worker/workflow-tasks/{task_id}/complete");
24084 let failure = server.request_body(&fail_path);
24085 assert_eq!(
24086 failure["failure"]["type"],
24087 WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE
24088 );
24089 assert_eq!(server.request_count(&completion_path), 0);
24090 }
24091
24092 let query_completion =
24093 server.request_body("/api/worker/query-tasks/snapshot-current/complete");
24094 assert_eq!(query_completion["result"], json!(8));
24095
24096 let terminal_path = "/api/worker/workflow-tasks/snapshot-finish/complete";
24097 assert_eq!(
24098 server.request_count(terminal_path),
24099 1,
24100 "the matching signal must settle the workflow exactly once"
24101 );
24102 let terminal = server.request_body(terminal_path);
24103 assert_eq!(terminal["commands"].as_array().map(Vec::len), Some(1));
24104 assert_eq!(terminal["commands"][0]["type"], "complete_workflow");
24105 assert_eq!(
24106 decode_wire_value(&terminal["commands"][0]["result"], DEFAULT_CODEC)
24107 .expect("terminal workflow result"),
24108 json!({"status": "finished"})
24109 );
24110 }
24111
24112 #[tokio::test]
24113 async fn worker_retries_poll_and_heartbeat_transport_failures_independently() {
24114 let server = MockWorkerServer::transient_worker_failures();
24115 let client = Client::builder(server.base_url())
24116 .timeout(Duration::from_secs(2))
24117 .build()
24118 .expect("client");
24119 let mut worker = Worker::new(client, "rust-workers")
24120 .worker_id("retry-worker")
24121 .poll_timeout(Duration::from_millis(10))
24122 .retry_policy(WorkerRetryPolicy {
24123 max_retries: 2,
24124 initial_backoff: Duration::from_millis(1),
24125 max_backoff: Duration::from_millis(1),
24126 });
24127 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
24128 worker.register_activity(
24129 "counter.activity",
24130 |_ctx, _input| async move { Ok(Value::Null) },
24131 );
24132 worker.register_query(
24133 "counter",
24134 "current",
24135 |_ctx, _args| async move { Ok(json!(8)) },
24136 );
24137
24138 worker
24139 .run_until(tokio::time::sleep(Duration::from_millis(75)))
24140 .await
24141 .expect("transient failures must not stop the worker");
24142
24143 for path in [
24144 "/api/worker/heartbeat",
24145 "/api/worker/workflow-tasks/poll",
24146 "/api/worker/activity-tasks/poll",
24147 "/api/worker/query-tasks/poll",
24148 ] {
24149 assert!(
24150 server.request_count(path) >= 2,
24151 "{path} must continue after its transient failure"
24152 );
24153 }
24154 }
24155
24156 #[tokio::test]
24157 async fn worker_continues_after_long_poll_capacity_backpressure() {
24158 let server = MockWorkerServer::capacity_limited_activity_poll();
24159 let client = Client::builder(server.base_url())
24160 .timeout(Duration::from_secs(2))
24161 .build()
24162 .expect("client");
24163 let mut worker = Worker::new(client, "rust-workers")
24164 .worker_id("capacity-worker")
24165 .poll_timeout(Duration::from_millis(10))
24166 .retry_policy(WorkerRetryPolicy {
24167 max_retries: 0,
24168 initial_backoff: Duration::from_millis(1),
24169 max_backoff: Duration::from_millis(1),
24170 });
24171 worker.register_activity("capacity.activity", |_ctx, _input| async move {
24172 Ok(json!({"handled": true}))
24173 });
24174
24175 worker
24176 .run_until(tokio::time::sleep(Duration::from_millis(50)))
24177 .await
24178 .expect("capacity backpressure must not stop the worker");
24179
24180 assert!(
24181 server.request_count("/api/worker/activity-tasks/poll") >= 2,
24182 "the activity poller must continue after capacity backpressure"
24183 );
24184 assert_eq!(
24185 server.request_count("/api/worker/activity-tasks/capacity-activity/complete"),
24186 1,
24187 "the worker must complete work returned after capacity recovers"
24188 );
24189 }
24190
24191 #[test]
24192 fn worker_poll_capacity_backpressure_requires_the_typed_retryable_contract() {
24193 let capacity = Error::Http {
24194 status: reqwest::StatusCode::TOO_MANY_REQUESTS,
24195 body: r#"{"poll_status":"long_poll_capacity_exhausted","retryable":true,"retry_after_seconds":3}"#.to_string(),
24196 };
24197 assert_eq!(
24198 worker_poll_capacity_retry_after(&capacity),
24199 Some(Duration::from_secs(3))
24200 );
24201
24202 let rejected_capacity = Error::Http {
24203 status: reqwest::StatusCode::TOO_MANY_REQUESTS,
24204 body: r#"{"reason":"long_poll_capacity_exhausted","retryable":false,"retry_after_seconds":3}"#.to_string(),
24205 };
24206 assert_eq!(worker_poll_capacity_retry_after(&rejected_capacity), None);
24207 assert!(!worker_operation_is_retryable(&rejected_capacity));
24208
24209 let ordinary_rate_limit = Error::Http {
24210 status: reqwest::StatusCode::TOO_MANY_REQUESTS,
24211 body: r#"{"reason":"rate_limited","retryable":true,"retry_after_seconds":3}"#
24212 .to_string(),
24213 };
24214 assert_eq!(worker_poll_capacity_retry_after(&ordinary_rate_limit), None);
24215 assert!(worker_operation_is_retryable(&ordinary_rate_limit));
24216 }
24217
24218 #[tokio::test]
24219 async fn worker_bounds_transport_retries() {
24220 let server = MockWorkerServer::unavailable_polls();
24221 let client = Client::builder(server.base_url())
24222 .timeout(Duration::from_secs(2))
24223 .build()
24224 .expect("client");
24225 let mut worker = Worker::new(client, "rust-workers")
24226 .worker_id("bounded-retry-worker")
24227 .poll_timeout(Duration::from_millis(10))
24228 .retry_policy(WorkerRetryPolicy {
24229 max_retries: 2,
24230 initial_backoff: Duration::from_millis(1),
24231 max_backoff: Duration::from_millis(1),
24232 });
24233 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
24234
24235 let error = worker.run().await.expect_err("retry bound must terminate");
24236 assert!(matches!(error, Error::Transport(_)));
24237 assert_eq!(
24238 server.request_count("/api/worker/workflow-tasks/poll"),
24239 3,
24240 "one initial request plus exactly two retries"
24241 );
24242 }
24243
24244 #[tokio::test]
24245 async fn worker_retry_policy_can_disable_poll_retries() {
24246 let server = MockWorkerServer::unavailable_polls();
24247 let client = Client::builder(server.base_url())
24248 .timeout(Duration::from_secs(2))
24249 .build()
24250 .expect("client");
24251 let mut worker = Worker::new(client, "rust-workers")
24252 .worker_id("no-retry-worker")
24253 .poll_timeout(Duration::from_millis(10))
24254 .retry_policy(WorkerRetryPolicy {
24255 max_retries: 0,
24256 initial_backoff: Duration::from_millis(1),
24257 max_backoff: Duration::from_millis(1),
24258 });
24259 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
24260
24261 let error = worker
24262 .run_once()
24263 .await
24264 .expect_err("disabled retries must return the first transport failure");
24265 assert!(matches!(error, Error::Transport(_)));
24266 assert_eq!(
24267 server.request_count("/api/worker/workflow-tasks/poll"),
24268 1,
24269 "max_retries=0 must send only the initial request"
24270 );
24271 }
24272
24273 #[tokio::test]
24274 async fn worker_does_not_retry_authentication_failures() {
24275 let server = MockWorkerServer::unauthorized_polls();
24276 let client = Client::builder(server.base_url())
24277 .timeout(Duration::from_secs(2))
24278 .build()
24279 .expect("client");
24280 let mut worker = Worker::new(client, "rust-workers")
24281 .worker_id("unauthorized-worker")
24282 .poll_timeout(Duration::from_millis(10));
24283 worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
24284
24285 let error = worker
24286 .run()
24287 .await
24288 .expect_err("authentication must terminate");
24289 let Error::Http { status, body } = error else {
24290 panic!("expected stable HTTP authentication error");
24291 };
24292 assert_eq!(status, reqwest::StatusCode::UNAUTHORIZED);
24293 assert!(body.contains("authentication_failed"));
24294 assert_eq!(
24295 server.request_count("/api/worker/workflow-tasks/poll"),
24296 1,
24297 "authentication failures must not be retried"
24298 );
24299 }
24300
24301 #[derive(Clone, Debug)]
24302 struct CapturedRequest {
24303 method: String,
24304 path: String,
24305 authorization: Option<String>,
24306 namespace: Option<String>,
24307 worker_protocol: Option<String>,
24308 control_protocol: Option<String>,
24309 body: String,
24310 received_at: Instant,
24311 }
24312
24313 struct MockWorkerServer {
24314 addr: SocketAddr,
24315 stop: Arc<AtomicBool>,
24316 requests: Arc<Mutex<Vec<CapturedRequest>>>,
24317 thread: Option<thread::JoinHandle<()>>,
24318 }
24319
24320 #[derive(Clone, Copy, Default)]
24321 struct MockWorkerBehavior {
24322 storage_refusals: usize,
24323 storage_path: Option<&'static str>,
24324 storage_unavailable: bool,
24325 storage_mid_poll: bool,
24326 storage_activity: bool,
24327 storage_query: bool,
24328 storage_wrong_poll_id: bool,
24329 reject_query_protocol: bool,
24330 reject_query_completion: bool,
24331 waiting_query_worker: bool,
24332 decline_registration: bool,
24333 complete_named_signal: bool,
24334 poll_failures_per_path: usize,
24335 long_poll_capacity_responses_per_path: usize,
24336 heartbeat_failures: usize,
24337 heartbeat_failure_request: Option<usize>,
24338 delayed_heartbeat_request: Option<usize>,
24339 heartbeat_response_delay: Duration,
24340 concurrent_requests: bool,
24341 unauthorized_polls: bool,
24342 reject_registration: bool,
24343 reject_registration_protocol: bool,
24344 reject_deregistration: bool,
24345 reject_deregistration_protocol: bool,
24346 cancelled_activity: bool,
24347 draining_polls: bool,
24348 invalid_task_payload_codec: Option<InvalidTaskPayloadCodec>,
24349 workflow_completion_status: Option<&'static str>,
24350 workflow_completion_body: Option<&'static str>,
24351 }
24352
24353 impl MockWorkerServer {
24354 fn start() -> Self {
24355 Self::start_with_behavior(MockWorkerBehavior::default())
24356 }
24357
24358 fn reject_query_protocol() -> Self {
24359 Self::start_with_behavior(MockWorkerBehavior {
24360 reject_query_protocol: true,
24361 ..MockWorkerBehavior::default()
24362 })
24363 }
24364
24365 fn reject_query_completion() -> Self {
24366 Self::start_with_behavior(MockWorkerBehavior {
24367 reject_query_completion: true,
24368 ..MockWorkerBehavior::default()
24369 })
24370 }
24371
24372 fn waiting_query_worker() -> Self {
24373 Self::start_with_behavior(MockWorkerBehavior {
24374 waiting_query_worker: true,
24375 complete_named_signal: true,
24376 ..MockWorkerBehavior::default()
24377 })
24378 }
24379
24380 fn transient_worker_failures() -> Self {
24381 Self::start_with_behavior(MockWorkerBehavior {
24382 poll_failures_per_path: 1,
24383 heartbeat_failures: 1,
24384 ..MockWorkerBehavior::default()
24385 })
24386 }
24387
24388 fn consecutive_poll_failures(count: usize) -> Self {
24389 Self::start_with_behavior(MockWorkerBehavior {
24390 poll_failures_per_path: count,
24391 ..MockWorkerBehavior::default()
24392 })
24393 }
24394
24395 fn capacity_limited_activity_poll() -> Self {
24396 Self::start_with_behavior(MockWorkerBehavior {
24397 long_poll_capacity_responses_per_path: 1,
24398 ..MockWorkerBehavior::default()
24399 })
24400 }
24401
24402 fn delayed_heartbeat_worker() -> Self {
24403 Self::start_with_behavior(MockWorkerBehavior {
24404 waiting_query_worker: true,
24405 delayed_heartbeat_request: Some(2),
24406 heartbeat_response_delay: Duration::from_millis(1_500),
24407 concurrent_requests: true,
24408 cancelled_activity: true,
24409 ..MockWorkerBehavior::default()
24410 })
24411 }
24412
24413 fn heartbeat_retry_worker() -> Self {
24414 Self::start_with_behavior(MockWorkerBehavior {
24415 waiting_query_worker: true,
24416 heartbeat_failure_request: Some(2),
24417 concurrent_requests: true,
24418 ..MockWorkerBehavior::default()
24419 })
24420 }
24421
24422 fn unavailable_polls() -> Self {
24423 Self::start_with_behavior(MockWorkerBehavior {
24424 poll_failures_per_path: usize::MAX,
24425 ..MockWorkerBehavior::default()
24426 })
24427 }
24428
24429 fn unauthorized_polls() -> Self {
24430 Self::start_with_behavior(MockWorkerBehavior {
24431 unauthorized_polls: true,
24432 ..MockWorkerBehavior::default()
24433 })
24434 }
24435
24436 fn rejected_registration() -> Self {
24437 Self::start_with_behavior(MockWorkerBehavior {
24438 reject_registration: true,
24439 ..MockWorkerBehavior::default()
24440 })
24441 }
24442
24443 fn rejected_registration_protocol() -> Self {
24444 Self::start_with_behavior(MockWorkerBehavior {
24445 reject_registration_protocol: true,
24446 ..MockWorkerBehavior::default()
24447 })
24448 }
24449
24450 fn declined_registration() -> Self {
24451 Self::start_with_behavior(MockWorkerBehavior {
24452 decline_registration: true,
24453 ..MockWorkerBehavior::default()
24454 })
24455 }
24456
24457 fn rejected_deregistration() -> Self {
24458 Self::start_with_behavior(MockWorkerBehavior {
24459 reject_deregistration: true,
24460 ..MockWorkerBehavior::default()
24461 })
24462 }
24463
24464 fn rejected_deregistration_protocol() -> Self {
24465 Self::start_with_behavior(MockWorkerBehavior {
24466 reject_deregistration_protocol: true,
24467 ..MockWorkerBehavior::default()
24468 })
24469 }
24470
24471 fn unauthorized_polls_and_rejected_deregistration() -> Self {
24472 Self::start_with_behavior(MockWorkerBehavior {
24473 unauthorized_polls: true,
24474 reject_deregistration: true,
24475 ..MockWorkerBehavior::default()
24476 })
24477 }
24478
24479 fn cancelled_activity() -> Self {
24480 Self::start_with_behavior(MockWorkerBehavior {
24481 cancelled_activity: true,
24482 ..MockWorkerBehavior::default()
24483 })
24484 }
24485
24486 fn draining_polls() -> Self {
24487 Self::start_with_behavior(MockWorkerBehavior {
24488 draining_polls: true,
24489 ..MockWorkerBehavior::default()
24490 })
24491 }
24492
24493 fn invalid_task_payload_codec(codec: InvalidTaskPayloadCodec) -> Self {
24494 Self::start_with_behavior(MockWorkerBehavior {
24495 invalid_task_payload_codec: Some(codec),
24496 ..MockWorkerBehavior::default()
24497 })
24498 }
24499
24500 fn workflow_completion(status: &'static str, body: &'static str) -> Self {
24501 Self::start_with_behavior(MockWorkerBehavior {
24502 workflow_completion_status: Some(status),
24503 workflow_completion_body: Some(body),
24504 ..MockWorkerBehavior::default()
24505 })
24506 }
24507
24508 fn start_with_behavior(behavior: MockWorkerBehavior) -> Self {
24509 let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock server");
24510 listener
24511 .set_nonblocking(true)
24512 .expect("configure mock listener");
24513 let addr = listener.local_addr().expect("mock server address");
24514 let stop = Arc::new(AtomicBool::new(false));
24515 let server_stop = Arc::clone(&stop);
24516 let requests = Arc::new(Mutex::new(Vec::new()));
24517 let server_requests = Arc::clone(&requests);
24518 let thread = thread::spawn(move || {
24519 let mut request_threads = Vec::new();
24520 while !server_stop.load(Ordering::SeqCst) {
24521 match listener.accept() {
24522 Ok((mut stream, _)) => {
24523 if behavior.concurrent_requests {
24524 let requests = Arc::clone(&server_requests);
24525 request_threads.push(thread::spawn(move || {
24526 handle_mock_worker_request(&mut stream, &requests, behavior)
24527 }));
24528 } else {
24529 handle_mock_worker_request(&mut stream, &server_requests, behavior);
24530 }
24531 }
24532 Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
24533 let mut index = 0;
24534 while index < request_threads.len() {
24535 if request_threads[index].is_finished() {
24536 request_threads
24537 .swap_remove(index)
24538 .join()
24539 .expect("join mock request");
24540 } else {
24541 index += 1;
24542 }
24543 }
24544 thread::sleep(Duration::from_millis(5));
24545 }
24546 Err(_) => break,
24547 }
24548 }
24549 for request_thread in request_threads {
24550 request_thread.join().expect("join mock request");
24551 }
24552 });
24553
24554 Self {
24555 addr,
24556 stop,
24557 requests,
24558 thread: Some(thread),
24559 }
24560 }
24561
24562 fn base_url(&self) -> String {
24563 format!("http://{}", self.addr)
24564 }
24565
24566 fn worker_protocol_for(&self, path: &str) -> Option<String> {
24567 self.requests
24568 .lock()
24569 .expect("captured requests")
24570 .iter()
24571 .find(|request| request.path == path)
24572 .and_then(|request| request.worker_protocol.clone())
24573 }
24574
24575 fn control_protocol_for(&self, path: &str) -> Option<String> {
24576 self.requests
24577 .lock()
24578 .expect("captured requests")
24579 .iter()
24580 .find(|request| request.path == path)
24581 .and_then(|request| request.control_protocol.clone())
24582 }
24583
24584 fn method_for(&self, path: &str) -> Option<String> {
24585 self.requests
24586 .lock()
24587 .expect("captured requests")
24588 .iter()
24589 .find(|request| request.path == path)
24590 .map(|request| request.method.clone())
24591 }
24592
24593 fn authorization_for(&self, path: &str) -> Option<String> {
24594 self.requests
24595 .lock()
24596 .expect("captured requests")
24597 .iter()
24598 .find(|request| request.path == path)
24599 .and_then(|request| request.authorization.clone())
24600 }
24601
24602 fn namespace_for(&self, path: &str) -> Option<String> {
24603 self.requests
24604 .lock()
24605 .expect("captured requests")
24606 .iter()
24607 .find(|request| request.path == path)
24608 .and_then(|request| request.namespace.clone())
24609 }
24610
24611 fn request_count(&self, path: &str) -> usize {
24612 self.requests
24613 .lock()
24614 .expect("captured requests")
24615 .iter()
24616 .filter(|request| request.path == path)
24617 .count()
24618 }
24619
24620 fn captured_paths(&self) -> Vec<String> {
24621 self.requests
24622 .lock()
24623 .expect("captured requests")
24624 .iter()
24625 .map(|request| request.path.clone())
24626 .collect()
24627 }
24628
24629 fn request_times(&self, path: &str) -> Vec<Instant> {
24630 self.requests
24631 .lock()
24632 .expect("captured requests")
24633 .iter()
24634 .filter(|request| request.path == path)
24635 .map(|request| request.received_at)
24636 .collect()
24637 }
24638
24639 fn request_body(&self, path: &str) -> Value {
24640 let requests = self.requests.lock().expect("captured requests");
24641 let body = &requests
24642 .iter()
24643 .find(|request| request.path == path)
24644 .unwrap_or_else(|| panic!("missing request for {path}"))
24645 .body;
24646 serde_json::from_str(body).unwrap_or_else(|error| {
24647 panic!("invalid JSON request body for {path}: {error}: {body:?}")
24648 })
24649 }
24650
24651 fn request_bodies(&self, path: &str) -> Vec<Value> {
24652 self.requests
24653 .lock()
24654 .expect("captured requests")
24655 .iter()
24656 .filter(|request| request.path == path)
24657 .map(|request| {
24658 serde_json::from_str(&request.body).unwrap_or_else(|error| {
24659 panic!(
24660 "invalid JSON request body for {path}: {error}: {:?}",
24661 request.body
24662 )
24663 })
24664 })
24665 .collect()
24666 }
24667 }
24668
24669 impl Drop for MockWorkerServer {
24670 fn drop(&mut self) {
24671 self.stop.store(true, Ordering::SeqCst);
24672 let _ = TcpStream::connect(self.addr);
24673
24674 if let Some(thread) = self.thread.take() {
24675 thread.join().expect("join mock server");
24676 }
24677 }
24678 }
24679
24680 fn handle_mock_worker_request(
24681 stream: &mut TcpStream,
24682 requests: &Arc<Mutex<Vec<CapturedRequest>>>,
24683 behavior: MockWorkerBehavior,
24684 ) {
24685 let _ = stream.set_read_timeout(Some(Duration::from_millis(200)));
24686 let mut buffer = [0_u8; 8192];
24687 let mut request = Vec::new();
24688
24689 loop {
24690 match stream.read(&mut buffer) {
24691 Ok(0) => break,
24692 Ok(read) => {
24693 request.extend_from_slice(&buffer[..read]);
24694 if mock_request_is_complete(&request) {
24695 break;
24696 }
24697 }
24698 Err(error)
24699 if matches!(
24700 error.kind(),
24701 std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
24702 ) =>
24703 {
24704 break;
24705 }
24706 Err(_) => return,
24707 }
24708 }
24709
24710 let request = String::from_utf8_lossy(&request);
24711 let body = request
24712 .split_once("\r\n\r\n")
24713 .map(|(_, body)| body)
24714 .unwrap_or_default();
24715 let path = request
24716 .lines()
24717 .next()
24718 .and_then(|line| line.split_whitespace().nth(1))
24719 .unwrap_or_default();
24720 let method = request
24721 .lines()
24722 .next()
24723 .and_then(|line| line.split_whitespace().next())
24724 .unwrap_or_default();
24725 let authorization = request.lines().find_map(|line| {
24726 let (name, value) = line.split_once(':')?;
24727 name.eq_ignore_ascii_case("Authorization")
24728 .then(|| value.trim().to_string())
24729 });
24730 let namespace = request.lines().find_map(|line| {
24731 let (name, value) = line.split_once(':')?;
24732 name.eq_ignore_ascii_case("X-Namespace")
24733 .then(|| value.trim().to_string())
24734 });
24735 let worker_protocol = request.lines().find_map(|line| {
24736 let (name, value) = line.split_once(':')?;
24737 name.eq_ignore_ascii_case("X-Durable-Workflow-Protocol-Version")
24738 .then(|| value.trim().to_string())
24739 });
24740 let control_protocol = request.lines().find_map(|line| {
24741 let (name, value) = line.split_once(':')?;
24742 name.eq_ignore_ascii_case("X-Durable-Workflow-Control-Plane-Version")
24743 .then(|| value.trim().to_string())
24744 });
24745 let request_number = {
24746 let mut requests = requests.lock().expect("captured requests");
24747 requests.push(CapturedRequest {
24748 method: method.to_string(),
24749 path: path.to_string(),
24750 authorization,
24751 namespace,
24752 worker_protocol: worker_protocol.clone(),
24753 control_protocol,
24754 body: body.to_string(),
24755 received_at: Instant::now(),
24756 });
24757 requests
24758 .iter()
24759 .filter(|request| request.path == path)
24760 .count()
24761 };
24762
24763 if path.ends_with("/poll") && request_number <= behavior.poll_failures_per_path {
24764 return;
24765 }
24766 let pressure_path = behavior
24767 .storage_path
24768 .is_some_and(|part| path.contains(part));
24769 let prior_failures = if path.ends_with("/poll") {
24770 behavior.poll_failures_per_path
24771 } else {
24772 0
24773 };
24774 if pressure_path
24775 && request_number.saturating_sub(prior_failures) <= behavior.storage_refusals
24776 {
24777 let request_body: Value = serde_json::from_str(body).unwrap_or(Value::Null);
24778 let poll_id = path
24779 .ends_with("/poll")
24780 .then(|| request_body["poll_request_id"].as_str().unwrap_or(""));
24781 let mut refusal = storage_refusal(
24782 poll_id,
24783 behavior.storage_unavailable,
24784 behavior.storage_mid_poll,
24785 );
24786 if behavior.storage_wrong_poll_id {
24787 refusal["poll_request_id"] = json!("wrong-poll");
24788 }
24789 write_mock_response(stream, "503 Service Unavailable", &refusal.to_string());
24790 return;
24791 }
24792 if path.contains("/storage-task/")
24793 || path.contains("/storage-activity/")
24794 || path.contains("/storage-query/")
24795 {
24796 write_mock_response(stream, "200 OK", "{}");
24797 return;
24798 }
24799 if behavior.storage_query && path == "/api/worker/query-tasks/poll" && request_number == 1 {
24800 write_mock_response(stream, "200 OK", &json!({"task":{
24801 "query_task_id":"storage-query", "query_task_attempt":7, "workflow_type":"storage.workflow",
24802 "query_name":"state", "workflow_id":"workflow", "run_id":"run", "payload_codec":"avro",
24803 "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC).unwrap(),
24804 "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC).unwrap(),
24805 "history_events":[], "run_status":"waiting", "lease_owner":"storage-worker"
24806 }}).to_string());
24807 return;
24808 }
24809 if behavior.storage_activity
24810 && path == "/api/worker/activity-tasks/poll"
24811 && request_number == 1
24812 {
24813 write_mock_response(stream, "200 OK", &json!({"task":{
24814 "task_id":"storage-activity", "activity_attempt_id":"storage-attempt", "activity_type":"storage.activity",
24815 "payload_codec":"avro", "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC).unwrap(),
24816 "attempt_number":7, "lease_owner":"storage-worker"
24817 }}).to_string());
24818 return;
24819 }
24820
24821 if path == "/api/worker/register" {
24822 if behavior.reject_registration_protocol {
24823 write_mock_response(
24824 stream,
24825 "400 Bad Request",
24826 r#"{"reason":"unsupported_protocol_version","message":"condition-wait occurrence identity requires worker protocol 1.17","supported_version":"1.16","requested_version":"1.17"}"#,
24827 );
24828 return;
24829 }
24830 if behavior.reject_registration {
24831 write_mock_response(
24832 stream,
24833 "503 Service Unavailable",
24834 r#"{"reason":"registration_unavailable","message":"registration failed"}"#,
24835 );
24836 return;
24837 }
24838 }
24839
24840 if path.starts_with("/api/worker/registrations/") {
24841 if behavior.reject_deregistration_protocol {
24842 write_mock_response(
24843 stream,
24844 "400 Bad Request",
24845 r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.17","requested_version":"1.19"}"#,
24846 );
24847 } else if behavior.reject_deregistration {
24848 write_mock_response(
24849 stream,
24850 "403 Forbidden",
24851 r#"{"reason":"authorization_failed","message":"worker cannot deregister"}"#,
24852 );
24853 } else {
24854 write_mock_response(
24855 stream,
24856 "200 OK",
24857 r#"{"worker_id":"deregistered-worker","outcome":"deregistered","recovered_workflow_task_count":2}"#,
24858 );
24859 }
24860 return;
24861 }
24862
24863 let is_poll = matches!(
24864 path,
24865 "/api/worker/workflow-tasks/poll"
24866 | "/api/worker/activity-tasks/poll"
24867 | "/api/worker/query-tasks/poll"
24868 );
24869 if is_poll && request_number <= behavior.long_poll_capacity_responses_per_path {
24870 write_mock_response(
24871 stream,
24872 "429 Too Many Requests",
24873 r#"{"task":null,"poll_status":"long_poll_capacity_exhausted","reason":"long_poll_capacity_exhausted","retryable":true,"retry_after_seconds":1}"#,
24874 );
24875 return;
24876 }
24877 if is_poll && request_number <= behavior.poll_failures_per_path {
24878 return;
24879 }
24880 if path == "/api/worker/heartbeat" && request_number <= behavior.heartbeat_failures {
24881 return;
24882 }
24883 if path == "/api/worker/heartbeat"
24884 && behavior.heartbeat_failure_request == Some(request_number)
24885 {
24886 return;
24887 }
24888 if path == "/api/worker/heartbeat"
24889 && behavior.delayed_heartbeat_request == Some(request_number)
24890 {
24891 thread::sleep(behavior.heartbeat_response_delay);
24892 }
24893 if behavior.unauthorized_polls && is_poll {
24894 write_mock_response(
24895 stream,
24896 "401 Unauthorized",
24897 r#"{"reason":"authentication_failed","message":"invalid worker token"}"#,
24898 );
24899 return;
24900 }
24901 if behavior.draining_polls && is_poll {
24902 write_mock_response(
24903 stream,
24904 "409 Conflict",
24905 r#"{"task":null,"poll_status":"draining","reason":"worker_draining","worker_status":"draining","drain_intent":"draining"}"#,
24906 );
24907 return;
24908 }
24909
24910 if let Some(codec_case) = behavior.invalid_task_payload_codec {
24911 if is_poll && request_number == 1 {
24912 let mut task = match path {
24913 "/api/worker/workflow-tasks/poll" => json!({
24914 "task_id": "codec-workflow",
24915 "workflow_type": "codec.workflow",
24916 "payload_codec": DEFAULT_CODEC,
24917 "workflow_task_attempt": 1,
24918 "lease_owner": "codec-worker"
24919 }),
24920 "/api/worker/activity-tasks/poll" => json!({
24921 "task_id": "codec-activity",
24922 "activity_attempt_id": "codec-activity-attempt",
24923 "activity_type": "codec.activity",
24924 "payload_codec": DEFAULT_CODEC,
24925 "attempt_number": 1,
24926 "lease_owner": "codec-worker"
24927 }),
24928 "/api/worker/query-tasks/poll" => json!({
24929 "query_task_id": "codec-query",
24930 "query_task_attempt": 1,
24931 "workflow_type": "codec.workflow",
24932 "query_name": "known",
24933 "payload_codec": DEFAULT_CODEC,
24934 "lease_owner": "codec-worker"
24935 }),
24936 _ => unreachable!("is_poll limits task codec probe paths"),
24937 };
24938 codec_case.apply(&mut task);
24939 write_mock_response(stream, "200 OK", &json!({"task": task}).to_string());
24940 return;
24941 }
24942
24943 if matches!(
24944 path,
24945 "/api/worker/workflow-tasks/codec-workflow/fail"
24946 | "/api/worker/activity-tasks/codec-activity/fail"
24947 | "/api/worker/query-tasks/codec-query/fail"
24948 ) {
24949 write_mock_response(stream, "200 OK", r#"{"outcome":"failed"}"#);
24950 return;
24951 }
24952 }
24953
24954 if behavior.reject_query_protocol && path.starts_with("/api/worker/query-tasks/") {
24955 let requested_version = worker_protocol.as_deref().unwrap_or("missing");
24956 let body = format!(
24957 r#"{{"reason":"unsupported_protocol_version","message":"Query tasks require worker protocol 1.8 or newer.","supported_version":"1.7","requested_version":"{requested_version}"}}"#
24958 );
24959 write_mock_response(stream, "400 Bad Request", &body);
24960 return;
24961 }
24962
24963 if behavior.reject_query_completion && path == "/api/worker/query-tasks/query-late/complete"
24964 {
24965 write_mock_response(
24966 stream,
24967 "409 Conflict",
24968 r#"{"reason":"query_task_timed_out","message":"query task timed out before completion"}"#,
24969 );
24970 return;
24971 }
24972
24973 if behavior.workflow_completion_status.is_some()
24974 && path == "/api/worker/workflow-tasks/poll"
24975 && request_number == 1
24976 {
24977 write_mock_response(
24978 stream,
24979 "200 OK",
24980 r#"{"task":{"task_id":"workflow-timeout-task","workflow_id":"reused-workflow-id","run_id":"run-selected-timeout","workflow_type":"timeout.workflow","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"workflow_task_attempt":3,"lease_owner":"timeout-worker"}}"#,
24981 );
24982 return;
24983 }
24984
24985 if path == "/api/worker/workflow-tasks/workflow-timeout-task/complete" {
24986 if let (Some(status), Some(body)) = (
24987 behavior.workflow_completion_status,
24988 behavior.workflow_completion_body,
24989 ) {
24990 write_mock_response(stream, status, body);
24991 return;
24992 }
24993 }
24994
24995 if behavior.waiting_query_worker {
24996 if behavior.complete_named_signal
24997 && path == "/api/worker/workflow-tasks/poll"
24998 && request_number == 1
24999 {
25000 let body = json!({
25001 "task": {
25002 "task_id": "snapshot-open",
25003 "workflow_id": "snapshot-1",
25004 "run_id": "snapshot-run-1",
25005 "workflow_type": "snapshot",
25006 "payload_codec": DEFAULT_CODEC,
25007 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25008 .expect("Avro workflow arguments"),
25009 "history_events": [],
25010 "workflow_task_attempt": 1,
25011 "lease_owner": "rust-snapshot-worker"
25012 }
25013 })
25014 .to_string();
25015 write_mock_response(stream, "200 OK", &body);
25016 return;
25017 }
25018
25019 let signal_request = request_number - usize::from(behavior.complete_named_signal);
25020 let signal_request_limit = 2 + usize::from(behavior.complete_named_signal);
25021 if path == "/api/worker/workflow-tasks/poll"
25022 && signal_request >= 1
25023 && signal_request <= signal_request_limit
25024 {
25025 let finish = behavior.complete_named_signal && signal_request == 3;
25026 let amounts = if signal_request == 1 {
25027 vec![3]
25028 } else {
25029 vec![3, 5]
25030 };
25031 let task_id = if signal_request == 1 {
25032 "snapshot-wait-3"
25033 } else if finish {
25034 "snapshot-finish"
25035 } else {
25036 "snapshot-wait-5"
25037 };
25038 let mut history_events = std::iter::once(json!({
25039 "event_type": "SignalWaitOpened",
25040 "payload": {"sequence": 1, "signal_name": "finish"}
25041 }))
25042 .chain(amounts.iter().enumerate().map(|(index, amount)| {
25043 json!({
25044 "event_type": "SignalReceived",
25045 "payload": {
25046 "signal_id": format!("increment-{amount}"),
25047 "signal_name": "increment",
25048 "workflow_sequence": index + 2,
25049 "payload_codec": DEFAULT_CODEC,
25050 "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
25051 .expect("Avro signal envelope")
25052 }
25053 })
25054 }))
25055 .collect::<Vec<_>>();
25056 let (resume_id, resume_name, resume_arguments) = if finish {
25057 history_events.push(json!({
25058 "event_type": "SignalReceived",
25059 "payload": {
25060 "signal_id": "finish",
25061 "signal_name": "finish",
25062 "workflow_sequence": 4,
25063 "payload_codec": DEFAULT_CODEC,
25064 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25065 .expect("Avro finish signal envelope")
25066 }
25067 }));
25068 (
25069 "finish".to_string(),
25070 "finish".to_string(),
25071 encode_value_envelope(&json!([]), DEFAULT_CODEC)
25072 .expect("Avro finish resume signal"),
25073 )
25074 } else {
25075 let amount = amounts.last().expect("amount");
25076 (
25077 format!("increment-{amount}"),
25078 "increment".to_string(),
25079 encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
25080 .expect("Avro increment resume signal"),
25081 )
25082 };
25083 let body = json!({
25084 "task": {
25085 "task_id": task_id,
25086 "workflow_id": "snapshot-1",
25087 "run_id": "snapshot-run-1",
25088 "workflow_type": "snapshot",
25089 "payload_codec": DEFAULT_CODEC,
25090 "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25091 .expect("Avro workflow arguments"),
25092 "history_events": history_events,
25093 "workflow_task_attempt": 1,
25094 "workflow_signal_id": resume_id,
25095 "signal_name": resume_name,
25096 "signal_arguments": resume_arguments,
25097 "lease_owner": "rust-snapshot-worker"
25098 }
25099 })
25100 .to_string();
25101 write_mock_response(stream, "200 OK", &body);
25102 return;
25103 }
25104
25105 if path == "/api/worker/query-tasks/poll" && request_number == 1 {
25106 let history_events = [3, 5]
25107 .into_iter()
25108 .enumerate()
25109 .map(|(index, amount)| {
25110 json!({
25111 "event_type": "SignalReceived",
25112 "payload": {
25113 "signal_id": format!("increment-{amount}"),
25114 "signal_name": "increment",
25115 "workflow_sequence": index + 2,
25116 "payload_codec": DEFAULT_CODEC,
25117 "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
25118 .expect("Avro query signal envelope")
25119 }
25120 })
25121 })
25122 .collect::<Vec<_>>();
25123 let body = json!({
25124 "task": {
25125 "query_task_id": "snapshot-current",
25126 "query_task_attempt": 1,
25127 "lease_owner": "rust-snapshot-worker",
25128 "workflow_id": "snapshot-1",
25129 "run_id": "snapshot-run-1",
25130 "workflow_type": "snapshot",
25131 "query_name": "current",
25132 "payload_codec": DEFAULT_CODEC,
25133 "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25134 .expect("Avro workflow arguments"),
25135 "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25136 .expect("Avro query arguments"),
25137 "history_events": history_events,
25138 "run_status": "waiting"
25139 }
25140 })
25141 .to_string();
25142 write_mock_response(stream, "200 OK", &body);
25143 return;
25144 }
25145
25146 if path == "/api/worker/workflow-tasks/snapshot-wait-3/fail"
25147 || path == "/api/worker/workflow-tasks/snapshot-wait-5/fail"
25148 {
25149 write_mock_response(
25150 stream,
25151 "200 OK",
25152 r#"{"outcome":"waiting_for_history","recorded":true}"#,
25153 );
25154 return;
25155 }
25156
25157 if path == "/api/worker/workflow-tasks/snapshot-open/complete" {
25158 write_mock_response(stream, "200 OK", r#"{"outcome":"waiting","recorded":true}"#);
25159 return;
25160 }
25161
25162 if path == "/api/worker/workflow-tasks/snapshot-finish/complete" {
25163 write_mock_response(
25164 stream,
25165 "200 OK",
25166 r#"{"outcome":"completed","run_status":"completed","recorded":true}"#,
25167 );
25168 return;
25169 }
25170
25171 if path == "/api/worker/query-tasks/snapshot-current/complete" {
25172 write_mock_response(stream, "200 OK", r#"{"outcome":"completed"}"#);
25173 return;
25174 }
25175 }
25176
25177 if matches!(
25178 path,
25179 "/api/workflows/typed-1/query/inspect" | "/api/workflows/typed-1/update/replace"
25180 ) {
25181 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
25182 .expect("typed mock result");
25183 let body = json!({
25184 "result": typed_fidelity_probe().into_json().expect("result projection"),
25185 "result_envelope": result,
25186 })
25187 .to_string();
25188 write_mock_response(stream, "200 OK", &body);
25189 return;
25190 }
25191
25192 if path == "/api/workflows/typed-1" {
25193 let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
25194 .expect("typed mock result");
25195 let body = json!({
25196 "workflow_id": "typed-1",
25197 "run_id": "run-typed-1",
25198 "workflow_type": "typed.echo",
25199 "status": "completed",
25200 "output": typed_fidelity_probe().into_json().expect("output projection"),
25201 "output_envelope": result,
25202 })
25203 .to_string();
25204 write_mock_response(stream, "200 OK", &body);
25205 return;
25206 }
25207
25208 let (status, body) = match path {
25209 "/api/health" => ("200 OK", r#"{"status":"ok"}"#),
25210 "/api/workflows" => (
25211 "201 Created",
25212 r#"{"workflow_id":"wf-start-options","run_id":"run-start-options","workflow_type":"rust.timeout"}"#,
25213 ),
25214 "/api/worker/register" if behavior.decline_registration => (
25215 "200 OK",
25216 r#"{"worker_id":"declined-worker","registered":false}"#,
25217 ),
25218 "/api/worker/register" if behavior.waiting_query_worker => (
25219 "200 OK",
25220 r#"{"worker_id":"rust-snapshot-worker","registered":true,"heartbeat_interval_seconds":1}"#,
25221 ),
25222 "/api/worker/register" => (
25223 "200 OK",
25224 r#"{"worker_id":"mock-worker","registered":true,"heartbeat_interval_seconds":3600}"#,
25225 ),
25226 "/api/worker/heartbeat" => ("200 OK", "{}"),
25227 "/api/worker/activity-tasks/poll"
25228 if behavior.cancelled_activity && request_number == 1 =>
25229 {
25230 (
25231 "200 OK",
25232 r#"{"task":{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","activity_type":"cancel-aware","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"rust-cancel-worker"}}"#,
25233 )
25234 }
25235 "/api/worker/activity-tasks/poll"
25236 if behavior.long_poll_capacity_responses_per_path > 0
25237 && request_number
25238 == behavior
25239 .long_poll_capacity_responses_per_path
25240 .saturating_add(1) =>
25241 {
25242 (
25243 "200 OK",
25244 r#"{"task":{"task_id":"capacity-activity","activity_attempt_id":"capacity-attempt","activity_type":"capacity.activity","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"capacity-worker"}}"#,
25245 )
25246 }
25247 "/api/worker/activity-tasks/poll" | "/api/worker/workflow-tasks/poll" => {
25248 ("200 OK", r#"{"task":null}"#)
25249 }
25250 "/api/worker/query-tasks/poll"
25251 if behavior.reject_query_completion && request_number == 1 =>
25252 {
25253 (
25254 "200 OK",
25255 r#"{"task":{"query_task_id":"query-late","query_task_attempt":1,"lease_owner":"late-worker","workflow_id":"counter-late","run_id":"run-late","workflow_type":"counter","query_name":"current","payload_codec":"avro","workflow_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"query_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"run_status":"running"}}"#,
25256 )
25257 }
25258 "/api/worker/query-tasks/poll" => ("200 OK", r#"{"task":null}"#),
25259 "/api/worker/query-tasks/query-capture/complete"
25260 | "/api/worker/query-tasks/query-capture/fail" => ("200 OK", "{}"),
25261 "/api/worker/activity-tasks/activity-cancel/heartbeat" => (
25262 "200 OK",
25263 r#"{"activity_attempt_id":"attempt-cancel","cancel_requested":true,"can_continue":false,"reason":"run_cancelled","run_closed_reason":"cancelled","heartbeat_recorded":false}"#,
25264 ),
25265 "/api/worker/activity-tasks/activity-cancel/complete" => (
25266 "409 Conflict",
25267 r#"{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","reason":"run_cancelled","cancel_requested":true,"can_continue":false,"run_closed_reason":"cancelled"}"#,
25268 ),
25269 "/api/worker/activity-tasks/activity-typed/complete"
25270 | "/api/worker/activity-tasks/activity-typed/fail"
25271 | "/api/worker/activity-tasks/capacity-activity/complete"
25272 | "/api/workflows/typed-1/signal/changed" => ("200 OK", "{}"),
25273 "/api/workflows/counter-1/query/current" => (
25274 "200 OK",
25275 r#"{"workflow_id":"counter-1","query_name":"current","result":{"count":8},"result_envelope":{"codec":"avro","blob":"wwHioz3/VYAiNw4CCmNvdW50BBAA"}}"#,
25276 ),
25277 "/api/workflows/counter-1/query/missing" => (
25278 "404 Not Found",
25279 r#"{"workflow_id":"counter-1","query_name":"missing","reason":"rejected_unknown_query","message":"unknown query"}"#,
25280 ),
25281 "/api/workflows/wf-lifecycle/cancel" => (
25282 "200 OK",
25283 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","reason":"cleanup requested","command_status":"accepted"}"#,
25284 ),
25285 "/api/workflows/wf-lifecycle/terminate" => (
25286 "200 OK",
25287 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","reason":"forced stop","command_status":"accepted"}"#,
25288 ),
25289 "/api/workflows/wf-lifecycle/runs/run-current/cancel" => (
25290 "200 OK",
25291 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","command_status":"accepted"}"#,
25292 ),
25293 "/api/workflows/wf-lifecycle/runs/run-current/terminate" => (
25294 "200 OK",
25295 r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","command_status":"accepted"}"#,
25296 ),
25297 "/api/workflows/wf-lifecycle/runs/run-stale/cancel"
25298 | "/api/workflows/wf-lifecycle/runs/run-stale/terminate" => (
25299 "409 Conflict",
25300 r#"{"workflow_id":"wf-lifecycle","run_id":"run-stale","reason":"historical_run_command_rejected","target_scope":"run","message":"Commands cannot target historical runs."}"#,
25301 ),
25302 "/api/workflows/wf-failed" | "/api/workflows/wf-failed/runs/run-failed" => (
25303 "200 OK",
25304 r#"{"workflow_id":"wf-failed","run_id":"run-failed","status":"failed","closed_reason":"failed","error":"payment failed","failure":{"message":"payment failed","failure_category":"application","exception_type":"PaymentError","exception_class":"billing::PaymentError","non_retryable":true,"exception":{"type":"PaymentError","class":"billing::PaymentError","message":"payment failed"},"failures":[{"id":"failure-17","failure_category":"application"}]}}"#,
25305 ),
25306 "/api/workflows/wf-cancelled" => (
25307 "200 OK",
25308 r#"{"workflow_id":"wf-cancelled","run_id":"run-cancelled","status":"cancelled","closed_reason":"cancelled","reason":"cleanup requested"}"#,
25309 ),
25310 "/api/workflows/wf-terminated" => (
25311 "200 OK",
25312 r#"{"workflow_id":"wf-terminated","run_id":"run-terminated","status":"terminated","closed_reason":"terminated","reason":"forced stop"}"#,
25313 ),
25314 "/api/workflows/wf-timed-out" => (
25315 "200 OK",
25316 r#"{"workflow_id":"wf-timed-out","run_id":"run-timed-out","status":"failed","closed_reason":"timed_out","reason":"run_timeout"}"#,
25317 ),
25318 "/api/workflows/wf-waiting" | "/api/workflows/wf-waiting/runs/run-waiting" => (
25319 "200 OK",
25320 r#"{"workflow_id":"wf-waiting","run_id":"run-waiting","status":"waiting"}"#,
25321 ),
25322 "/api/workflows/wf-selected" => (
25323 "200 OK",
25324 r#"{"workflow_id":"wf-selected","run_id":"run-current","status":"completed","output":"current run output"}"#,
25325 ),
25326 "/api/workflows/wf-selected/runs/run-selected" => (
25327 "200 OK",
25328 r#"{"workflow_id":"wf-selected","run_id":"run-selected","status":"cancelled","closed_reason":"cancelled","reason":"selected run cancelled"}"#,
25329 ),
25330 _ => ("404 Not Found", r#"{"message":"not found"}"#),
25331 };
25332 write_mock_response(stream, status, body);
25333 }
25334
25335 fn mock_request_is_complete(request: &[u8]) -> bool {
25336 let Some(header_end) = request
25337 .windows(4)
25338 .position(|window| window == b"\r\n\r\n")
25339 .map(|position| position + 4)
25340 else {
25341 return false;
25342 };
25343 let headers = String::from_utf8_lossy(&request[..header_end]);
25344 let content_length = headers.lines().find_map(|line| {
25345 let (name, value) = line.split_once(':')?;
25346 name.eq_ignore_ascii_case("content-length")
25347 .then(|| value.trim().parse::<usize>().ok())
25348 .flatten()
25349 });
25350
25351 request.len() >= header_end + content_length.unwrap_or(0)
25352 }
25353
25354 fn write_mock_response(stream: &mut TcpStream, status: &str, body: &str) {
25355 let response = format!(
25356 "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
25357 body.len()
25358 );
25359
25360 let _ = stream.write_all(response.as_bytes());
25361 let _ = stream.flush();
25362 }
25363}